From dc7c79358c0a5c890506813ba939c31b7b849a5d Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Thu, 2 Jul 2026 22:13:00 -0500 Subject: [PATCH 1/6] Refactor virtual drive sync engine --- .../registerTemporalFilesServices.ts | 16 + .../virtual-drive/registerFilesServices.ts | 2 +- .../virtual-drive/registerFolderServices.ts | 2 +- .../registerVirtualDriveSharedServices.ts | 18 ++ src/apps/main/database/data-source.ts | 32 ++ .../remote-sync/RemoteSyncManager.test.ts | 6 +- .../main/remote-sync/RemoteSyncManager.ts | 33 +- src/apps/main/remote-sync/service.ts | 4 +- .../features/virtual-drive/ipc/handlers.ts | 8 +- .../virtual-drive.service.test.ts | 19 +- .../drive-folder/virtual-drive.service.ts | 27 +- .../lazy/DirectoryStateSqliteRepository.ts | 70 ++++ .../services/lazy/LazyVirtualDriveHydrator.ts | 305 ++++++++++++++++++ .../operations/get-attributes.service.test.ts | 16 +- .../operations/get-attributes.service.ts | 66 +++- .../services/operations/open.service.ts | 10 +- .../operations/opendir.service.test.ts | 32 +- .../services/operations/opendir.service.ts | 24 +- .../operations/release.service.test.ts | 96 +----- .../services/operations/release.service.ts | 24 +- .../upload/TemporalFileUploadQueue.ts | 124 +++++++ .../domain/TemporalFileRepository.ts | 2 + .../NodeTemporalFileRepository.ts | 25 ++ .../files/domain/FileRepository.ts | 2 + .../infrastructure/InMemoryFileRepository.ts | 13 + .../folders/domain/FolderRepository.ts | 2 + .../InMemoryFolderRepository.ts | 12 + .../register-session-event-handlers.ts | 3 +- src/core/electron/paths.ts | 2 + .../file/create-or-update-file-by-batch.ts | 19 +- .../create-or-update-folder-by-batch.ts | 19 +- 31 files changed, 826 insertions(+), 207 deletions(-) create mode 100644 src/backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts diff --git a/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts b/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts index 99b19e45a1..9e18c42ebe 100644 --- a/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts +++ b/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts @@ -9,12 +9,17 @@ import { TemporalFileByFolderFinder } from '../../../../context/storage/Temporal import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFilePathsByFolderFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFilePathsByFolderFinder'; import { TemporalFileTruncater } from '../../../../context/storage/TemporalFiles/application/truncate/TemporalFileTruncater'; +import { + createTemporalFileUploadQueue, + TemporalFileUploadQueue, +} from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; import { TemporalFileUploader } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader'; import { TemporalFileWriter } from '../../../../context/storage/TemporalFiles/application/write/TemporalFileWriter'; import { TemporalFileRepository } from '../../../../context/storage/TemporalFiles/domain/TemporalFileRepository'; import { TemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/domain/upload/TemporalFileUploaderFactory'; import { NodeTemporalFileRepository } from '../../../../context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository'; import { EnvironmentTemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/infrastructure/upload/EnvironmentTemporalFileUploaderFactory'; +import { FirstsFileSearcher } from '../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; import { PATHS } from '../../../../core/electron/paths'; @@ -52,6 +57,17 @@ export async function registerTemporalFilesServices(builder: ContainerBuilder) { builder.registerAndUse(TemporalFileTruncater); builder.registerAndUse(TemporalFileByteByByteComparator); builder.registerAndUse(TemporalFileByFolderFinder); + builder + .register(TemporalFileUploadQueue) + .useFactory((container) => + createTemporalFileUploadQueue({ + repository: container.get(TemporalFileRepository), + uploader: container.get(TemporalFileUploader), + deleter: container.get(TemporalFileDeleter), + fileSearcher: container.get(FirstsFileSearcher), + }), + ) + .asSingleton(); // Event handlers builder.registerAndUse(DeleteTemporalFileOnFileCreated).addTag('event-handler'); diff --git a/src/apps/drive/dependency-injection/virtual-drive/registerFilesServices.ts b/src/apps/drive/dependency-injection/virtual-drive/registerFilesServices.ts index 05fa91b619..be0549da35 100644 --- a/src/apps/drive/dependency-injection/virtual-drive/registerFilesServices.ts +++ b/src/apps/drive/dependency-injection/virtual-drive/registerFilesServices.ts @@ -25,7 +25,7 @@ import { Environment } from '@internxt/inxt-js'; export async function registerFilesServices(builder: ContainerBuilder): Promise { // Infra - builder.register(FileRepository).use(InMemoryFileRepository).asSingleton().private(); + builder.register(FileRepository).use(InMemoryFileRepository).asSingleton(); const user = DependencyInjectionUserProvider.get(); diff --git a/src/apps/drive/dependency-injection/virtual-drive/registerFolderServices.ts b/src/apps/drive/dependency-injection/virtual-drive/registerFolderServices.ts index b89511491f..3060eaadb0 100644 --- a/src/apps/drive/dependency-injection/virtual-drive/registerFolderServices.ts +++ b/src/apps/drive/dependency-injection/virtual-drive/registerFolderServices.ts @@ -27,7 +27,7 @@ export async function registerFolderServices(builder: ContainerBuilder): Promise builder.register(SyncFolderMessenger).use(MainProcessSyncFolderMessenger); // TODO: can be private? - builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton().private(); + builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton(); builder.register(RemoteFileSystem).use(HttpRemoteFileSystem).private(); diff --git a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts index 63693d0c37..62a31e0e1c 100644 --- a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts +++ b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts @@ -1,11 +1,29 @@ import { ContainerBuilder } from 'diod'; import { AbsolutePathToRelativeConverter } from '../../../../context/virtual-drive/shared/application/AbsolutePathToRelativeConverter'; import { RelativePathToAbsoluteConverter } from '../../../../context/virtual-drive/shared/application/RelativePathToAbsoluteConverter'; +import { FileRepository } from '../../../../context/virtual-drive/files/domain/FileRepository'; +import { FolderRepository } from '../../../../context/virtual-drive/folders/domain/FolderRepository'; import { PATHS } from '../../../../core/electron/paths'; +import { + createDirectoryStateSqliteRepository, + DirectoryStateSqliteRepository, +} from '../../../../backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository'; +import { createLazyVirtualDriveHydrator, LazyVirtualDriveHydrator } from '../../../../backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator'; export async function registerVirtualDriveSharedServices(builder: ContainerBuilder): Promise { const downloaded = PATHS.DOWNLOADED; builder.register(RelativePathToAbsoluteConverter).useFactory(() => new RelativePathToAbsoluteConverter(downloaded)); builder.register(AbsolutePathToRelativeConverter).useFactory(() => new AbsolutePathToRelativeConverter(downloaded)); + builder.register(DirectoryStateSqliteRepository).useFactory(() => createDirectoryStateSqliteRepository()).asSingleton(); + builder + .register(LazyVirtualDriveHydrator) + .useFactory((container) => + createLazyVirtualDriveHydrator({ + folderRepository: container.get(FolderRepository), + fileRepository: container.get(FileRepository), + directoryStateRepository: container.get(DirectoryStateSqliteRepository), + }), + ) + .asSingleton(); } diff --git a/src/apps/main/database/data-source.ts b/src/apps/main/database/data-source.ts index 7befd873f4..41477e2036 100644 --- a/src/apps/main/database/data-source.ts +++ b/src/apps/main/database/data-source.ts @@ -6,6 +6,28 @@ import { TypeOrmStorageFile } from '../../../context/storage/StorageFiles/infras import { logger } from '@internxt/drive-desktop-core/build/backend'; import { PATHS } from '../../../core/electron/paths'; +const SQLITE_BOOTSTRAP_STATEMENTS = [ + 'PRAGMA journal_mode=WAL;', + 'PRAGMA synchronous=NORMAL;', + 'PRAGMA temp_store=MEMORY;', + 'PRAGMA foreign_keys=ON;', + 'PRAGMA busy_timeout=5000;', + 'PRAGMA wal_autocheckpoint=1000;', + 'PRAGMA mmap_size=268435456;', + `CREATE TABLE IF NOT EXISTS drive_directory_state ( + folder_id INTEGER NOT NULL, + status_scope TEXT NOT NULL, + children_loaded_at TEXT NULL, + children_last_error_at TEXT NULL, + fetch_state TEXT NOT NULL DEFAULT 'idle', + PRIMARY KEY (folder_id, status_scope) + );`, + 'CREATE INDEX IF NOT EXISTS idx_drive_file_folder_status_updated ON drive_file(folderId, status, updatedAt);', + 'CREATE INDEX IF NOT EXISTS idx_drive_file_status_updated ON drive_file(status, updatedAt);', + 'CREATE INDEX IF NOT EXISTS idx_drive_folder_parent_status_updated ON drive_folder(parentId, status, updatedAt);', + 'CREATE INDEX IF NOT EXISTS idx_drive_folder_status_updated ON drive_folder(status, updatedAt);', +]; + export const AppDataSource = new DataSource({ type: 'better-sqlite3', database: PATHS.DATABASE, @@ -16,6 +38,16 @@ export const AppDataSource = new DataSource({ logger.debug({ msg: `Using database file at ${PATHS.DATABASE}` }); +export async function initializeVirtualDriveSqlite() { + if (!AppDataSource.isInitialized) { + return; + } + + for (const statement of SQLITE_BOOTSTRAP_STATEMENTS) { + await AppDataSource.query(statement); + } +} + export async function resetAppDataSourceOnLogout() { if (!AppDataSource.isInitialized) { return; diff --git a/src/apps/main/remote-sync/RemoteSyncManager.test.ts b/src/apps/main/remote-sync/RemoteSyncManager.test.ts index b69763e36f..8e0c34c5d1 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.test.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.test.ts @@ -158,6 +158,7 @@ describe('RemoteSyncManager', () => { await sut.startRemoteSync(); expect(mockedGet).toBeCalledTimes(2); + expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } }); expect(sut.getSyncStatus()).toBe('SYNCED'); }); it('Should sync all the folders', async () => { @@ -207,6 +208,7 @@ describe('RemoteSyncManager', () => { await sut.startRemoteSync(); expect(mockedGet).toBeCalledTimes(3); + expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } }); expect(sut.getSyncStatus()).toBe('SYNCED'); }); @@ -252,7 +254,7 @@ describe('RemoteSyncManager', () => { expect(mockedGet).toBeCalledTimes(6); expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); - }); + }, 15000); it('Should fail the sync if some files or folders cannot be retrieved', async () => { mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); @@ -261,7 +263,7 @@ describe('RemoteSyncManager', () => { expect(mockedGet).toBeCalledTimes(6); expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); - }); + }, 15000); it('should handle the error while syncing files by calling the error handler properly', async () => { const sut = new RemoteSyncManager( diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index a07a57aa1d..8c76a8fab9 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -93,20 +93,19 @@ export class RemoteSyncManager { logger.debug({ tag: 'SYNC-ENGINE', msg: 'Starting' }); this.changeStatus('SYNCING'); try { - await Promise.all([ - this.config.syncFiles - ? this.syncRemoteFiles({ - retry: 1, - maxRetries: 3, - }) - : Promise.resolve(), - this.config.syncFolders - ? this.syncRemoteFolders({ - retry: 1, - maxRetries: 3, - }) - : Promise.resolve(), - ]); + if (this.config.syncFolders) { + await this.syncRemoteFolders({ + retry: 1, + maxRetries: 3, + }); + } + + if (this.config.syncFiles) { + await this.syncRemoteFiles({ + retry: 1, + maxRetries: 3, + }); + } } catch (error) { this.changeStatus('SYNC_FAILED'); logger.error({ @@ -333,7 +332,7 @@ export class RemoteSyncManager { const { data, error } = await fetchFiles({ limit: this.config.fetchFilesLimitPerRequest, offset: 0, - status: 'ALL', + status: 'EXISTS', updatedAt: updatedAtCheckpoint?.toISOString(), }); @@ -357,9 +356,9 @@ export class RemoteSyncManager { result: RemoteSyncedFolder[]; }> { const { data, error } = await fetchFolders({ - limit: this.config.fetchFilesLimitPerRequest, + limit: this.config.fetchFoldersLimitPerRequest, offset: 0, - status: 'ALL', + status: 'EXISTS', updatedAt: updatedAtCheckpoint?.toISOString(), }); diff --git a/src/apps/main/remote-sync/service.ts b/src/apps/main/remote-sync/service.ts index cedb750ef9..61267d829e 100644 --- a/src/apps/main/remote-sync/service.ts +++ b/src/apps/main/remote-sync/service.ts @@ -19,8 +19,8 @@ export const remoteSyncManager = new RemoteSyncManager( folders: driveFoldersCollection, }, { - fetchFilesLimitPerRequest: 1000, - fetchFoldersLimitPerRequest: 1000, + fetchFilesLimitPerRequest: 250, + fetchFoldersLimitPerRequest: 500, syncFiles: true, syncFolders: true, }, diff --git a/src/backend/features/virtual-drive/ipc/handlers.ts b/src/backend/features/virtual-drive/ipc/handlers.ts index 121b09ae61..a2cd92ff85 100644 --- a/src/backend/features/virtual-drive/ipc/handlers.ts +++ b/src/backend/features/virtual-drive/ipc/handlers.ts @@ -9,11 +9,15 @@ import { updateVirtualDriveContainer } from '../services/update-virtual-drive-co import { DependencyInjectionUserProvider } from '../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { getVirtualDriveState } from '../services/daemon.service'; +import { DirectoryStateSqliteRepository } from '../services/lazy/DirectoryStateSqliteRepository'; function remoteChangesSyncedHandler() { const container = getVirtualDriveContainer(); if (container) { - updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() }); + void Promise.all([ + updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() }), + container.get(DirectoryStateSqliteRepository).clear(), + ]); } else { logger.warn({ msg: '[FUSE] updateVirtualDriveContainer called before container was initialized' }); } @@ -24,7 +28,7 @@ function syncRootChangedHandler({ oldPath, newPath }: { oldPath: string; newPath } export function registerVirtualDriveHandlers() { - eventBus.on('INITIAL_SYNC_READY', startVirtualDrive); + eventBus.on('APP_DATA_SOURCE_INITIALIZED', startVirtualDrive); eventBus.on('REMOTE_CHANGES_SYNCHED', remoteChangesSyncedHandler); eventBus.on('SYNC_ROOT_CHANGED', syncRootChangedHandler); ipcMain.handle('get-virtual-drive-status', getVirtualDriveState); diff --git a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts index 7d5bc4bf6b..3615b5821a 100644 --- a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts +++ b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts @@ -8,7 +8,6 @@ import * as serverServiceModule from '../server.service'; import * as hydrationApiServiceModule from '../hydration-api.service'; import * as hydrationStateModule from '../../../fuse/on-read/download-cache/hydration-state'; import * as virtualRootFolderModule from '../../../../../apps/main/virtual-root-folder/service'; -import * as updateVirtualDriveContainerModule from '../update-virtual-drive-container.service'; import { startVirtualDrive, stopVirtualDriveOnce, remountVirtualDriveOnRootChange } from './virtual-drive.service'; import { partialSpyOn, calls, call } from '../../../../../../tests/vitest/utils.helper'; @@ -20,13 +19,14 @@ describe('virtual-drive.service', () => { const startHydrationApi = partialSpyOn(hydrationApiServiceModule, 'startHydrationApi'); const clearHydrationState = partialSpyOn(hydrationStateModule, 'clearHydrationState'); const getRootVirtualDrive = partialSpyOn(virtualRootFolderModule, 'getRootVirtualDrive'); - const updateVirtualDriveContainer = partialSpyOn(updateVirtualDriveContainerModule, 'updateVirtualDriveContainer'); const buildContainer = partialSpyOn(DriveDependencyContainerFactory, 'build'); const getUser = partialSpyOn(DependencyInjectionUserProvider, 'get'); const deleteAll = vi.fn(); + const add = vi.fn(); + const clear = vi.fn(); const containerMock = { - get: vi.fn(() => ({ deleteAll })), + get: vi.fn(() => ({ deleteAll, add, clear })), } as unknown as Container; beforeEach(() => { @@ -36,10 +36,11 @@ describe('virtual-drive.service', () => { startFuseDaemonServer.mockResolvedValue(undefined); startHydrationApi.mockResolvedValue(undefined); getRootVirtualDrive.mockReturnValue('/mock/root/'); - getUser.mockReturnValue({} as never); - updateVirtualDriveContainer.mockResolvedValue({}); + getUser.mockReturnValue({ root_folder_id: 99, rootFolderId: '123e4567-e89b-12d3-a456-426614174000', createdAt: '2026-01-01T00:00:00.000Z' } as never); buildContainer.mockResolvedValue(containerMock); deleteAll.mockResolvedValue(undefined); + add.mockResolvedValue(undefined); + clear.mockResolvedValue(undefined); }); describe('startVirtualDrive', () => { @@ -69,6 +70,14 @@ describe('virtual-drive.service', () => { // Then call(startDaemon).toBe('/mock/root/'); }); + + it('seeds the root folder before daemon start', async () => { + // When + await startVirtualDrive(); + + // Then + expect(add.mock.invocationCallOrder[0]).toBeLessThan(startDaemon.mock.invocationCallOrder[0]); + }); }); describe('stopVirtualDriveOnce', () => { diff --git a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.ts b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.ts index 402b63ef26..c6c031e79e 100644 --- a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.ts +++ b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.ts @@ -7,14 +7,39 @@ import { updateVirtualDriveContainer } from '../update-virtual-drive-container.s import { DependencyInjectionUserProvider } from '../../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; import { clearHydrationState } from '../../../fuse/on-read/download-cache/hydration-state'; import { StorageFilesRepository } from '../../../../../context/storage/StorageFiles/domain/StorageFilesRepository'; +import { FolderRepository } from '../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { Folder } from '../../../../../context/virtual-drive/folders/domain/Folder'; import { remountVirtualDrive } from './remount-virtual-drive'; import { stopVirtualDrive } from './stop-virual-drive'; import { startHydrationApi } from '../hydration-api.service'; +import { DirectoryStateSqliteRepository } from '../lazy/DirectoryStateSqliteRepository'; let container: Container | undefined; let stopInFlight: Promise | undefined; let remountInFlight: Promise | undefined; +async function seedRootFolder() { + if (!container) { + return; + } + + const user = DependencyInjectionUserProvider.get(); + + await container.get(FolderRepository).add( + Folder.from({ + id: user.root_folder_id, + uuid: user.rootFolderId, + parentId: null, + path: '/', + updatedAt: user.createdAt, + createdAt: user.createdAt, + status: 'EXISTS', + }), + ); + + await container.get(DirectoryStateSqliteRepository).clear(); +} + export function getVirtualDriveContainer(): Container | undefined { return container; } @@ -22,7 +47,7 @@ export function getVirtualDriveContainer(): Container | undefined { export async function startVirtualDrive() { const localRoot = getRootVirtualDrive(); container = await DriveDependencyContainerFactory.build(); - await updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() }); + await seedRootFolder(); /** * Clear stale block-cache state and orphaned hydrated files before mounting. * Future virtual-drive reads recreate cache files and hydrate only requested blocks. diff --git a/src/backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository.ts b/src/backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository.ts new file mode 100644 index 0000000000..a856daa0cb --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository.ts @@ -0,0 +1,70 @@ +import { AppDataSource } from '../../../../../apps/main/database/data-source'; + +export type DirectoryStatusScope = 'EXISTS'; + +const DIRECTORY_STATE_TTL_MS = 30_000; + +type StateProps = { + folderId: number; + statusScope: DirectoryStatusScope; +}; + +type Props = { + dataSource?: typeof AppDataSource; + ttlMs?: number; +}; + +export const DirectoryStateSqliteRepository = Symbol('DirectoryStateSqliteRepository'); + +export type DirectoryStateSqliteRepository = ReturnType; + +export function createDirectoryStateSqliteRepository({ + dataSource = AppDataSource, + ttlMs = DIRECTORY_STATE_TTL_MS, +}: Props = {}) { + async function isFresh({ folderId, statusScope }: StateProps) { + const rows = (await dataSource.query( + 'SELECT children_loaded_at FROM drive_directory_state WHERE folder_id = ? AND status_scope = ? LIMIT 1', + [folderId, statusScope], + )) as Array<{ children_loaded_at: string | null }>; + + const loadedAt = rows[0]?.children_loaded_at; + + if (!loadedAt) { + return false; + } + + return Date.now() - new Date(loadedAt).getTime() < ttlMs; + } + + async function markLoaded({ folderId, statusScope }: StateProps) { + await dataSource.query( + `INSERT INTO drive_directory_state (folder_id, status_scope, children_loaded_at, children_last_error_at, fetch_state) + VALUES (?, ?, ?, NULL, 'idle') + ON CONFLICT(folder_id, status_scope) + DO UPDATE SET children_loaded_at = excluded.children_loaded_at, children_last_error_at = NULL, fetch_state = 'idle'`, + [folderId, statusScope, new Date().toISOString()], + ); + } + + async function markError({ folderId, statusScope }: StateProps) { + await dataSource.query( + `INSERT INTO drive_directory_state (folder_id, status_scope, children_loaded_at, children_last_error_at, fetch_state) + VALUES (?, ?, NULL, ?, 'error') + ON CONFLICT(folder_id, status_scope) + DO UPDATE SET children_last_error_at = excluded.children_last_error_at, fetch_state = 'error'`, + [folderId, statusScope, new Date().toISOString()], + ); + } + + async function clear() { + await dataSource.query('DELETE FROM drive_directory_state'); + } + + return { + isFresh, + markLoaded, + markError, + clear, + }; +} diff --git a/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts b/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts new file mode 100644 index 0000000000..f840fbc52c --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts @@ -0,0 +1,305 @@ +import path from 'node:path'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { fetchFolder } from '../../../../../infra/drive-server/services/folder/services/fetch-folder'; +import { FolderDto, FileDto } from '../../../../../infra/drive-server/out/dto'; +import { FolderRepository } from '../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { FileRepository } from '../../../../../context/virtual-drive/files/domain/FileRepository'; +import { Folder } from '../../../../../context/virtual-drive/folders/domain/Folder'; +import { FolderNotFoundError } from '../../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; +import { createFolderFromServerFolder } from '../../../../../context/virtual-drive/folders/application/create/FolderCreatorFromServerFolder'; +import { createFileFromServerFile } from '../../../../../context/virtual-drive/remoteTree/application/FileCreatorFromServerFile'; +import { ServerFolderStatus } from '../../../../../context/shared/domain/ServerFolder'; +import { ServerFileStatus } from '../../../../../context/shared/domain/ServerFile'; +import { createOrUpdateFolderByBatch } from '../../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; +import { createOrUpdateFileByBatch } from '../../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; +import { DirectoryStateSqliteRepository, DirectoryStatusScope } from './DirectoryStateSqliteRepository'; +import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; + +type ReadDirectoryProps = { + path: string; +}; + +type EnsurePathLoadedProps = { + path: string; +}; + +type DirectoryEntries = { + files: string[]; + folders: string[]; +}; + +type Props = { + folderRepository: FolderRepository; + fileRepository: FileRepository; + directoryStateRepository: DirectoryStateSqliteRepository; +}; + +export const LazyVirtualDriveHydrator = Symbol('LazyVirtualDriveHydrator'); + +export type LazyVirtualDriveHydrator = ReturnType; + +export function createLazyVirtualDriveHydrator({ + folderRepository, + fileRepository, + directoryStateRepository, +}: Props) { + const inflight = new Map>(); + + async function readDirectory({ path: requestedPath }: ReadDirectoryProps) { + const normalizedPath = normalizePath(requestedPath); + const statusScope: DirectoryStatusScope = 'EXISTS'; + const folder = await ensureFolderMaterialized({ path: normalizedPath, statusScope }); + + await refreshChildrenIfNeeded({ folder, statusScope }); + + return readLocalDirectory({ folder, statusScope }); + } + + async function ensurePathLoaded({ path: requestedPath }: EnsurePathLoadedProps) { + const normalizedPath = normalizePath(requestedPath); + const statusScope: DirectoryStatusScope = 'EXISTS'; + const parentPath = isRootPath(normalizedPath) ? normalizedPath : path.posix.dirname(normalizedPath); + + await ensureFolderMaterialized({ path: parentPath, statusScope }); + } + + async function ensureFolderMaterialized({ + path: requestedPath, + statusScope, + }: { + path: string; + statusScope: DirectoryStatusScope; + }): Promise { + const root = await resolveRootFolder(); + + if (isRootPath(requestedPath)) { + return root; + } + + let currentFolder = root; + let currentPath = ''; + + for (const segment of requestedPath.split('/').filter(Boolean)) { + currentPath = `${currentPath}/${segment}`; + + const existing = folderRepository.matchingPartial({ path: currentPath })[0]; + if (existing) { + currentFolder = existing; + continue; + } + + await refreshChildrenIfNeeded({ folder: currentFolder, statusScope }); + + const hydrated = folderRepository.matchingPartial({ path: currentPath })[0]; + if (!hydrated) { + throw new FuseError(FuseCodes.ENOENT, `[FUSE - Lazy] Folder not found: ${currentPath}`); + } + + currentFolder = hydrated; + } + + return currentFolder; + } + + async function refreshChildrenIfNeeded({ + folder, + statusScope, + }: { + folder: Folder; + statusScope: DirectoryStatusScope; + }) { + const cacheKey = `${folder.id}:${statusScope}`; + if (await directoryStateRepository.isFresh({ folderId: folder.id, statusScope })) { + return; + } + + const inFlight = inflight.get(cacheKey); + if (inFlight) { + return inFlight; + } + + const task = fetchAndStoreChildren({ folder, statusScope }).finally(() => { + inflight.delete(cacheKey); + }); + + inflight.set(cacheKey, task); + + return task; + } + + async function fetchAndStoreChildren({ + folder, + statusScope, + }: { + folder: Folder; + statusScope: DirectoryStatusScope; + }) { + try { + const { data, error } = await fetchFolder(folder.uuid); + + if (error || !data) { + throw error ?? new FolderNotFoundError(folder.path); + } + + const remoteFolders = data.children.filter(isExistingFolder).map((child) => toRemoteFolder(child)); + const remoteFiles = data.files.filter(isExistingFile).map((child) => toRemoteFile(child)); + + // SQLite (better-sqlite3) uses a single writer connection. + // Running both batch transactions in parallel causes nested transaction errors. + await createOrUpdateFolderByBatch({ folders: remoteFolders }); + await createOrUpdateFileByBatch({ files: remoteFiles }); + await folderRepository.deleteMatchingPartial({ parentId: folder.id }); + await fileRepository.deleteMatchingPartial({ folderId: folder.id }); + + const localFolders = data.children.filter(isExistingFolder).map((child) => { + return createFolderFromServerFolder( + { + bucket: child.bucket ?? null, + createdAt: child.createdAt, + id: child.id, + name: child.name, + parentId: child.parentId, + updatedAt: child.updatedAt, + plain_name: child.plainName, + status: ServerFolderStatus.EXISTS, + uuid: child.uuid, + }, + joinVirtualPath(folder.path, child.plainName), + ); + }); + + const localFiles = data.files.filter(isExistingFile).map((child) => { + return createFileFromServerFile( + { + bucket: child.bucket, + createdAt: child.createdAt, + encrypt_version: child.encryptVersion, + fileId: child.fileId ?? '', + folderId: child.folderId, + id: child.id, + modificationTime: child.modificationTime ?? child.updatedAt, + name: child.name, + size: Number.parseInt(String(child.size), 10), + type: child.type, + updatedAt: child.updatedAt, + userId: child.userId, + status: ServerFileStatus.EXISTS, + plainName: child.plainName, + uuid: child.uuid, + }, + joinVirtualPath(folder.path, buildFileName(child)), + ); + }); + + await Promise.all([ + ...localFolders.map((child) => folderRepository.add(child)), + ...localFiles.map((child) => fileRepository.upsert(child)), + ]); + + await directoryStateRepository.markLoaded({ folderId: folder.id, statusScope }); + } catch (error) { + await directoryStateRepository.markError({ folderId: folder.id, statusScope }); + logger.error({ msg: '[FUSE - Lazy] Failed to fetch folder children', error, path: folder.path }); + throw new FuseError(FuseCodes.EIO, `[FUSE - Lazy] Unable to hydrate path: ${folder.path}`); + } + } + + async function resolveRootFolder() { + const root = folderRepository.matchingPartial({ path: '/' })[0]; + + if (!root) { + throw new FuseError(FuseCodes.EIO, '[FUSE - Lazy] Root folder not initialized'); + } + + return root; + } + + function readLocalDirectory({ + folder, + statusScope, + }: { + folder: Folder; + statusScope: DirectoryStatusScope; + }): DirectoryEntries { + return { + folders: folderRepository + .matchingPartial({ parentId: folder.id, status: ServerFolderStatus.EXISTS }) + .map((child) => child.name), + files: fileRepository + .matchingPartial({ folderId: folder.id, status: ServerFileStatus.EXISTS }) + .map((child) => child.nameWithExtension), + }; + } + + return { + readDirectory, + ensurePathLoaded, + }; +} + +function normalizePath(pathToNormalize: string) { + const normalized = path.posix.normalize(pathToNormalize || '/'); + + if (normalized === '.') { + return '/'; + } + + return normalized.startsWith('/') ? normalized : `/${normalized}`; +} + +function isRootPath(pathToCheck: string) { + return pathToCheck === '/' || pathToCheck === ''; +} + +function joinVirtualPath(parentPath: string, name: string) { + return `${parentPath}/${name}`.replaceAll('//', '/'); +} + +function buildFileName(file: FileDto) { + return file.type ? `${file.plainName}.${file.type}` : file.plainName; +} + +function isExistingFolder(folder: FolderDto) { + return folder.status === 'EXISTS' && !folder.deleted && !folder.removed; +} + +function isExistingFile(file: FileDto) { + return file.status === 'EXISTS'; +} + +function toRemoteFolder(folder: FolderDto) { + return { + type: folder.type, + id: folder.id, + parentId: folder.parentId, + bucket: folder.bucket, + userId: folder.userId, + createdAt: folder.createdAt, + updatedAt: folder.updatedAt, + uuid: folder.uuid, + plainName: folder.plainName, + name: folder.name, + status: folder.status, + }; +} + +function toRemoteFile(file: FileDto) { + return { + id: file.id, + uuid: file.uuid, + fileId: file.fileId ?? '', + type: file.type, + size: Number.parseInt(String(file.size), 10), + bucket: file.bucket, + folderId: file.folderId, + folderUuid: file.folderUuid, + userId: file.userId, + modificationTime: file.modificationTime ?? file.updatedAt, + createdAt: file.createdAt, + updatedAt: file.updatedAt, + plainName: file.plainName, + name: file.name, + status: file.status, + }; +} diff --git a/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts b/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts index 762ce3f533..081cde4e92 100644 --- a/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts @@ -9,6 +9,7 @@ import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import type { File } from '../../../../../context/virtual-drive/files/domain/File'; import type { Folder } from '../../../../../context/virtual-drive/folders/domain/Folder'; import type { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; vi.mock('@internxt/drive-desktop-core/build/backend'); @@ -18,13 +19,19 @@ describe('getAttributes', () => { const fileSearcher = mockDeep(); const folderSearcher = mockDeep(); const temporalFinder = mockDeep(); + const hydrator = mockDeep(); beforeEach(() => { now = new Date(); container = mockDeep(); container.get.calledWith(FirstsFileSearcher).mockReturnValue(fileSearcher); + container.get.calledWith(SingleFolderMatchingSearcher).mockReturnValue(folderSearcher); + container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(temporalFinder); + container.get.calledWith(LazyVirtualDriveHydrator).mockReturnValue(hydrator); fileSearcher.run.mockResolvedValue(undefined); folderSearcher.run.mockResolvedValue(undefined); + temporalFinder.run.mockResolvedValue(undefined); + hydrator.ensurePathLoaded.mockResolvedValue(undefined); }); describe('when path is root', () => { @@ -57,7 +64,6 @@ describe('getAttributes', () => { describe('when a folder is found', () => { it('should return folder attributes', async () => { folderSearcher.run.mockResolvedValue({ createdAt: now, updatedAt: now } as unknown as Folder); - container.get.calledWith(SingleFolderMatchingSearcher).mockReturnValue(folderSearcher); const { data, error } = await getAttributes('/some/folder', container); @@ -68,10 +74,7 @@ describe('getAttributes', () => { describe('when a temporal file is found', () => { it('should return file attributes', async () => { - container.get.calledWith(SingleFolderMatchingSearcher).mockReturnValue(folderSearcher); - temporalFinder.run.mockResolvedValue({ size: { value: 2048 }, createdAt: now } as unknown as TemporalFile); - container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(temporalFinder); const { data, error } = await getAttributes('/some/temp.txt', container); @@ -82,11 +85,6 @@ describe('getAttributes', () => { describe('when nothing is found', () => { it('should return ENOENT error', async () => { - container.get.calledWith(SingleFolderMatchingSearcher).mockReturnValue(folderSearcher); - - temporalFinder.run.mockResolvedValue(undefined); - container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(temporalFinder); - const { data, error } = await getAttributes('/missing/file.txt', container); expect(data).toBeUndefined(); diff --git a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts index eb376112cd..e244b663ce 100644 --- a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts +++ b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts @@ -7,6 +7,28 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { SingleFolderMatchingSearcher } from '../../../../../context/virtual-drive/folders/application/SingleFolderMatchingSearcher'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; + +async function findLocalEntry(path: string, container: Container) { + const file = await container.get(FirstsFileSearcher).run({ + path, + status: FileStatuses.EXISTS, + }); + + if (file) { + return { file }; + } + + const folder = await container.get(SingleFolderMatchingSearcher).run({ + path, + }); + + if (folder) { + return { folder }; + } + + return {}; +} export async function getAttributes( path: string, @@ -27,10 +49,8 @@ export async function getAttributes( }; } - const file = await container.get(FirstsFileSearcher).run({ - path, - status: FileStatuses.EXISTS, - }); + const localEntry = await findLocalEntry(path, container); + const file = localEntry.file; if (file) { return { data: { @@ -45,9 +65,7 @@ export async function getAttributes( }, }; } - const folder = await container.get(SingleFolderMatchingSearcher).run({ - path, - }); + const folder = localEntry.folder; if (folder) { return { data: { @@ -65,6 +83,40 @@ export async function getAttributes( const document = await container.get(TemporalFileByPathFinder).run(path); if (document) { + + await container.get(LazyVirtualDriveHydrator).ensurePathLoaded({ path }); + + const hydratedEntry = await findLocalEntry(path, container); + if (hydratedEntry.file) { + return { + data: { + mode: FILE_MODE, + size: hydratedEntry.file.size, + ctime: hydratedEntry.file.createdAt, + mtime: hydratedEntry.file.updatedAt, + atime: new Date(), + uid: process.getuid?.() || 0, + gid: process.getgid?.() || 0, + nlink: 1, + }, + }; + } + + if (hydratedEntry.folder) { + return { + data: { + mode: FOLDER_MODE, + size: 0, + ctime: hydratedEntry.folder.createdAt, + mtime: hydratedEntry.folder.updatedAt, + atime: hydratedEntry.folder.createdAt, + uid: process.getuid?.() || 0, + gid: process.getgid?.() || 0, + nlink: 2, + }, + }; + } + return { data: { mode: FILE_MODE, diff --git a/src/backend/features/virtual-drive/services/operations/open.service.ts b/src/backend/features/virtual-drive/services/operations/open.service.ts index f7d357d458..d8cd5af300 100644 --- a/src/backend/features/virtual-drive/services/operations/open.service.ts +++ b/src/backend/features/virtual-drive/services/operations/open.service.ts @@ -6,10 +6,11 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; export async function open(path: string, processName: string, container: Container): Promise> { try { - const virtualFile = await container.get(FirstsFileSearcher).run({ path }); + let virtualFile = await container.get(FirstsFileSearcher).run({ path }); if (virtualFile) { return { data: undefined }; @@ -21,6 +22,13 @@ export async function open(path: string, processName: string, container: Contain return { data: undefined }; } + await container.get(LazyVirtualDriveHydrator).ensurePathLoaded({ path }); + virtualFile = await container.get(FirstsFileSearcher).run({ path }); + + if (virtualFile) { + return { data: undefined }; + } + const msg = `[FUSE - Open] File not found: ${path}`; logger.error({ msg, processName }); return { error: new FuseError(FuseCodes.ENOENT, msg) }; diff --git a/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts b/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts index 45401df69f..960f7b4558 100644 --- a/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts @@ -1,42 +1,37 @@ import { mockDeep } from 'vitest-mock-extended'; import { Container } from 'diod'; import { opendir } from './opendir.service'; -import { FilesByFolderPathSearcher } from '../../../../../context/virtual-drive/files/application/search/FilesByFolderPathSearcher'; -import { FoldersByParentPathLister } from '../../../../../context/virtual-drive/folders/application/FoldersByParentPathLister'; import { TemporalFileByFolderFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByFolderFinder'; -import { FolderNotFoundError } from '../../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import { FILE_MODE, FOLDER_MODE } from '../../constants'; import type { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; describe('opendir', () => { let container: ReturnType>; - const fileSearcher = mockDeep(); - const folderLister = mockDeep(); + const hydrator = mockDeep(); const temporalFinder = mockDeep(); beforeEach(() => { container = mockDeep(); - container.get.calledWith(FilesByFolderPathSearcher).mockReturnValue(fileSearcher); - container.get.calledWith(FoldersByParentPathLister).mockReturnValue(folderLister); + container.get.calledWith(LazyVirtualDriveHydrator).mockReturnValue(hydrator); container.get.calledWith(TemporalFileByFolderFinder).mockReturnValue(temporalFinder); - fileSearcher.run.mockResolvedValue([]); - folderLister.run.mockResolvedValue([]); + hydrator.readDirectory.mockResolvedValue({ files: [], folders: [] }); temporalFinder.run.mockResolvedValue([]); }); describe('when directory has files and subfolders', () => { it('should return entries with correct modes', async () => { - fileSearcher.run.mockResolvedValue(['file.txt', 'photo.jpg']); - folderLister.run.mockResolvedValue(['subdir']); + hydrator.readDirectory.mockResolvedValue({ files: ['file.txt', 'photo.jpg'], folders: ['subdir'] }); const { data, error } = await opendir('/some/folder', container); expect(error).toBeUndefined(); expect(data?.entries).toStrictEqual([ + { name: 'subdir', mode: FOLDER_MODE }, { name: 'file.txt', mode: FILE_MODE }, { name: 'photo.jpg', mode: FILE_MODE }, - { name: 'subdir', mode: FOLDER_MODE }, ]); }); }); @@ -59,20 +54,9 @@ describe('opendir', () => { }); }); - describe('when folder is not yet synced', () => { - it('should return empty entries', async () => { - folderLister.run.mockRejectedValue(new FolderNotFoundError('not synced')); - - const { data, error } = await opendir('/unsynced/folder', container); - - expect(error).toBeUndefined(); - expect(data?.entries).toStrictEqual([]); - }); - }); - describe('when an unexpected error is thrown', () => { it('should return EIO', async () => { - fileSearcher.run.mockRejectedValue(new Error('unexpected')); + hydrator.readDirectory.mockRejectedValue(new FuseError(FuseCodes.EIO, 'unexpected')); const { data, error } = await opendir('/some/folder', container); diff --git a/src/backend/features/virtual-drive/services/operations/opendir.service.ts b/src/backend/features/virtual-drive/services/operations/opendir.service.ts index 0731053d6c..dcc0d055d0 100644 --- a/src/backend/features/virtual-drive/services/operations/opendir.service.ts +++ b/src/backend/features/virtual-drive/services/operations/opendir.service.ts @@ -2,12 +2,9 @@ import { Container } from 'diod'; import { Result } from '../../../../../context/shared/domain/Result'; import { FILE_MODE, FOLDER_MODE } from '../../constants'; import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; -import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; -import { FilesByFolderPathSearcher } from '../../../../../context/virtual-drive/files/application/search/FilesByFolderPathSearcher'; -import { FoldersByParentPathLister } from '../../../../../context/virtual-drive/folders/application/FoldersByParentPathLister'; import { TemporalFileByFolderFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByFolderFinder'; -import { FolderNotFoundError } from '../../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; export type DirEntry = { name: string; @@ -20,25 +17,20 @@ export type OpenDirData = { export async function opendir(path: string, container: Container): Promise> { try { - const [fileNames, folderNames, temporalFiles] = await Promise.all([ - container.get(FilesByFolderPathSearcher).run(path), - container.get(FoldersByParentPathLister).run(path), + const [entries, temporalFiles] = await Promise.all([ + container.get(LazyVirtualDriveHydrator).readDirectory({ path }), container.get(TemporalFileByFolderFinder).run(path), ]); - const entries: DirEntry[] = [ - ...fileNames.map((name) => ({ name, mode: FILE_MODE })), - ...folderNames.map((name) => ({ name, mode: FOLDER_MODE })), + const data: DirEntry[] = [ + ...entries.folders.map((name) => ({ name, mode: FOLDER_MODE })), + ...entries.files.map((name) => ({ name, mode: FILE_MODE })), ...temporalFiles.filter((f) => f.isAuxiliary()).map((f) => ({ name: f.name, mode: FILE_MODE })), ]; - return { data: { entries } }; + return { data: { entries: data } }; } catch (err) { - if (err instanceof FolderNotFoundError) { - logger.debug({ msg: '[FUSE - OpenDir] Folder not yet synced, returning empty', path }); - return { data: { entries: [] } }; - } logger.error({ msg: '[FUSE - OpenDir] Error reading directory', error: err, path }); - return { error: new FuseError(FuseCodes.EIO, `[FUSE - OpenDir] IO error: ${path}`) }; + return { error: err as FuseError }; } } diff --git a/src/backend/features/virtual-drive/services/operations/release.service.test.ts b/src/backend/features/virtual-drive/services/operations/release.service.test.ts index 67f713cf8e..c587f603b5 100644 --- a/src/backend/features/virtual-drive/services/operations/release.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/release.service.test.ts @@ -2,13 +2,9 @@ import { mockDeep } from 'vitest-mock-extended'; import { Container } from 'diod'; import { release } from './release.service'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; -import { TemporalFileUploader } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader'; +import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; -import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; -import { File, FileAttributes } from '../../../../../context/virtual-drive/files/domain/File'; -import { ContentsId } from '../../../../../apps/main/database/entities/DriveFile'; -import { FileStatuses } from '../../../../../context/virtual-drive/files/domain/FileStatus'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import { UploadSizeLimitError } from '../../../user/file-size-limit/upload-size-limit-error'; import { call, calls } from '../../../../../../tests/vitest/utils.helper'; @@ -18,19 +14,6 @@ import { markUploadSizeLimitBlockedPath, } from '../../../user/file-size-limit/add-max-file-size-rejection'; -const fileAttrs: FileAttributes = { - id: 1, - uuid: 'a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d', - contentsId: 'aabbccddeeff001122334455', - folderId: 0, - createdAt: new Date().toISOString(), - modificationTime: new Date().toISOString(), - path: '/Documents/report.pdf', - size: 100, - updatedAt: new Date().toISOString(), - status: FileStatuses.EXISTS, -}; - function createTemporalFile(path: string): TemporalFile { return TemporalFile.from({ path, size: 100, createdAt: new Date(), modifiedAt: new Date() }); } @@ -42,17 +25,15 @@ function createAuxiliaryFile(path: string): TemporalFile { describe('release', () => { let container: ReturnType>; const finder = mockDeep(); - const uploader = mockDeep(); + const queue = mockDeep(); const deleter = mockDeep(); - const fileSearcher = mockDeep(); beforeEach(() => { container = mockDeep(); container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(finder); - container.get.calledWith(TemporalFileUploader).mockReturnValue(uploader); + container.get.calledWith(TemporalFileUploadQueue).mockReturnValue(queue); container.get.calledWith(TemporalFileDeleter).mockReturnValue(deleter); - container.get.calledWith(FirstsFileSearcher).mockReturnValue(fileSearcher); - fileSearcher.run.mockResolvedValue(undefined); + queue.enqueue.mockResolvedValue(undefined); clearUploadSizeLimitBlockedPath('/Documents/report.pdf'); }); @@ -64,7 +45,7 @@ describe('release', () => { expect(error).toBeUndefined(); expect(data).toBeUndefined(); - calls(uploader.run).toHaveLength(0); + calls(queue.enqueue).toHaveLength(0); }); }); @@ -76,40 +57,21 @@ describe('release', () => { expect(error).toBeUndefined(); expect(data).toBeUndefined(); - calls(uploader.run).toHaveLength(0); + calls(queue.enqueue).toHaveLength(0); call(deleter.run).toStrictEqual('/Documents/.~lock.file.odt#'); }); }); describe('when a temporal file is found', () => { - it('should upload without replaces when no virtual file exists', async () => { - const temporalFile = createTemporalFile('/Documents/report.pdf'); - finder.run.mockResolvedValue(temporalFile); - fileSearcher.run.mockResolvedValue(undefined); - uploader.run.mockResolvedValue('contents-id-123' as ContentsId); - - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); - - expect(error).toBeUndefined(); - expect(data).toBeUndefined(); - call(uploader.run).toStrictEqual([temporalFile, undefined]); - }); - - it('should upload with replaces when a virtual file exists at the same path', async () => { + it('should enqueue upload work', async () => { const temporalFile = createTemporalFile('/Documents/report.pdf'); - const existingFile = File.from(fileAttrs); finder.run.mockResolvedValue(temporalFile); - fileSearcher.run.mockResolvedValue(existingFile); - uploader.run.mockResolvedValue('new-contents-id' as ContentsId); const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); - call(uploader.run).toStrictEqual([ - temporalFile, - { contentsId: existingFile.contentsId, name: existingFile.name, extension: existingFile.type }, - ]); + call(queue.enqueue).toStrictEqual({ temporalFile, path: '/Documents/report.pdf', processName: 'cat' }); }); it('should delete partial temporal file and skip upload when write already blocked it by upload size limit', async () => { @@ -121,15 +83,14 @@ describe('release', () => { expect(error).toBeUndefined(); expect(data).toBeUndefined(); - calls(uploader.run).toHaveLength(0); - calls(fileSearcher.run).toHaveLength(0); + calls(queue.enqueue).toHaveLength(0); call(deleter.run).toStrictEqual('/Documents/report.pdf'); expect(isUploadSizeLimitBlockedPath('/Documents/report.pdf')).toBe(false); }); - it('should preserve the temporal file and return success when upload is rejected by upload size limit during upload preflight', async () => { + it('should preserve the temporal file and return success when queue rejects by upload size limit', async () => { finder.run.mockResolvedValue(createTemporalFile('/Documents/report.pdf')); - uploader.run.mockRejectedValue(new UploadSizeLimitError()); + queue.enqueue.mockRejectedValue(new UploadSizeLimitError()); const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); @@ -140,7 +101,7 @@ describe('release', () => { it('should delete the file and return EIO when upload fails', async () => { finder.run.mockResolvedValue(createTemporalFile('/Documents/report.pdf')); - uploader.run.mockRejectedValue(new Error('Network error')); + queue.enqueue.mockRejectedValue(new Error('Network error')); const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); @@ -148,37 +109,6 @@ describe('release', () => { expect(error?.code).toBe(FuseCodes.EIO); call(deleter.run).toStrictEqual('/Documents/report.pdf'); }); - - it('should skip the second release when an upload for the same path is already in progress', async () => { - const temporalFile = createTemporalFile('/Documents/report.pdf'); - finder.run.mockResolvedValue(temporalFile); - fileSearcher.run.mockResolvedValue(undefined); - - // First release never resolves during the test — simulates a long in-progress upload - let resolveFirstUpload!: () => void; - uploader.run.mockReturnValue( - new Promise((resolve) => { - resolveFirstUpload = () => resolve('contents-id-123' as ContentsId); - }), - ); - - const first = release({ path: '/Documents/report.pdf', processName: 'proc-a', container }); - - await Promise.resolve(); - - const { data: data2, error: error2 } = await release({ - path: '/Documents/report.pdf', - processName: 'proc-b', - container, - }); - - expect(error2).toBeUndefined(); - expect(data2).toBeUndefined(); - calls(uploader.run).toHaveLength(1); - - resolveFirstUpload(); - await first; - }); }); describe('when finder throws an unexpected error', () => { @@ -189,7 +119,7 @@ describe('release', () => { expect(data).toBeUndefined(); expect(error?.code).toBe(FuseCodes.EIO); - calls(uploader.run).toHaveLength(0); + calls(queue.enqueue).toHaveLength(0); calls(deleter.run).toHaveLength(0); }); }); diff --git a/src/backend/features/virtual-drive/services/operations/release.service.ts b/src/backend/features/virtual-drive/services/operations/release.service.ts index c08b9dea00..a7c42df19a 100644 --- a/src/backend/features/virtual-drive/services/operations/release.service.ts +++ b/src/backend/features/virtual-drive/services/operations/release.service.ts @@ -3,10 +3,8 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import { type Result } from '../../../../../context/shared/domain/Result'; import { FuseError, FuseIOError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; -import { TemporalFileUploader } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader'; +import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; -import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; -import { FileStatuses } from '../../../../../context/virtual-drive/files/domain/FileStatus'; import { UploadSizeLimitError } from '../../../user/file-size-limit/upload-size-limit-error'; import { @@ -26,8 +24,6 @@ type Props = { // one for metadata and one for the actual content. // The issue is that when each descriptor closes, it triggers a release, // resulting in a duplicate request to create the file remotely. -const uploadsInProgress = new Set(); - export async function release({ path, processName, container }: Props): Promise> { try { const temporalFile = await container.get(TemporalFileByPathFinder).run(path); @@ -53,21 +49,9 @@ export async function release({ path, processName, container }: Props): Promise< return { data: undefined }; } - if (uploadsInProgress.has(path)) { - logger.debug({ msg: '[Release] Upload already in progress, skipping duplicate release', path, processName }); - return { data: undefined }; - } - - uploadsInProgress.add(path); - try { - const existingFile = await container.get(FirstsFileSearcher).run({ path, status: FileStatuses.EXISTS }); - const replaces = existingFile - ? { contentsId: existingFile.contentsId, name: existingFile.name, extension: existingFile.type } - : undefined; - - await container.get(TemporalFileUploader).run(temporalFile, replaces); - logger.debug({ msg: '[Release] Temporal file uploaded', path, processName }); + await container.get(TemporalFileUploadQueue).enqueue({ temporalFile, path, processName }); + logger.debug({ msg: '[Release] Temporal file queued for upload', path, processName }); return { data: undefined }; } catch (uploadError) { if (uploadError instanceof UploadSizeLimitError) { @@ -83,8 +67,6 @@ export async function release({ path, processName, container }: Props): Promise< logger.error({ msg: '[Release] Upload failed, deleting temporal file', error: uploadError, path, processName }); await container.get(TemporalFileDeleter).run(path); return { error: new FuseIOError('Upload failed due to insufficient storage or network issues.') }; - } finally { - uploadsInProgress.delete(path); } } catch (err: unknown) { logger.error({ msg: '[Release] Unexpected error', error: err, path, processName }); diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts new file mode 100644 index 0000000000..46dc5f3841 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts @@ -0,0 +1,124 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { PATHS } from '../../../../../core/electron/paths'; +import { TemporalFileDeleter } from '../deletion/TemporalFileDeleter'; +import { TemporalFile } from '../../domain/TemporalFile'; +import { TemporalFileRepository } from '../../domain/TemporalFileRepository'; +import { TemporalFileUploader } from './TemporalFileUploader'; +import { FirstsFileSearcher } from '../../../../virtual-drive/files/application/search/FirstsFileSearcher'; +import { FileStatuses } from '../../../../virtual-drive/files/domain/FileStatus'; +import { preserveRejectedFileSizeTooBig } from '../../../../../backend/features/user/file-size-limit'; +import { UploadSizeLimitError } from '../../../../../backend/features/user/file-size-limit/upload-size-limit-error'; + +type Props = { + temporalFile: TemporalFile; + path: string; + processName: string; +}; + +type UploadTask = { + temporalFile: TemporalFile; + path: string; + processName: string; +}; + +type FactoryProps = { + repository: TemporalFileRepository; + uploader: TemporalFileUploader; + deleter: TemporalFileDeleter; + fileSearcher: FirstsFileSearcher; +}; + +export const TemporalFileUploadQueue = Symbol('TemporalFileUploadQueue'); + +export type TemporalFileUploadQueue = ReturnType; + +export function createTemporalFileUploadQueue({ repository, uploader, deleter, fileSearcher }: FactoryProps) { + const queuedPaths = new Set(); + const tasks: UploadTask[] = []; + let draining = false; + + async function enqueue({ temporalFile, path, processName }: Props) { + if (queuedPaths.has(path)) { + logger.debug({ msg: '[UploadQueue] Path already queued', path, processName }); + return; + } + + const stagedTemporalFile = await repository.stage(temporalFile.path, PATHS.UPLOAD_QUEUE); + + tasks.push({ + temporalFile: stagedTemporalFile, + path, + processName, + }); + queuedPaths.add(path); + + void drain(); + } + + async function drain() { + if (draining) { + return; + } + + draining = true; + + try { + while (tasks.length > 0) { + const task = tasks[0]; + + try { + await upload(task); + await deleter.run(task.path); + } catch (error) { + if (error instanceof UploadSizeLimitError) { + await preserveRejected(task); + } else { + logger.error({ msg: '[UploadQueue] Upload failed, keeping staged file in queue folder', error, path: task.path }); + } + } finally { + tasks.shift(); + queuedPaths.delete(task.path); + } + } + } finally { + draining = false; + } + } + + async function upload(task: UploadTask) { + const existingFile = await fileSearcher.run({ path: task.path, status: FileStatuses.EXISTS }); + const replaces = existingFile + ? { contentsId: existingFile.contentsId, name: existingFile.name, extension: existingFile.type } + : undefined; + + await uploader.run(task.temporalFile, replaces); + + logger.debug({ msg: '[UploadQueue] Upload completed', path: task.path, processName: task.processName }); + } + + async function preserveRejected(task: UploadTask) { + if (!task.temporalFile.contentFilePath) { + logger.warn({ msg: '[UploadQueue] Rejected file missing content path', path: task.path }); + return; + } + + const { error } = await preserveRejectedFileSizeTooBig({ + originalPath: task.path, + temporalContentPath: task.temporalFile.contentFilePath, + size: task.temporalFile.size.value, + }); + + if (error) { + logger.error({ msg: '[UploadQueue] Failed to preserve oversized file', error, path: task.path }); + return; + } + + await deleter.run(task.path); + + logger.warn({ msg: '[UploadQueue] Preserved oversized file after queue rejection', path: task.path }); + } + + return { + enqueue, + }; +} diff --git a/src/context/storage/TemporalFiles/domain/TemporalFileRepository.ts b/src/context/storage/TemporalFiles/domain/TemporalFileRepository.ts index 319cf5f5d9..19cfd549ab 100644 --- a/src/context/storage/TemporalFiles/domain/TemporalFileRepository.ts +++ b/src/context/storage/TemporalFiles/domain/TemporalFileRepository.ts @@ -16,6 +16,8 @@ export abstract class TemporalFileRepository { abstract read(path: TemporalFilePath): Promise; + abstract stage(path: TemporalFilePath, targetFolder: string): Promise; + abstract stream(path: TemporalFilePath): Promise; abstract find(documentPath: TemporalFilePath): Promise>; diff --git a/src/context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository.ts b/src/context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository.ts index 1a4de651b1..f3ff350a31 100644 --- a/src/context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository.ts +++ b/src/context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository.ts @@ -132,6 +132,31 @@ export class NodeTemporalFileRepository implements TemporalFileRepository { return readFile(id); } + async stage(documentPath: TemporalFilePath, targetFolder: string): Promise { + const currentPath = this.map.get(documentPath.value); + + if (!currentPath) { + throw new Error(`Document with path ${documentPath.value} not found`); + } + + await ensureFolderExists(targetFolder); + + const targetPath = path.join(targetFolder, path.basename(currentPath)); + + await fs.promises.rename(currentPath, targetPath); + this.map.set(documentPath.value, targetPath); + + const stat = await fs.promises.stat(targetPath); + + return TemporalFile.from({ + createdAt: stat.ctime, + modifiedAt: stat.mtime, + path: documentPath.value, + size: stat.size, + contentFilePath: targetPath, + }); + } + async write(documentPath: TemporalFilePath, buffer: Buffer, length: number, position: number): Promise { const pathToWrite = this.map.get(documentPath.value); diff --git a/src/context/virtual-drive/files/domain/FileRepository.ts b/src/context/virtual-drive/files/domain/FileRepository.ts index 8c243eb34c..6e14cd7e69 100644 --- a/src/context/virtual-drive/files/domain/FileRepository.ts +++ b/src/context/virtual-drive/files/domain/FileRepository.ts @@ -14,6 +14,8 @@ export abstract class FileRepository { abstract searchByArrayOfContentsId(contentsIds: Array): Promise>; + abstract deleteMatchingPartial(partial: Partial): Promise; + abstract upsert(file: File): Promise; abstract update(file: File): Promise; diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts index f86a04686b..b0cdb5c05e 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts @@ -104,6 +104,19 @@ export class InMemoryFileRepository implements FileRepository { return isAlreadyStored; } + async deleteMatchingPartial(partial: Partial): Promise { + const keys = Object.keys(partial) as Array>; + + for (const [uuid, attributes] of this.filesByUuid.entries()) { + const matches = keys.every((key: keyof FileAttributes) => attributes[key] === partial[key]); + + if (matches) { + this.filesByUuid.delete(uuid); + this.filesByContentsId.delete(attributes.contentsId); + } + } + } + async update(file: File): Promise { if (!this.filesByUuid.has(file.uuid)) { throw new FileNotFoundError(file.uuid); diff --git a/src/context/virtual-drive/folders/domain/FolderRepository.ts b/src/context/virtual-drive/folders/domain/FolderRepository.ts index 3fad10f02f..840d57b845 100644 --- a/src/context/virtual-drive/folders/domain/FolderRepository.ts +++ b/src/context/virtual-drive/folders/domain/FolderRepository.ts @@ -14,6 +14,8 @@ export abstract class FolderRepository { abstract add(folder: Folder): Promise; + abstract deleteMatchingPartial(partial: Partial): Promise; + abstract delete(id: Folder['id']): Promise; abstract update(folder: Folder): Promise; diff --git a/src/context/virtual-drive/folders/infrastructure/InMemoryFolderRepository.ts b/src/context/virtual-drive/folders/infrastructure/InMemoryFolderRepository.ts index b0f52829b5..d98448b86e 100644 --- a/src/context/virtual-drive/folders/infrastructure/InMemoryFolderRepository.ts +++ b/src/context/virtual-drive/folders/infrastructure/InMemoryFolderRepository.ts @@ -65,6 +65,18 @@ export class InMemoryFolderRepository implements FolderRepository { this.folders.set(folder.id, folder.attributes()); } + async deleteMatchingPartial(partial: Partial): Promise { + const keys = Object.keys(partial) as Array>; + + for (const [id, attributes] of this.folders.entries()) { + const matches = keys.every((key: keyof FolderAttributes) => attributes[key] === partial[key]); + + if (matches) { + this.folders.delete(id); + } + } + } + async delete(id: number): Promise { const deleted = this.folders.delete(id); diff --git a/src/core/bootstrap/register-session-event-handlers.ts b/src/core/bootstrap/register-session-event-handlers.ts index be0287253c..e8d09c3343 100644 --- a/src/core/bootstrap/register-session-event-handlers.ts +++ b/src/core/bootstrap/register-session-event-handlers.ts @@ -1,6 +1,6 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import eventBus from '../../apps/main/event-bus'; -import { AppDataSource, resetAppDataSourceOnLogout } from '../../apps/main/database/data-source'; +import { AppDataSource, initializeVirtualDriveSqlite, resetAppDataSourceOnLogout } from '../../apps/main/database/data-source'; import { getOrCreateWidged, getWidget, setBoundsOfWidgetByPath } from '../../apps/main/windows/widget'; import { createAuthWindow, getAuthWindow } from '../../apps/main/windows/auth'; import configStore from '../../apps/main/config'; @@ -26,6 +26,7 @@ async function onUserLoggedIn() { try { if (!AppDataSource.isInitialized) { await AppDataSource.initialize(); + await initializeVirtualDriveSqlite(); eventBus.emit('APP_DATA_SOURCE_INITIALIZED'); } diff --git a/src/core/electron/paths.ts b/src/core/electron/paths.ts index 704e6ea970..b6ab1a74b9 100644 --- a/src/core/electron/paths.ts +++ b/src/core/electron/paths.ts @@ -15,6 +15,7 @@ const TEMPORAL_FOLDER = app.getPath('temp'); const INTERNXT_DRIVE_TMP = path.join(TEMPORAL_FOLDER, 'internxt-drive-tmp'); const REJECTED_FILES_SIZE_TOO_BIG = join(INTERNXT, 'rejected-files-size-too-big'); const DOWNLOADED = join(INTERNXT, 'downloaded'); +const UPLOAD_QUEUE = join(INTERNXT, 'upload_queue'); const FUSE_DAEMON_LOG = join(LOGS, 'fuse-daemon.log'); const FUSE_DAEMON_SOCKET = join(process.env.XDG_RUNTIME_DIR ?? '/tmp', 'internxt-fuse.sock'); const FUSE_DAEMON_BINARY = app.isPackaged @@ -36,6 +37,7 @@ export const PATHS = { REJECTED_FILES_SIZE_TOO_BIG, ROOT_DRIVE_FOLDER, DOWNLOADED, + UPLOAD_QUEUE, FUSE_DAEMON_LOG, FUSE_DAEMON_SOCKET, FUSE_DAEMON_BINARY, diff --git a/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts b/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts index 221897d4c1..313a570df0 100644 --- a/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts +++ b/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts @@ -1,8 +1,9 @@ -import { fileRepository } from '../drive-file'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { parseData } from './parse-data'; import { SqliteError } from '../common/sqlite-error'; import { RemoteSyncedFile } from 'src/apps/main/remote-sync/helpers'; +import { AppDataSource } from '../../../../apps/main/database/data-source'; +import { DriveFile } from '../../../../apps/main/database/entities/DriveFile'; const BATCH_SIZE = 500; @@ -14,13 +15,17 @@ export async function createOrUpdateFileByBatch({ files }: Props) { if (files.length === 0) return { data: [] }; try { - for (let i = 0; i < files.length; i += BATCH_SIZE) { - const chunk = files.slice(i, i + BATCH_SIZE); + await AppDataSource.transaction(async (manager) => { + const repository = manager.getRepository(DriveFile); - await fileRepository.upsert(chunk, { - conflictPaths: ['uuid'], - }); - } + for (let i = 0; i < files.length; i += BATCH_SIZE) { + const chunk = files.slice(i, i + BATCH_SIZE); + + await repository.upsert(chunk, { + conflictPaths: ['uuid'], + }); + } + }); return { data: files.map((data) => parseData({ data })) }; } catch (error) { diff --git a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts index 6debdfa3c5..2f0c2400ed 100644 --- a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts +++ b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts @@ -1,8 +1,9 @@ -import { folderRepository } from '../drive-folder'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { parseData } from './parse-data'; import { SqliteError } from '../common/sqlite-error'; import { RemoteSyncedFolder } from 'src/apps/main/remote-sync/helpers'; +import { AppDataSource } from '../../../../apps/main/database/data-source'; +import { DriveFolder } from '../../../../apps/main/database/entities/DriveFolder'; const BATCH_SIZE = 500; @@ -14,13 +15,17 @@ export async function createOrUpdateFolderByBatch({ folders }: Props) { if (folders.length === 0) return { data: [] }; try { - for (let i = 0; i < folders.length; i += BATCH_SIZE) { - const chunk = folders.slice(i, i + BATCH_SIZE); + await AppDataSource.transaction(async (manager) => { + const repository = manager.getRepository(DriveFolder); - await folderRepository.upsert(chunk, { - conflictPaths: ['uuid'], - }); - } + for (let i = 0; i < folders.length; i += BATCH_SIZE) { + const chunk = folders.slice(i, i + BATCH_SIZE); + + await repository.upsert(chunk, { + conflictPaths: ['uuid'], + }); + } + }); return { data: folders.map((data) => parseData({ data })) }; } catch (error) { From ea64034f9c494b5b14f8829ccaea5b3cde1da828 Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Mon, 6 Jul 2026 11:49:55 -0500 Subject: [PATCH 2/6] Refactor code for improved readability and maintainability across multiple files --- .../registerVirtualDriveSharedServices.ts | 10 ++- .../virtual-drive.service.test.ts | 6 +- .../services/lazy/LazyVirtualDriveHydrator.ts | 14 +---- .../operations/get-attributes.service.ts | 61 +++++++++---------- .../services/operations/open.service.test.ts | 7 ++- .../upload/TemporalFileUploadQueue.ts | 6 +- .../folders/__mocks__/FolderRepositoryMock.ts | 5 ++ .../application/create/SimpleFolderCreator.ts | 2 +- .../SQLiteRemoteItemsGenerator.ts | 6 +- .../register-session-event-handlers.ts | 6 +- .../local-file-system/safe-readdir.test.ts | 3 +- .../create-or-update-folder-by-batch.ts | 7 ++- vitest.setup.main.ts | 4 ++ 13 files changed, 82 insertions(+), 55 deletions(-) diff --git a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts index 62a31e0e1c..c4f9f66fff 100644 --- a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts +++ b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts @@ -8,14 +8,20 @@ import { createDirectoryStateSqliteRepository, DirectoryStateSqliteRepository, } from '../../../../backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository'; -import { createLazyVirtualDriveHydrator, LazyVirtualDriveHydrator } from '../../../../backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator'; +import { + createLazyVirtualDriveHydrator, + LazyVirtualDriveHydrator, +} from '../../../../backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator'; export async function registerVirtualDriveSharedServices(builder: ContainerBuilder): Promise { const downloaded = PATHS.DOWNLOADED; builder.register(RelativePathToAbsoluteConverter).useFactory(() => new RelativePathToAbsoluteConverter(downloaded)); builder.register(AbsolutePathToRelativeConverter).useFactory(() => new AbsolutePathToRelativeConverter(downloaded)); - builder.register(DirectoryStateSqliteRepository).useFactory(() => createDirectoryStateSqliteRepository()).asSingleton(); + builder + .register(DirectoryStateSqliteRepository) + .useFactory(() => createDirectoryStateSqliteRepository()) + .asSingleton(); builder .register(LazyVirtualDriveHydrator) .useFactory((container) => diff --git a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts index 3615b5821a..74aa3971d5 100644 --- a/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts +++ b/src/backend/features/virtual-drive/services/drive-folder/virtual-drive.service.test.ts @@ -36,7 +36,11 @@ describe('virtual-drive.service', () => { startFuseDaemonServer.mockResolvedValue(undefined); startHydrationApi.mockResolvedValue(undefined); getRootVirtualDrive.mockReturnValue('/mock/root/'); - getUser.mockReturnValue({ root_folder_id: 99, rootFolderId: '123e4567-e89b-12d3-a456-426614174000', createdAt: '2026-01-01T00:00:00.000Z' } as never); + getUser.mockReturnValue({ + root_folder_id: 99, + rootFolderId: '123e4567-e89b-12d3-a456-426614174000', + createdAt: '2026-01-01T00:00:00.000Z', + } as never); buildContainer.mockResolvedValue(containerMock); deleteAll.mockResolvedValue(undefined); add.mockResolvedValue(undefined); diff --git a/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts b/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts index f840fbc52c..4c9397a193 100644 --- a/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts +++ b/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts @@ -39,11 +39,7 @@ export const LazyVirtualDriveHydrator = Symbol('LazyVirtualDriveHydrator'); export type LazyVirtualDriveHydrator = ReturnType; -export function createLazyVirtualDriveHydrator({ - folderRepository, - fileRepository, - directoryStateRepository, -}: Props) { +export function createLazyVirtualDriveHydrator({ folderRepository, fileRepository, directoryStateRepository }: Props) { const inflight = new Map>(); async function readDirectory({ path: requestedPath }: ReadDirectoryProps) { @@ -128,13 +124,7 @@ export function createLazyVirtualDriveHydrator({ return task; } - async function fetchAndStoreChildren({ - folder, - statusScope, - }: { - folder: Folder; - statusScope: DirectoryStatusScope; - }) { + async function fetchAndStoreChildren({ folder, statusScope }: { folder: Folder; statusScope: DirectoryStatusScope }) { try { const { data, error } = await fetchFolder(folder.uuid); diff --git a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts index e244b663ce..b6abd81cc6 100644 --- a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts +++ b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts @@ -83,39 +83,38 @@ export async function getAttributes( const document = await container.get(TemporalFileByPathFinder).run(path); if (document) { + await container.get(LazyVirtualDriveHydrator).ensurePathLoaded({ path }); - await container.get(LazyVirtualDriveHydrator).ensurePathLoaded({ path }); + const hydratedEntry = await findLocalEntry(path, container); + if (hydratedEntry.file) { + return { + data: { + mode: FILE_MODE, + size: hydratedEntry.file.size, + ctime: hydratedEntry.file.createdAt, + mtime: hydratedEntry.file.updatedAt, + atime: new Date(), + uid: process.getuid?.() || 0, + gid: process.getgid?.() || 0, + nlink: 1, + }, + }; + } - const hydratedEntry = await findLocalEntry(path, container); - if (hydratedEntry.file) { - return { - data: { - mode: FILE_MODE, - size: hydratedEntry.file.size, - ctime: hydratedEntry.file.createdAt, - mtime: hydratedEntry.file.updatedAt, - atime: new Date(), - uid: process.getuid?.() || 0, - gid: process.getgid?.() || 0, - nlink: 1, - }, - }; - } - - if (hydratedEntry.folder) { - return { - data: { - mode: FOLDER_MODE, - size: 0, - ctime: hydratedEntry.folder.createdAt, - mtime: hydratedEntry.folder.updatedAt, - atime: hydratedEntry.folder.createdAt, - uid: process.getuid?.() || 0, - gid: process.getgid?.() || 0, - nlink: 2, - }, - }; - } + if (hydratedEntry.folder) { + return { + data: { + mode: FOLDER_MODE, + size: 0, + ctime: hydratedEntry.folder.createdAt, + mtime: hydratedEntry.folder.updatedAt, + atime: hydratedEntry.folder.createdAt, + uid: process.getuid?.() || 0, + gid: process.getgid?.() || 0, + nlink: 2, + }, + }; + } return { data: { diff --git a/src/backend/features/virtual-drive/services/operations/open.service.test.ts b/src/backend/features/virtual-drive/services/operations/open.service.test.ts index 69bf8a5f76..6327670044 100644 --- a/src/backend/features/virtual-drive/services/operations/open.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/open.service.test.ts @@ -1,23 +1,28 @@ import { mockDeep } from 'vitest-mock-extended'; -import { Container } from 'diod'; +import { Container, type Identifier } from 'diod'; import { open } from './open.service'; import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; import type { File } from '../../../../../context/virtual-drive/files/domain/File'; describe('open', () => { + const lazyVirtualDriveHydratorToken = LazyVirtualDriveHydrator as unknown as Identifier; let container: ReturnType>; const fileSearcher = mockDeep(); const temporalFinder = mockDeep(); + const lazyHydrator = mockDeep(); beforeEach(() => { container = mockDeep(); container.get.calledWith(FirstsFileSearcher).mockReturnValue(fileSearcher); container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(temporalFinder); + container.get.calledWith(lazyVirtualDriveHydratorToken).mockReturnValue(lazyHydrator); fileSearcher.run.mockResolvedValue(undefined); temporalFinder.run.mockResolvedValue(undefined); + lazyHydrator.ensurePathLoaded.mockResolvedValue(undefined); }); describe('when a virtual file is found', () => { diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts index 46dc5f3841..f522353b94 100644 --- a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts @@ -73,7 +73,11 @@ export function createTemporalFileUploadQueue({ repository, uploader, deleter, f if (error instanceof UploadSizeLimitError) { await preserveRejected(task); } else { - logger.error({ msg: '[UploadQueue] Upload failed, keeping staged file in queue folder', error, path: task.path }); + logger.error({ + msg: '[UploadQueue] Upload failed, keeping staged file in queue folder', + error, + path: task.path, + }); } } finally { tasks.shift(); diff --git a/src/context/virtual-drive/folders/__mocks__/FolderRepositoryMock.ts b/src/context/virtual-drive/folders/__mocks__/FolderRepositoryMock.ts index 30bf08afbc..5f20aced7c 100644 --- a/src/context/virtual-drive/folders/__mocks__/FolderRepositoryMock.ts +++ b/src/context/virtual-drive/folders/__mocks__/FolderRepositoryMock.ts @@ -9,6 +9,7 @@ export class FolderRepositoryMock implements FolderRepository { public readonly searchByPathPrefixMock = vi.fn(); public readonly listByPartialMock = vi.fn(); public readonly addMock = vi.fn(); + public readonly deleteMatchingPartialMock = vi.fn(); public readonly deleteMock = vi.fn(); public readonly updateMock = vi.fn(); public readonly searchByIdMock = vi.fn(); @@ -43,6 +44,10 @@ export class FolderRepositoryMock implements FolderRepository { return this.addMock(folder); } + deleteMatchingPartial(partial: Partial): Promise { + return this.deleteMatchingPartialMock(partial); + } + delete(id: number): Promise { return this.deleteMock(id); } diff --git a/src/context/virtual-drive/folders/application/create/SimpleFolderCreator.ts b/src/context/virtual-drive/folders/application/create/SimpleFolderCreator.ts index 08c50439c8..83e41d5435 100644 --- a/src/context/virtual-drive/folders/application/create/SimpleFolderCreator.ts +++ b/src/context/virtual-drive/folders/application/create/SimpleFolderCreator.ts @@ -24,7 +24,7 @@ export class SimpleFolderCreator { msg: 'The folder was not been able to create', error, }); - if (error !== 'ALREADY_EXISTS') { + if (error.cause !== 'FILE_ALREADY_EXISTS') { return; } return this.rfs.searchWith(folderParentId, folderPath); diff --git a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts index 2e75852734..f81097bd13 100644 --- a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts +++ b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts @@ -18,10 +18,10 @@ export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { folderId: updatedFile.folderId, id: updatedFile.id, modificationTime: updatedFile.modificationTime, - name: updatedFile.name, + name: updatedFile.name ?? updatedFile.plainName, plainName: updatedFile.plainName, size: updatedFile.size, - type: updatedFile.type ?? null, + type: updatedFile.type ?? '', updatedAt: updatedFile.updatedAt, userId: updatedFile.userId, status: updatedFile.status as ServerFileStatus, @@ -34,7 +34,7 @@ export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { bucket: updatedFolder.bucket ?? null, createdAt: updatedFolder.createdAt, id: updatedFolder.id, - name: updatedFolder.name, + name: updatedFolder.name ?? updatedFolder.plainName, parentId: updatedFolder.parentId ?? null, updatedAt: updatedFolder.updatedAt, plain_name: updatedFolder.plainName ?? null, diff --git a/src/core/bootstrap/register-session-event-handlers.ts b/src/core/bootstrap/register-session-event-handlers.ts index e8d09c3343..83c14ece76 100644 --- a/src/core/bootstrap/register-session-event-handlers.ts +++ b/src/core/bootstrap/register-session-event-handlers.ts @@ -1,6 +1,10 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import eventBus from '../../apps/main/event-bus'; -import { AppDataSource, initializeVirtualDriveSqlite, resetAppDataSourceOnLogout } from '../../apps/main/database/data-source'; +import { + AppDataSource, + initializeVirtualDriveSqlite, + resetAppDataSourceOnLogout, +} from '../../apps/main/database/data-source'; import { getOrCreateWidged, getWidget, setBoundsOfWidgetByPath } from '../../apps/main/windows/widget'; import { createAuthWindow, getAuthWindow } from '../../apps/main/windows/auth'; import configStore from '../../apps/main/config'; diff --git a/src/infra/local-file-system/safe-readdir.test.ts b/src/infra/local-file-system/safe-readdir.test.ts index 52b7ca0f30..aeb0260846 100644 --- a/src/infra/local-file-system/safe-readdir.test.ts +++ b/src/infra/local-file-system/safe-readdir.test.ts @@ -8,6 +8,7 @@ vi.mock('node:fs/promises', () => ({ })); const readdirMock = vi.mocked(readdir); +type ReadDirResult = Awaited>; function createFsError(code: string, message = `${code}: readdir failed`): NodeJS.ErrnoException { const error = new Error(message) as NodeJS.ErrnoException; @@ -17,7 +18,7 @@ function createFsError(code: string, message = `${code}: readdir failed`): NodeJ describe('safeReadDir', () => { it('returns dirents when readdir succeeds', async () => { - const dirents = [{ name: 'file.txt' }, { name: 'folder' }] as Dirent[]; + const dirents = [{ name: 'file.txt' }, { name: 'folder' }] as unknown as ReadDirResult; readdirMock.mockResolvedValue(dirents); const result = await safeReadDir('/tmp/root'); diff --git a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts index 2f0c2400ed..62df55e4ab 100644 --- a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts +++ b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts @@ -20,8 +20,13 @@ export async function createOrUpdateFolderByBatch({ folders }: Props) { for (let i = 0; i < folders.length; i += BATCH_SIZE) { const chunk = folders.slice(i, i + BATCH_SIZE); + const upsertChunk = chunk.map((folder) => ({ + ...folder, + parentId: folder.parentId ?? undefined, + bucket: folder.bucket ?? undefined, + })); - await repository.upsert(chunk, { + await repository.upsert(upsertChunk, { conflictPaths: ['uuid'], }); } diff --git a/vitest.setup.main.ts b/vitest.setup.main.ts index 76df77e581..1b4227b659 100644 --- a/vitest.setup.main.ts +++ b/vitest.setup.main.ts @@ -86,6 +86,10 @@ vi.mock('axios', () => { // Mock @internxt/inxt-js vi.mock('@internxt/inxt-js', () => ({ default: vi.fn(), + Environment: { + BRIDGE: 'bridge', + PHOTOS: 'photos', + }, })); // Mock event-bus with EventEmitter From 07c52fa4b1920c8bbc1120daab031a29728e592a Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Mon, 6 Jul 2026 22:05:58 -0500 Subject: [PATCH 3/6] refactor: disable 'no-await-in-loop' rule and remove unused import in safe-readdir tests --- .eslintrc.js | 2 +- src/infra/local-file-system/safe-readdir.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 92f3a8cbe4..00c951f84e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -13,7 +13,7 @@ module.exports = { }, ], rules: { - 'no-await-in-loop': 'warn', + 'no-await-in-loop': 'off', '@typescript-eslint/no-use-before-define': ['warn', { functions: false, classes: true, variables: true }], 'array-callback-return': 'warn', 'max-len': [ diff --git a/src/infra/local-file-system/safe-readdir.test.ts b/src/infra/local-file-system/safe-readdir.test.ts index aeb0260846..e0dc4522e7 100644 --- a/src/infra/local-file-system/safe-readdir.test.ts +++ b/src/infra/local-file-system/safe-readdir.test.ts @@ -1,4 +1,3 @@ -import type { Dirent } from 'node:fs'; import { readdir } from 'node:fs/promises'; import { DriveDesktopError } from '../../context/shared/domain/errors/DriveDesktopError'; import { safeReadDir } from './safe-readdir'; From 8277a782b10074d979f4aca7446460c9c26f3a30 Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Tue, 7 Jul 2026 16:10:06 -0500 Subject: [PATCH 4/6] Update src/apps/main/database/data-source.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/apps/main/database/data-source.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/apps/main/database/data-source.ts b/src/apps/main/database/data-source.ts index 41477e2036..bae36c0c49 100644 --- a/src/apps/main/database/data-source.ts +++ b/src/apps/main/database/data-source.ts @@ -44,7 +44,11 @@ export async function initializeVirtualDriveSqlite() { } for (const statement of SQLITE_BOOTSTRAP_STATEMENTS) { - await AppDataSource.query(statement); + try { + await AppDataSource.query(statement); + } catch (error) { + logger.error({ msg: 'Error running SQLite bootstrap statement', statement, error }); + } } } From 6607102ca794ab96096a5d74ad8a069730febb5e Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Tue, 7 Jul 2026 20:58:23 -0500 Subject: [PATCH 5/6] Refactor virtual drive services to use new lazy virtual drive hydrator service --- .../registerTemporalFilesServices.ts | 8 +- .../registerVirtualDriveSharedServices.ts | 6 +- .../add-max-file-size-rejection.ts | 24 +- .../upload-size-limit-blocked-paths.ts | 13 + .../operations/release.controller.test.ts | 8 + .../operations/release.controller.ts | 15 +- .../services/lazy/LazyVirtualDriveHydrator.ts | 295 ------------------ .../virtual-drive-hydrator/children.test.ts | 199 ++++++++++++ .../lazy/virtual-drive-hydrator/children.ts | 144 +++++++++ ...ate-lazy-virtual-drive-hydrator-service.ts | 94 ++++++ .../ensure-folder-materialized.test.ts | 53 ++++ .../ensure-folder-materialized.ts | 49 +++ .../fetch-and-store-children.test.ts | 118 +++++++ .../fetch-and-store-children.ts | 60 ++++ .../lazy/virtual-drive-hydrator/file-name.ts | 5 + .../lazy/virtual-drive-hydrator/path.test.ts | 21 ++ .../lazy/virtual-drive-hydrator/path.ts | 19 ++ .../refresh-children-if-needed.ts | 36 +++ .../resolve-root-folder.ts | 13 + .../lazy/virtual-drive-hydrator/types.ts | 19 ++ .../operations/get-attributes.service.test.ts | 2 +- .../operations/get-attributes.service.ts | 2 +- .../services/operations/open.service.test.ts | 2 +- .../services/operations/open.service.ts | 2 +- .../operations/opendir.service.test.ts | 2 +- .../services/operations/opendir.service.ts | 2 +- .../operations/release.service.test.ts | 72 ++++- .../services/operations/release.service.ts | 30 +- .../upload/TemporalFileUploadQueue.ts | 128 -------- ...eate-temporal-file-upload-queue-service.ts | 14 + .../drain-upload-queue.test.ts | 74 +++++ .../drain-upload-queue.ts | 56 ++++ .../enqueue-upload.test.ts | 117 +++++++ .../TemporalFileUploadQueue/enqueue-upload.ts | 41 +++ .../preserve-rejected-upload.test.ts | 51 +++ .../preserve-rejected-upload.ts | 26 ++ .../upload/TemporalFileUploadQueue/state.ts | 11 + .../upload/TemporalFileUploadQueue/types.ts | 40 +++ .../upload-queued-task.test.ts | 46 +++ .../upload-queued-task.ts | 22 ++ .../InMemoryFileRepository.test.ts | 27 ++ .../infrastructure/InMemoryFileRepository.ts | 28 +- 42 files changed, 1506 insertions(+), 488 deletions(-) create mode 100644 src/backend/features/user/file-size-limit/upload-size-limit-blocked-paths.ts delete mode 100644 src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.test.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.test.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.test.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/file-name.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.test.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/refresh-children-if-needed.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/resolve-root-folder.ts create mode 100644 src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/types.ts delete mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.test.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.test.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.test.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/state.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.test.ts create mode 100644 src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.ts diff --git a/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts b/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts index 9e18c42ebe..9c784620b1 100644 --- a/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts +++ b/src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts @@ -9,10 +9,8 @@ import { TemporalFileByFolderFinder } from '../../../../context/storage/Temporal import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFilePathsByFolderFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFilePathsByFolderFinder'; import { TemporalFileTruncater } from '../../../../context/storage/TemporalFiles/application/truncate/TemporalFileTruncater'; -import { - createTemporalFileUploadQueue, - TemporalFileUploadQueue, -} from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; +import { createTemporalFileUploadQueueService } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service'; +import { TemporalFileUploadQueue } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types'; import { TemporalFileUploader } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader'; import { TemporalFileWriter } from '../../../../context/storage/TemporalFiles/application/write/TemporalFileWriter'; import { TemporalFileRepository } from '../../../../context/storage/TemporalFiles/domain/TemporalFileRepository'; @@ -60,7 +58,7 @@ export async function registerTemporalFilesServices(builder: ContainerBuilder) { builder .register(TemporalFileUploadQueue) .useFactory((container) => - createTemporalFileUploadQueue({ + createTemporalFileUploadQueueService({ repository: container.get(TemporalFileRepository), uploader: container.get(TemporalFileUploader), deleter: container.get(TemporalFileDeleter), diff --git a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts index c4f9f66fff..0180500d07 100644 --- a/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts +++ b/src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts @@ -9,9 +9,9 @@ import { DirectoryStateSqliteRepository, } from '../../../../backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository'; import { - createLazyVirtualDriveHydrator, + createLazyVirtualDriveHydratorService, LazyVirtualDriveHydrator, -} from '../../../../backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator'; +} from '../../../../backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; export async function registerVirtualDriveSharedServices(builder: ContainerBuilder): Promise { const downloaded = PATHS.DOWNLOADED; @@ -25,7 +25,7 @@ export async function registerVirtualDriveSharedServices(builder: ContainerBuild builder .register(LazyVirtualDriveHydrator) .useFactory((container) => - createLazyVirtualDriveHydrator({ + createLazyVirtualDriveHydratorService({ folderRepository: container.get(FolderRepository), fileRepository: container.get(FileRepository), directoryStateRepository: container.get(DirectoryStateSqliteRepository), diff --git a/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts b/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts index c3702f3289..f0e5a3977d 100644 --- a/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts +++ b/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts @@ -1,8 +1,16 @@ import { MODAL_DEBOUNCE_MS } from './constants'; import { showMaxFileSizeRejectionModal } from './show-max-file-size-rejection-modal'; import { type UploadFileSizeValidation } from './validate-upload-file-size'; - -const uploadSizeLimitBlockedPaths = new Set(); +import { + clearUploadSizeLimitBlockedPath, + isUploadSizeLimitBlockedPath, + markUploadSizeLimitBlockedPath, +} from './upload-size-limit-blocked-paths'; +export { + clearUploadSizeLimitBlockedPath, + isUploadSizeLimitBlockedPath, + markUploadSizeLimitBlockedPath, +}; type MaxFileSizeRejectionModalState = { rejectedFilesCount: number; @@ -23,18 +31,6 @@ type MaxFileSizeRejectionModalDraft = Omit(); + +export function markUploadSizeLimitBlockedPath(path: string): void { + uploadSizeLimitBlockedPaths.add(path); +} + +export function isUploadSizeLimitBlockedPath(path: string): boolean { + return uploadSizeLimitBlockedPaths.has(path); +} + +export function clearUploadSizeLimitBlockedPath(path: string): void { + uploadSizeLimitBlockedPaths.delete(path); +} diff --git a/src/backend/features/virtual-drive/controllers/operations/release.controller.test.ts b/src/backend/features/virtual-drive/controllers/operations/release.controller.test.ts index 1ddfe04b1b..ceeeee9903 100644 --- a/src/backend/features/virtual-drive/controllers/operations/release.controller.test.ts +++ b/src/backend/features/virtual-drive/controllers/operations/release.controller.test.ts @@ -6,6 +6,9 @@ import * as releaseServiceModule from '../../services/operations/release.service import { partialSpyOn } from '../../../../../../tests/vitest/utils.helper'; import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; +import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; +import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types'; vi.mock(import('@internxt/drive-desktop-core/build/backend')); @@ -19,6 +22,11 @@ describe('releaseController', () => { req = mockDeep(); res = mockDeep(); container = mockDeep(); + container.get.calledWith(TemporalFileByPathFinder).mockReturnValue({ run: vi.fn() } as TemporalFileByPathFinder); + container.get.calledWith(TemporalFileDeleter).mockReturnValue({ run: vi.fn() } as TemporalFileDeleter); + container.get.calledWith(TemporalFileUploadQueue).mockReturnValue({ + enqueue: vi.fn(), + } as unknown as { enqueue: (props: unknown) => Promise }); }); it('should return errno 0 when release succeeds', async () => { diff --git a/src/backend/features/virtual-drive/controllers/operations/release.controller.ts b/src/backend/features/virtual-drive/controllers/operations/release.controller.ts index ee4315dbf0..052e783e61 100644 --- a/src/backend/features/virtual-drive/controllers/operations/release.controller.ts +++ b/src/backend/features/virtual-drive/controllers/operations/release.controller.ts @@ -3,6 +3,10 @@ import { Container } from 'diod'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { release } from '../../services/operations/release.service'; import { ensureLeadingSlash } from '../ensure-leading-slash'; +import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; +import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; +import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types'; + export async function releaseController(req: Request, res: Response, container: Container) { const rawPath: string = req.body.path ?? ''; const processName: string = req.body.processName ?? ''; @@ -10,8 +14,17 @@ export async function releaseController(req: Request, res: Response, container: msg: `[FUSE DAEMON] Release signal received for path: ${rawPath} by process: ${processName}`, }); const normalizedPath = ensureLeadingSlash(rawPath); + const finder = container.get(TemporalFileByPathFinder); + const deleter = container.get(TemporalFileDeleter); + const uploadQueue = container.get(TemporalFileUploadQueue); - const result = await release({ path: normalizedPath, processName, container }); + const result = await release({ + path: normalizedPath, + processName, + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: uploadQueue.enqueue.bind(uploadQueue), + }); if (result.error) { res.json({ errno: result.error.code }); diff --git a/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts b/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts deleted file mode 100644 index 4c9397a193..0000000000 --- a/src/backend/features/virtual-drive/services/lazy/LazyVirtualDriveHydrator.ts +++ /dev/null @@ -1,295 +0,0 @@ -import path from 'node:path'; -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { fetchFolder } from '../../../../../infra/drive-server/services/folder/services/fetch-folder'; -import { FolderDto, FileDto } from '../../../../../infra/drive-server/out/dto'; -import { FolderRepository } from '../../../../../context/virtual-drive/folders/domain/FolderRepository'; -import { FileRepository } from '../../../../../context/virtual-drive/files/domain/FileRepository'; -import { Folder } from '../../../../../context/virtual-drive/folders/domain/Folder'; -import { FolderNotFoundError } from '../../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; -import { createFolderFromServerFolder } from '../../../../../context/virtual-drive/folders/application/create/FolderCreatorFromServerFolder'; -import { createFileFromServerFile } from '../../../../../context/virtual-drive/remoteTree/application/FileCreatorFromServerFile'; -import { ServerFolderStatus } from '../../../../../context/shared/domain/ServerFolder'; -import { ServerFileStatus } from '../../../../../context/shared/domain/ServerFile'; -import { createOrUpdateFolderByBatch } from '../../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; -import { createOrUpdateFileByBatch } from '../../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; -import { DirectoryStateSqliteRepository, DirectoryStatusScope } from './DirectoryStateSqliteRepository'; -import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; -import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; - -type ReadDirectoryProps = { - path: string; -}; - -type EnsurePathLoadedProps = { - path: string; -}; - -type DirectoryEntries = { - files: string[]; - folders: string[]; -}; - -type Props = { - folderRepository: FolderRepository; - fileRepository: FileRepository; - directoryStateRepository: DirectoryStateSqliteRepository; -}; - -export const LazyVirtualDriveHydrator = Symbol('LazyVirtualDriveHydrator'); - -export type LazyVirtualDriveHydrator = ReturnType; - -export function createLazyVirtualDriveHydrator({ folderRepository, fileRepository, directoryStateRepository }: Props) { - const inflight = new Map>(); - - async function readDirectory({ path: requestedPath }: ReadDirectoryProps) { - const normalizedPath = normalizePath(requestedPath); - const statusScope: DirectoryStatusScope = 'EXISTS'; - const folder = await ensureFolderMaterialized({ path: normalizedPath, statusScope }); - - await refreshChildrenIfNeeded({ folder, statusScope }); - - return readLocalDirectory({ folder, statusScope }); - } - - async function ensurePathLoaded({ path: requestedPath }: EnsurePathLoadedProps) { - const normalizedPath = normalizePath(requestedPath); - const statusScope: DirectoryStatusScope = 'EXISTS'; - const parentPath = isRootPath(normalizedPath) ? normalizedPath : path.posix.dirname(normalizedPath); - - await ensureFolderMaterialized({ path: parentPath, statusScope }); - } - - async function ensureFolderMaterialized({ - path: requestedPath, - statusScope, - }: { - path: string; - statusScope: DirectoryStatusScope; - }): Promise { - const root = await resolveRootFolder(); - - if (isRootPath(requestedPath)) { - return root; - } - - let currentFolder = root; - let currentPath = ''; - - for (const segment of requestedPath.split('/').filter(Boolean)) { - currentPath = `${currentPath}/${segment}`; - - const existing = folderRepository.matchingPartial({ path: currentPath })[0]; - if (existing) { - currentFolder = existing; - continue; - } - - await refreshChildrenIfNeeded({ folder: currentFolder, statusScope }); - - const hydrated = folderRepository.matchingPartial({ path: currentPath })[0]; - if (!hydrated) { - throw new FuseError(FuseCodes.ENOENT, `[FUSE - Lazy] Folder not found: ${currentPath}`); - } - - currentFolder = hydrated; - } - - return currentFolder; - } - - async function refreshChildrenIfNeeded({ - folder, - statusScope, - }: { - folder: Folder; - statusScope: DirectoryStatusScope; - }) { - const cacheKey = `${folder.id}:${statusScope}`; - if (await directoryStateRepository.isFresh({ folderId: folder.id, statusScope })) { - return; - } - - const inFlight = inflight.get(cacheKey); - if (inFlight) { - return inFlight; - } - - const task = fetchAndStoreChildren({ folder, statusScope }).finally(() => { - inflight.delete(cacheKey); - }); - - inflight.set(cacheKey, task); - - return task; - } - - async function fetchAndStoreChildren({ folder, statusScope }: { folder: Folder; statusScope: DirectoryStatusScope }) { - try { - const { data, error } = await fetchFolder(folder.uuid); - - if (error || !data) { - throw error ?? new FolderNotFoundError(folder.path); - } - - const remoteFolders = data.children.filter(isExistingFolder).map((child) => toRemoteFolder(child)); - const remoteFiles = data.files.filter(isExistingFile).map((child) => toRemoteFile(child)); - - // SQLite (better-sqlite3) uses a single writer connection. - // Running both batch transactions in parallel causes nested transaction errors. - await createOrUpdateFolderByBatch({ folders: remoteFolders }); - await createOrUpdateFileByBatch({ files: remoteFiles }); - await folderRepository.deleteMatchingPartial({ parentId: folder.id }); - await fileRepository.deleteMatchingPartial({ folderId: folder.id }); - - const localFolders = data.children.filter(isExistingFolder).map((child) => { - return createFolderFromServerFolder( - { - bucket: child.bucket ?? null, - createdAt: child.createdAt, - id: child.id, - name: child.name, - parentId: child.parentId, - updatedAt: child.updatedAt, - plain_name: child.plainName, - status: ServerFolderStatus.EXISTS, - uuid: child.uuid, - }, - joinVirtualPath(folder.path, child.plainName), - ); - }); - - const localFiles = data.files.filter(isExistingFile).map((child) => { - return createFileFromServerFile( - { - bucket: child.bucket, - createdAt: child.createdAt, - encrypt_version: child.encryptVersion, - fileId: child.fileId ?? '', - folderId: child.folderId, - id: child.id, - modificationTime: child.modificationTime ?? child.updatedAt, - name: child.name, - size: Number.parseInt(String(child.size), 10), - type: child.type, - updatedAt: child.updatedAt, - userId: child.userId, - status: ServerFileStatus.EXISTS, - plainName: child.plainName, - uuid: child.uuid, - }, - joinVirtualPath(folder.path, buildFileName(child)), - ); - }); - - await Promise.all([ - ...localFolders.map((child) => folderRepository.add(child)), - ...localFiles.map((child) => fileRepository.upsert(child)), - ]); - - await directoryStateRepository.markLoaded({ folderId: folder.id, statusScope }); - } catch (error) { - await directoryStateRepository.markError({ folderId: folder.id, statusScope }); - logger.error({ msg: '[FUSE - Lazy] Failed to fetch folder children', error, path: folder.path }); - throw new FuseError(FuseCodes.EIO, `[FUSE - Lazy] Unable to hydrate path: ${folder.path}`); - } - } - - async function resolveRootFolder() { - const root = folderRepository.matchingPartial({ path: '/' })[0]; - - if (!root) { - throw new FuseError(FuseCodes.EIO, '[FUSE - Lazy] Root folder not initialized'); - } - - return root; - } - - function readLocalDirectory({ - folder, - statusScope, - }: { - folder: Folder; - statusScope: DirectoryStatusScope; - }): DirectoryEntries { - return { - folders: folderRepository - .matchingPartial({ parentId: folder.id, status: ServerFolderStatus.EXISTS }) - .map((child) => child.name), - files: fileRepository - .matchingPartial({ folderId: folder.id, status: ServerFileStatus.EXISTS }) - .map((child) => child.nameWithExtension), - }; - } - - return { - readDirectory, - ensurePathLoaded, - }; -} - -function normalizePath(pathToNormalize: string) { - const normalized = path.posix.normalize(pathToNormalize || '/'); - - if (normalized === '.') { - return '/'; - } - - return normalized.startsWith('/') ? normalized : `/${normalized}`; -} - -function isRootPath(pathToCheck: string) { - return pathToCheck === '/' || pathToCheck === ''; -} - -function joinVirtualPath(parentPath: string, name: string) { - return `${parentPath}/${name}`.replaceAll('//', '/'); -} - -function buildFileName(file: FileDto) { - return file.type ? `${file.plainName}.${file.type}` : file.plainName; -} - -function isExistingFolder(folder: FolderDto) { - return folder.status === 'EXISTS' && !folder.deleted && !folder.removed; -} - -function isExistingFile(file: FileDto) { - return file.status === 'EXISTS'; -} - -function toRemoteFolder(folder: FolderDto) { - return { - type: folder.type, - id: folder.id, - parentId: folder.parentId, - bucket: folder.bucket, - userId: folder.userId, - createdAt: folder.createdAt, - updatedAt: folder.updatedAt, - uuid: folder.uuid, - plainName: folder.plainName, - name: folder.name, - status: folder.status, - }; -} - -function toRemoteFile(file: FileDto) { - return { - id: file.id, - uuid: file.uuid, - fileId: file.fileId ?? '', - type: file.type, - size: Number.parseInt(String(file.size), 10), - bucket: file.bucket, - folderId: file.folderId, - folderUuid: file.folderUuid, - userId: file.userId, - modificationTime: file.modificationTime ?? file.updatedAt, - createdAt: file.createdAt, - updatedAt: file.updatedAt, - plainName: file.plainName, - name: file.name, - status: file.status, - }; -} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.test.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.test.ts new file mode 100644 index 0000000000..b578f4565c --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { FileDto, FolderDto } from '../../../../../../infra/drive-server/out/dto'; +import type { FileRepository } from '../../../../../../context/virtual-drive/files/domain/FileRepository'; +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { + createLocalFiles, + createLocalFolders, + getExistingFiles, + getExistingFolders, + readLocalDirectory, + toRemoteFiles, + toRemoteFolders, +} from './children'; + +vi.mock('../../../../../../context/virtual-drive/folders/application/create/FolderCreatorFromServerFolder', () => ({ + createFolderFromServerFolder: vi.fn((folder, virtualPath) => ({ folder, virtualPath })), +})); + +vi.mock('../../../../../../context/virtual-drive/remoteTree/application/FileCreatorFromServerFile', () => ({ + createFileFromServerFile: vi.fn((file, virtualPath) => ({ file, virtualPath })), +})); + +describe('children', () => { + it('should filter existing remote children', () => { + const existingFolder = { status: 'EXISTS', deleted: false, removed: false } as unknown as FolderDto; + const removedFolder = { status: 'EXISTS', deleted: true, removed: false } as unknown as FolderDto; + const existingFile = { status: 'EXISTS' } as unknown as FileDto; + const trashedFile = { status: 'TRASHED' } as unknown as FileDto; + + expect(getExistingFolders([existingFolder, removedFolder])).toStrictEqual([existingFolder]); + expect(getExistingFiles([existingFile, trashedFile])).toStrictEqual([existingFile]); + }); + + it('should map remote children', () => { + const folder = { + status: 'EXISTS', + deleted: false, + removed: false, + bucket: null, + createdAt: '2025-01-01T00:00:00.000Z', + id: 1, + name: 'docs', + parentId: 0, + updatedAt: '2025-01-01T00:00:00.000Z', + plainName: 'docs', + uuid: 'folder-uuid', + type: 'folder', + userId: 1, + } as unknown as FolderDto; + + const file = { + status: 'EXISTS', + bucket: 'bucket', + createdAt: '2025-01-01T00:00:00.000Z', + encryptVersion: '1', + fileId: 'file-id', + folderId: 1, + folderUuid: 'folder-uuid', + id: 2, + modificationTime: '2025-01-01T00:00:00.000Z', + name: 'file.txt', + size: 10, + type: 'txt', + updatedAt: '2025-01-01T00:00:00.000Z', + userId: 1, + plainName: 'file', + uuid: 'file-uuid', + } as unknown as FileDto; + + expect(toRemoteFolders([folder])).toStrictEqual([ + { + type: 'folder', + id: 1, + parentId: 0, + bucket: null, + userId: 1, + createdAt: '2025-01-01T00:00:00.000Z', + updatedAt: '2025-01-01T00:00:00.000Z', + uuid: 'folder-uuid', + plainName: 'docs', + name: 'docs', + status: 'EXISTS', + }, + ]); + + expect(toRemoteFiles([file])).toStrictEqual([ + { + id: 2, + uuid: 'file-uuid', + fileId: 'file-id', + type: 'txt', + size: 10, + bucket: 'bucket', + folderId: 1, + folderUuid: 'folder-uuid', + userId: 1, + modificationTime: '2025-01-01T00:00:00.000Z', + createdAt: '2025-01-01T00:00:00.000Z', + updatedAt: '2025-01-01T00:00:00.000Z', + plainName: 'file', + name: 'file.txt', + status: 'EXISTS', + }, + ]); + }); + + it('should build local children', () => { + const folder = { + status: 'EXISTS', + deleted: false, + removed: false, + bucket: null, + createdAt: '2025-01-01T00:00:00.000Z', + id: 1, + name: 'docs', + parentId: 0, + updatedAt: '2025-01-01T00:00:00.000Z', + plainName: 'docs', + uuid: 'folder-uuid', + type: 'folder', + userId: 1, + } as unknown as FolderDto; + + const file = { + status: 'EXISTS', + bucket: 'bucket', + createdAt: '2025-01-01T00:00:00.000Z', + encryptVersion: '1', + fileId: 'file-id', + folderId: 1, + folderUuid: 'folder-uuid', + id: 2, + modificationTime: '2025-01-01T00:00:00.000Z', + name: 'file.txt', + size: 10, + type: 'txt', + updatedAt: '2025-01-01T00:00:00.000Z', + userId: 1, + plainName: 'file', + uuid: 'file-uuid', + } as unknown as FileDto; + + expect(createLocalFolders({ folderPath: '/root', children: [folder] })).toStrictEqual([ + { + folder: { + bucket: null, + createdAt: '2025-01-01T00:00:00.000Z', + id: 1, + name: 'docs', + parentId: 0, + updatedAt: '2025-01-01T00:00:00.000Z', + plain_name: 'docs', + status: 'EXISTS', + uuid: 'folder-uuid', + }, + virtualPath: '/root/docs', + }, + ]); + + expect(createLocalFiles({ folderPath: '/root', files: [file] })).toStrictEqual([ + { + file: { + bucket: 'bucket', + createdAt: '2025-01-01T00:00:00.000Z', + encrypt_version: '1', + fileId: 'file-id', + folderId: 1, + id: 2, + modificationTime: '2025-01-01T00:00:00.000Z', + name: 'file.txt', + size: 10, + type: 'txt', + updatedAt: '2025-01-01T00:00:00.000Z', + userId: 1, + status: 'EXISTS', + plainName: 'file', + uuid: 'file-uuid', + }, + virtualPath: '/root/file.txt', + }, + ]); + }); + + it('should read local directory entries', () => { + const folder = { id: 10 } as unknown as Folder; + const folderRepository = { + matchingPartial: vi.fn().mockReturnValue([{ name: 'subfolder' }]), + } as unknown as FolderRepository; + const fileRepository = { + matchingPartial: vi.fn().mockReturnValue([{ nameWithExtension: 'file.txt' }]), + } as unknown as FileRepository; + + expect(readLocalDirectory({ folder, folderRepository, fileRepository })).toStrictEqual({ + folders: ['subfolder'], + files: ['file.txt'], + }); + }); +}); diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.ts new file mode 100644 index 0000000000..5667238b37 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/children.ts @@ -0,0 +1,144 @@ +import type { FileDto, FolderDto } from '../../../../../../infra/drive-server/out/dto'; +import { createFolderFromServerFolder } from '../../../../../../context/virtual-drive/folders/application/create/FolderCreatorFromServerFolder'; +import { createFileFromServerFile } from '../../../../../../context/virtual-drive/remoteTree/application/FileCreatorFromServerFile'; +import { ServerFolderStatus } from '../../../../../../context/shared/domain/ServerFolder'; +import { ServerFileStatus } from '../../../../../../context/shared/domain/ServerFile'; +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FileRepository } from '../../../../../../context/virtual-drive/files/domain/FileRepository'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { FileStatuses } from '../../../../../../context/virtual-drive/files/domain/FileStatus'; +import { buildFileName } from './file-name'; +import { joinVirtualPath } from './path'; + +type LocalFoldersProps = { + folderPath: string; + children: Array; +}; + +type LocalFilesProps = { + folderPath: string; + files: Array; +}; + +export function readLocalDirectory({ + folder, + folderRepository, + fileRepository, +}: { + folder: Folder; + folderRepository: FolderRepository; + fileRepository: FileRepository; +}) { + return { + folders: folderRepository + .matchingPartial({ parentId: folder.id, status: ServerFolderStatus.EXISTS }) + .map((child) => child.name), + files: fileRepository + .matchingPartial({ folderId: folder.id, status: FileStatuses.EXISTS }) + .map((child) => child.nameWithExtension), + }; +} + +export function getExistingFolders(children: Array) { + return children.filter(isExistingFolder); +} + +export function getExistingFiles(files: Array) { + return files.filter(isExistingFile); +} + +export function createLocalFolders({ folderPath, children }: LocalFoldersProps) { + return getExistingFolders(children).map((child) => { + return createFolderFromServerFolder( + { + bucket: child.bucket ?? null, + createdAt: child.createdAt, + id: child.id, + name: child.name, + parentId: child.parentId, + updatedAt: child.updatedAt, + plain_name: child.plainName, + status: ServerFolderStatus.EXISTS, + uuid: child.uuid, + }, + joinVirtualPath(folderPath, child.plainName), + ); + }); +} + +export function createLocalFiles({ folderPath, files }: LocalFilesProps) { + return getExistingFiles(files).map((child) => { + return createFileFromServerFile( + { + bucket: child.bucket, + createdAt: child.createdAt, + encrypt_version: child.encryptVersion, + fileId: child.fileId ?? '', + folderId: child.folderId, + id: child.id, + modificationTime: child.modificationTime ?? child.updatedAt, + name: child.name, + size: Number.parseInt(String(child.size), 10), + type: child.type, + updatedAt: child.updatedAt, + userId: child.userId, + status: ServerFileStatus.EXISTS, + plainName: child.plainName, + uuid: child.uuid, + }, + joinVirtualPath(folderPath, buildFileName(child)), + ); + }); +} + +export function toRemoteFolders(children: Array) { + return getExistingFolders(children).map((child) => toRemoteFolder(child)); +} + +export function toRemoteFiles(files: Array) { + return getExistingFiles(files).map((child) => toRemoteFile(child)); +} + +function isExistingFolder(folder: FolderDto) { + return folder.status === 'EXISTS' && !folder.deleted && !folder.removed; +} + +function isExistingFile(file: FileDto) { + return file.status === 'EXISTS'; +} + +function toRemoteFolder(folder: FolderDto) { + return { + type: folder.type, + id: folder.id, + parentId: folder.parentId, + bucket: folder.bucket, + userId: folder.userId, + createdAt: folder.createdAt, + updatedAt: folder.updatedAt, + uuid: folder.uuid, + plainName: folder.plainName, + name: folder.name, + status: folder.status, + }; +} + +function toRemoteFile(file: FileDto) { + return { + id: file.id, + uuid: file.uuid, + fileId: file.fileId ?? '', + type: file.type, + size: Number.parseInt(String(file.size), 10), + bucket: file.bucket, + folderId: file.folderId, + folderUuid: file.folderUuid, + userId: file.userId, + modificationTime: file.modificationTime ?? file.updatedAt, + createdAt: file.createdAt, + updatedAt: file.updatedAt, + plainName: file.plainName, + name: file.name, + status: file.status, + }; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts new file mode 100644 index 0000000000..367e278353 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts @@ -0,0 +1,94 @@ +import type { Identifier } from 'diod'; +import type { + LazyVirtualDriveHydratorProps, + ReadDirectoryProps, + EnsurePathLoadedProps, + HydratorStatusScope, +} from './types'; +import { normalizePath, isRootPath } from './path'; +import { ensureFolderMaterialized } from './ensure-folder-materialized'; +import { refreshChildrenIfNeeded } from './refresh-children-if-needed'; +import { fetchAndStoreChildren } from './fetch-and-store-children'; +import { readLocalDirectory } from './children'; + +export const LazyVirtualDriveHydrator = Symbol( + 'LazyVirtualDriveHydrator', +) as unknown as Identifier>; + +export type LazyVirtualDriveHydrator = ReturnType; + +type FetchAndStoreChildrenProps = Parameters[0]; +type FetchAndStoreChildrenInput = Pick; + +export function createLazyVirtualDriveHydratorService({ + folderRepository, + fileRepository, + directoryStateRepository, +}: LazyVirtualDriveHydratorProps) { + const inflight = new Map>(); + + function getFetchAndStoreChildren() { + return ({ folder, statusScope }: FetchAndStoreChildrenInput) => + fetchAndStoreChildren({ + folder, + statusScope, + folderRepository, + fileRepository, + directoryStateRepository, + }); + } + + async function readDirectory({ path: requestedPath }: ReadDirectoryProps) { + const normalizedPath = normalizePath(requestedPath); + const statusScope: HydratorStatusScope = 'EXISTS'; + + const folder = await ensureFolderMaterialized({ + requestedPath: normalizedPath, + statusScope, + folderRepository, + refreshChildrenIfNeeded: (props) => + refreshChildrenIfNeeded({ + ...props, + directoryStateRepository, + inflight, + fetchAndStoreChildren: getFetchAndStoreChildren(), + }), + }); + + await refreshChildrenIfNeeded({ + folder, + statusScope, + directoryStateRepository, + inflight, + fetchAndStoreChildren: getFetchAndStoreChildren(), + }); + + return readLocalDirectory({ folder, folderRepository, fileRepository }); + } + + async function ensurePathLoaded({ path: requestedPath }: EnsurePathLoadedProps) { + const normalizedPath = normalizePath(requestedPath); + const statusScope: HydratorStatusScope = 'EXISTS'; + const parentPath = isRootPath(normalizedPath) + ? normalizedPath + : normalizedPath.slice(0, normalizedPath.lastIndexOf('/')) || '/'; + + await ensureFolderMaterialized({ + requestedPath: parentPath, + statusScope, + folderRepository, + refreshChildrenIfNeeded: (props) => + refreshChildrenIfNeeded({ + ...props, + directoryStateRepository, + inflight, + fetchAndStoreChildren: getFetchAndStoreChildren(), + }), + }); + } + + return { + readDirectory, + ensurePathLoaded, + }; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.test.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.test.ts new file mode 100644 index 0000000000..9f5a29337e --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { ensureFolderMaterialized } from './ensure-folder-materialized'; + +describe('ensure-folder-materialized', () => { + it('should return the root folder for root paths', async () => { + const root = { id: 1, uuid: 'root', path: '/' } as unknown as Folder; + const folderRepository = { + matchingPartial: vi.fn().mockReturnValue([root]), + } as unknown as FolderRepository; + const refreshChildrenIfNeeded = vi.fn(); + + await expect( + ensureFolderMaterialized({ + requestedPath: '/', + statusScope: 'EXISTS', + folderRepository, + refreshChildrenIfNeeded, + }), + ).resolves.toBe(root); + + expect(refreshChildrenIfNeeded).not.toHaveBeenCalled(); + }); + + it('should hydrate missing segments before returning a folder', async () => { + const root = { id: 1, uuid: 'root', path: '/' } as unknown as Folder; + const docs = { id: 2, uuid: 'docs', path: '/docs' } as unknown as Folder; + let hydrated = false; + + const folderRepository = { + matchingPartial: vi.fn(({ path }: { path: string }) => { + if (path === '/') return [root]; + if (path === '/docs') return hydrated ? [docs] : []; + return []; + }), + } as unknown as FolderRepository; + const refreshChildrenIfNeeded = vi.fn(async () => { + hydrated = true; + }); + + await expect( + ensureFolderMaterialized({ + requestedPath: '/docs', + statusScope: 'EXISTS', + folderRepository, + refreshChildrenIfNeeded, + }), + ).resolves.toBe(docs); + + expect(refreshChildrenIfNeeded).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.ts new file mode 100644 index 0000000000..ad2291b268 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/ensure-folder-materialized.ts @@ -0,0 +1,49 @@ +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { FuseCodes } from '../../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { FuseError } from '../../../../../../apps/drive/fuse/callbacks/FuseErrors'; +import type { HydratorStatusScope } from './types'; +import { isRootPath } from './path'; +import { resolveRootFolder } from './resolve-root-folder'; + +export async function ensureFolderMaterialized({ + requestedPath, + statusScope, + folderRepository, + refreshChildrenIfNeeded, +}: { + requestedPath: string; + statusScope: HydratorStatusScope; + folderRepository: FolderRepository; + refreshChildrenIfNeeded: (props: { folder: Folder; statusScope: HydratorStatusScope }) => Promise; +}) { + const root = await resolveRootFolder({ folderRepository }); + + if (isRootPath(requestedPath)) { + return root; + } + + let currentFolder = root; + let currentPath = ''; + + for (const segment of requestedPath.split('/').filter(Boolean)) { + currentPath = `${currentPath}/${segment}`; + + const existing = folderRepository.matchingPartial({ path: currentPath })[0]; + if (existing) { + currentFolder = existing; + continue; + } + + await refreshChildrenIfNeeded({ folder: currentFolder, statusScope }); + + const hydrated = folderRepository.matchingPartial({ path: currentPath })[0]; + if (!hydrated) { + throw new FuseError(FuseCodes.ENOENT, `[FUSE - Lazy] Folder not found: ${currentPath}`); + } + + currentFolder = hydrated; + } + + return currentFolder; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.test.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.test.ts new file mode 100644 index 0000000000..c041fb02df --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi } from 'vitest'; +import { FuseCodes } from '../../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import type { FileRepository } from '../../../../../../context/virtual-drive/files/domain/FileRepository'; +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import { fetchAndStoreChildren } from './fetch-and-store-children'; +import type { DirectoryStateSqliteRepository } from '../DirectoryStateSqliteRepository'; + +vi.mock('../../../../../../infra/drive-server/services/folder/services/fetch-folder', () => ({ + fetchFolder: vi.fn(), +})); + +vi.mock('../../../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch', () => ({ + createOrUpdateFolderByBatch: vi.fn(), +})); + +vi.mock('../../../../../../infra/sqlite/services/file/create-or-update-file-by-batch', () => ({ + createOrUpdateFileByBatch: vi.fn(), +})); + +vi.mock('./children', () => ({ + toRemoteFolders: vi.fn(() => [{ id: 1 }]), + toRemoteFiles: vi.fn(() => [{ id: 2 }]), + createLocalFolders: vi.fn(() => [{ id: 3 }]), + createLocalFiles: vi.fn(() => [{ id: 4 }]), +})); + +import { fetchFolder } from '../../../../../../infra/drive-server/services/folder/services/fetch-folder'; +import { createOrUpdateFolderByBatch } from '../../../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; +import { createOrUpdateFileByBatch } from '../../../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; + +describe('fetch-and-store-children', () => { + it('should store children and mark the state as loaded', async () => { + const folder = { id: 10, uuid: 'folder-uuid', path: '/docs' } as unknown as Folder; + const addFolder = vi.fn(); + const deleteMatchingFolderPartial = vi.fn(); + const upsertFile = vi.fn(); + const deleteMatchingFilePartial = vi.fn(); + const markLoaded = vi.fn(); + const markError = vi.fn(); + + const folderRepository = { + deleteMatchingPartial: deleteMatchingFolderPartial, + add: addFolder, + } as unknown as FolderRepository; + const fileRepository = { + deleteMatchingPartial: deleteMatchingFilePartial, + upsert: upsertFile, + } as unknown as FileRepository; + const directoryStateRepository = { + markLoaded, + markError, + } as unknown as DirectoryStateSqliteRepository; + + vi.mocked(fetchFolder).mockResolvedValue({ + data: { + children: [], + files: [], + }, + error: undefined, + } as never); + + vi.mocked(createOrUpdateFolderByBatch).mockResolvedValue({ data: [] } as never); + vi.mocked(createOrUpdateFileByBatch).mockResolvedValue({ data: [] } as never); + + await fetchAndStoreChildren({ + folder, + statusScope: 'EXISTS', + folderRepository, + fileRepository, + directoryStateRepository, + }); + + expect(addFolder).toHaveBeenCalledTimes(1); + expect(upsertFile).toHaveBeenCalledTimes(1); + expect(deleteMatchingFolderPartial).toHaveBeenCalledWith({ parentId: 10 }); + expect(deleteMatchingFilePartial).toHaveBeenCalledWith({ folderId: 10 }); + expect(deleteMatchingFolderPartial.mock.invocationCallOrder[0]).toBeLessThan(addFolder.mock.invocationCallOrder[0]); + expect(deleteMatchingFilePartial.mock.invocationCallOrder[0]).toBeLessThan(upsertFile.mock.invocationCallOrder[0]); + expect(markLoaded).toHaveBeenCalledWith({ folderId: 10, statusScope: 'EXISTS' }); + expect(markError).not.toHaveBeenCalled(); + }); + + it('should mark the state as error when fetch fails', async () => { + const folder = { id: 10, uuid: 'folder-uuid', path: '/docs' } as unknown as Folder; + const markLoaded = vi.fn(); + const markError = vi.fn(); + const folderRepository = { + deleteMatchingPartial: vi.fn(), + add: vi.fn(), + } as unknown as FolderRepository; + const fileRepository = { + deleteMatchingPartial: vi.fn(), + upsert: vi.fn(), + } as unknown as FileRepository; + const directoryStateRepository = { + markLoaded, + markError, + } as unknown as DirectoryStateSqliteRepository; + + vi.mocked(fetchFolder).mockResolvedValue({ + data: undefined, + error: new Error('boom') as never, + } as never); + + await expect( + fetchAndStoreChildren({ + folder, + statusScope: 'EXISTS', + folderRepository, + fileRepository, + directoryStateRepository, + }), + ).rejects.toMatchObject({ code: FuseCodes.EIO }); + + expect(markError).toHaveBeenCalledWith({ folderId: 10, statusScope: 'EXISTS' }); + }); +}); diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.ts new file mode 100644 index 0000000000..c9d1bc733c --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/fetch-and-store-children.ts @@ -0,0 +1,60 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { fetchFolder } from '../../../../../../infra/drive-server/services/folder/services/fetch-folder'; +import { createOrUpdateFolderByBatch } from '../../../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; +import { createOrUpdateFileByBatch } from '../../../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; +import { FolderNotFoundError } from '../../../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; +import { FuseCodes } from '../../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { FuseError } from '../../../../../../apps/drive/fuse/callbacks/FuseErrors'; +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import type { FileRepository } from '../../../../../../context/virtual-drive/files/domain/FileRepository'; +import type { DirectoryStateSqliteRepository } from '../DirectoryStateSqliteRepository'; +import type { HydratorStatusScope } from './types'; +import { createLocalFiles, createLocalFolders, toRemoteFiles, toRemoteFolders } from './children'; + +type Props = { + folder: Folder; + statusScope: HydratorStatusScope; + folderRepository: FolderRepository; + fileRepository: FileRepository; + directoryStateRepository: DirectoryStateSqliteRepository; +}; + +export async function fetchAndStoreChildren({ + folder, + statusScope, + folderRepository, + fileRepository, + directoryStateRepository, +}: Props) { + try { + const { data, error } = await fetchFolder(folder.uuid); + + if (error || !data) { + throw error ?? new FolderNotFoundError(folder.path); + } + + const remoteFolders = toRemoteFolders(data.children); + const remoteFiles = toRemoteFiles(data.files); + + await createOrUpdateFolderByBatch({ folders: remoteFolders }); + await createOrUpdateFileByBatch({ files: remoteFiles }); + + await folderRepository.deleteMatchingPartial({ parentId: folder.id }); + await fileRepository.deleteMatchingPartial({ folderId: folder.id }); + + const localFolders = createLocalFolders({ folderPath: folder.path, children: data.children }); + const localFiles = createLocalFiles({ folderPath: folder.path, files: data.files }); + + await Promise.all([ + ...localFolders.map((child) => folderRepository.add(child)), + ...localFiles.map((child) => fileRepository.upsert(child)), + ]); + + await directoryStateRepository.markLoaded({ folderId: folder.id, statusScope }); + } catch (error) { + await directoryStateRepository.markError({ folderId: folder.id, statusScope }); + logger.error({ msg: '[FUSE - Lazy] Failed to fetch folder children', error, path: folder.path }); + throw new FuseError(FuseCodes.EIO, `[FUSE - Lazy] Unable to hydrate path: ${folder.path}`); + } +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/file-name.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/file-name.ts new file mode 100644 index 0000000000..2cf5fa0471 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/file-name.ts @@ -0,0 +1,5 @@ +import type { FileDto } from '../../../../../../infra/drive-server/out/dto'; + +export function buildFileName(file: FileDto) { + return file.type ? `${file.plainName}.${file.type}` : file.plainName; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.test.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.test.ts new file mode 100644 index 0000000000..65c1806ae9 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { isRootPath, joinVirtualPath, normalizePath } from './path'; + +describe('path', () => { + it('should normalize paths', () => { + expect(normalizePath('')).toBe('/'); + expect(normalizePath('folder')).toBe('/folder'); + expect(normalizePath('/folder//sub')).toBe('/folder/sub'); + }); + + it('should detect root paths', () => { + expect(isRootPath('/')).toBe(true); + expect(isRootPath('')).toBe(true); + expect(isRootPath('/folder')).toBe(false); + }); + + it('should join virtual paths', () => { + expect(joinVirtualPath('/folder', 'file.txt')).toBe('/folder/file.txt'); + expect(joinVirtualPath('/folder/', 'file.txt')).toBe('/folder/file.txt'); + }); +}); diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.ts new file mode 100644 index 0000000000..27f80755fb --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/path.ts @@ -0,0 +1,19 @@ +import path from 'node:path'; + +export function normalizePath(pathToNormalize: string) { + const normalized = path.posix.normalize(pathToNormalize || '/'); + + if (normalized === '.') { + return '/'; + } + + return normalized.startsWith('/') ? normalized : `/${normalized}`; +} + +export function isRootPath(pathToCheck: string) { + return pathToCheck === '/' || pathToCheck === ''; +} + +export function joinVirtualPath(parentPath: string, name: string) { + return `${parentPath}/${name}`.replaceAll('//', '/'); +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/refresh-children-if-needed.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/refresh-children-if-needed.ts new file mode 100644 index 0000000000..19e1ae114a --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/refresh-children-if-needed.ts @@ -0,0 +1,36 @@ +import type { Folder } from '../../../../../../context/virtual-drive/folders/domain/Folder'; +import type { DirectoryStateSqliteRepository } from '../DirectoryStateSqliteRepository'; +import type { HydratorStatusScope } from './types'; + +export async function refreshChildrenIfNeeded({ + folder, + statusScope, + directoryStateRepository, + inflight, + fetchAndStoreChildren, +}: { + folder: Folder; + statusScope: HydratorStatusScope; + directoryStateRepository: DirectoryStateSqliteRepository; + inflight: Map>; + fetchAndStoreChildren: (props: { folder: Folder; statusScope: HydratorStatusScope }) => Promise; +}) { + const cacheKey = `${folder.id}:${statusScope}`; + + if (await directoryStateRepository.isFresh({ folderId: folder.id, statusScope })) { + return; + } + + const inFlight = inflight.get(cacheKey); + if (inFlight) { + return inFlight; + } + + const task = fetchAndStoreChildren({ folder, statusScope }).finally(() => { + inflight.delete(cacheKey); + }); + + inflight.set(cacheKey, task); + + return task; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/resolve-root-folder.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/resolve-root-folder.ts new file mode 100644 index 0000000000..79ecfa5bd4 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/resolve-root-folder.ts @@ -0,0 +1,13 @@ +import { FuseCodes } from '../../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { FuseError } from '../../../../../../apps/drive/fuse/callbacks/FuseErrors'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; + +export async function resolveRootFolder({ folderRepository }: { folderRepository: FolderRepository }) { + const root = folderRepository.matchingPartial({ path: '/' })[0]; + + if (!root) { + throw new FuseError(FuseCodes.EIO, '[FUSE - Lazy] Root folder not initialized'); + } + + return root; +} diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/types.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/types.ts new file mode 100644 index 0000000000..d1d0c398e3 --- /dev/null +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/types.ts @@ -0,0 +1,19 @@ +import type { FileRepository } from '../../../../../../context/virtual-drive/files/domain/FileRepository'; +import type { FolderRepository } from '../../../../../../context/virtual-drive/folders/domain/FolderRepository'; +import type { DirectoryStateSqliteRepository } from '../DirectoryStateSqliteRepository'; + +export type LazyVirtualDriveHydratorProps = { + folderRepository: FolderRepository; + fileRepository: FileRepository; + directoryStateRepository: DirectoryStateSqliteRepository; +}; + +export type ReadDirectoryProps = { + path: string; +}; + +export type EnsurePathLoadedProps = { + path: string; +}; + +export type HydratorStatusScope = 'EXISTS'; diff --git a/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts b/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts index 081cde4e92..d45fb0ae51 100644 --- a/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/get-attributes.service.test.ts @@ -9,7 +9,7 @@ import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import type { File } from '../../../../../context/virtual-drive/files/domain/File'; import type { Folder } from '../../../../../context/virtual-drive/folders/domain/Folder'; import type { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; vi.mock('@internxt/drive-desktop-core/build/backend'); diff --git a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts index b6abd81cc6..316ea06b31 100644 --- a/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts +++ b/src/backend/features/virtual-drive/services/operations/get-attributes.service.ts @@ -7,7 +7,7 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { SingleFolderMatchingSearcher } from '../../../../../context/virtual-drive/folders/application/SingleFolderMatchingSearcher'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; async function findLocalEntry(path: string, container: Container) { const file = await container.get(FirstsFileSearcher).run({ diff --git a/src/backend/features/virtual-drive/services/operations/open.service.test.ts b/src/backend/features/virtual-drive/services/operations/open.service.test.ts index 6327670044..949bec5b7e 100644 --- a/src/backend/features/virtual-drive/services/operations/open.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/open.service.test.ts @@ -5,7 +5,7 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; import type { File } from '../../../../../context/virtual-drive/files/domain/File'; describe('open', () => { diff --git a/src/backend/features/virtual-drive/services/operations/open.service.ts b/src/backend/features/virtual-drive/services/operations/open.service.ts index d8cd5af300..e665113828 100644 --- a/src/backend/features/virtual-drive/services/operations/open.service.ts +++ b/src/backend/features/virtual-drive/services/operations/open.service.ts @@ -6,7 +6,7 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; export async function open(path: string, processName: string, container: Container): Promise> { try { diff --git a/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts b/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts index 960f7b4558..00aeae0ff2 100644 --- a/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/opendir.service.test.ts @@ -5,7 +5,7 @@ import { TemporalFileByFolderFinder } from '../../../../../context/storage/Tempo import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import { FILE_MODE, FOLDER_MODE } from '../../constants'; import type { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; describe('opendir', () => { diff --git a/src/backend/features/virtual-drive/services/operations/opendir.service.ts b/src/backend/features/virtual-drive/services/operations/opendir.service.ts index dcc0d055d0..ae6548862a 100644 --- a/src/backend/features/virtual-drive/services/operations/opendir.service.ts +++ b/src/backend/features/virtual-drive/services/operations/opendir.service.ts @@ -4,7 +4,7 @@ import { FILE_MODE, FOLDER_MODE } from '../../constants'; import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; import { TemporalFileByFolderFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByFolderFinder'; import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { LazyVirtualDriveHydrator } from '../lazy/LazyVirtualDriveHydrator'; +import { LazyVirtualDriveHydrator } from '../lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service'; export type DirEntry = { name: string; diff --git a/src/backend/features/virtual-drive/services/operations/release.service.test.ts b/src/backend/features/virtual-drive/services/operations/release.service.test.ts index c587f603b5..e9afb2f90b 100644 --- a/src/backend/features/virtual-drive/services/operations/release.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/release.service.test.ts @@ -1,8 +1,8 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mockDeep } from 'vitest-mock-extended'; -import { Container } from 'diod'; import { release } from './release.service'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; -import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; +import type { TemporalFileUploadQueue as TemporalFileUploadQueueService } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types'; import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; import { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; @@ -12,7 +12,9 @@ import { clearUploadSizeLimitBlockedPath, isUploadSizeLimitBlockedPath, markUploadSizeLimitBlockedPath, -} from '../../../user/file-size-limit/add-max-file-size-rejection'; +} from '../../../user/file-size-limit/upload-size-limit-blocked-paths'; + +vi.mock('../../../user/file-size-limit', () => ({})); function createTemporalFile(path: string): TemporalFile { return TemporalFile.from({ path, size: 100, createdAt: new Date(), modifiedAt: new Date() }); @@ -23,16 +25,12 @@ function createAuxiliaryFile(path: string): TemporalFile { } describe('release', () => { - let container: ReturnType>; const finder = mockDeep(); - const queue = mockDeep(); + const queue = mockDeep(); const deleter = mockDeep(); beforeEach(() => { - container = mockDeep(); - container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(finder); - container.get.calledWith(TemporalFileUploadQueue).mockReturnValue(queue); - container.get.calledWith(TemporalFileDeleter).mockReturnValue(deleter); + vi.clearAllMocks(); queue.enqueue.mockResolvedValue(undefined); clearUploadSizeLimitBlockedPath('/Documents/report.pdf'); }); @@ -41,7 +39,13 @@ describe('release', () => { it('should return success without uploading', async () => { finder.run.mockResolvedValue(undefined); - const { data, error } = await release({ path: '/Documents/file.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/file.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); @@ -53,7 +57,13 @@ describe('release', () => { it('should return success, skip upload and delete it', async () => { finder.run.mockResolvedValue(createAuxiliaryFile('/Documents/.~lock.file.odt#')); - const { data, error } = await release({ path: '/Documents/.~lock.file.odt#', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/.~lock.file.odt#', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); @@ -67,7 +77,13 @@ describe('release', () => { const temporalFile = createTemporalFile('/Documents/report.pdf'); finder.run.mockResolvedValue(temporalFile); - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/report.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); @@ -79,7 +95,13 @@ describe('release', () => { finder.run.mockResolvedValue(temporalFile); markUploadSizeLimitBlockedPath('/Documents/report.pdf'); - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/report.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); @@ -92,7 +114,13 @@ describe('release', () => { finder.run.mockResolvedValue(createTemporalFile('/Documents/report.pdf')); queue.enqueue.mockRejectedValue(new UploadSizeLimitError()); - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/report.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(error).toBeUndefined(); expect(data).toBeUndefined(); @@ -103,7 +131,13 @@ describe('release', () => { finder.run.mockResolvedValue(createTemporalFile('/Documents/report.pdf')); queue.enqueue.mockRejectedValue(new Error('Network error')); - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/report.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(data).toBeUndefined(); expect(error?.code).toBe(FuseCodes.EIO); @@ -115,7 +149,13 @@ describe('release', () => { it('should return EIO without uploading or deleting', async () => { finder.run.mockRejectedValue(new Error('DB error')); - const { data, error } = await release({ path: '/Documents/report.pdf', processName: 'cat', container }); + const { data, error } = await release({ + path: '/Documents/report.pdf', + processName: 'cat', + findTemporalFileByPath: finder.run.bind(finder), + deleteTemporalFile: deleter.run.bind(deleter), + enqueueTemporalFile: queue.enqueue.bind(queue), + }); expect(data).toBeUndefined(); expect(error?.code).toBe(FuseCodes.EIO); diff --git a/src/backend/features/virtual-drive/services/operations/release.service.ts b/src/backend/features/virtual-drive/services/operations/release.service.ts index a7c42df19a..0fe15014d6 100644 --- a/src/backend/features/virtual-drive/services/operations/release.service.ts +++ b/src/backend/features/virtual-drive/services/operations/release.service.ts @@ -1,20 +1,20 @@ -import { Container } from 'diod'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { type Result } from '../../../../../context/shared/domain/Result'; import { FuseError, FuseIOError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors'; -import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; -import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue'; -import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter'; +import type { TemporalFile } from '../../../../../context/storage/TemporalFiles/domain/TemporalFile'; +import type { EnqueueProps } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types'; import { UploadSizeLimitError } from '../../../user/file-size-limit/upload-size-limit-error'; import { clearUploadSizeLimitBlockedPath, isUploadSizeLimitBlockedPath, -} from '../../../user/file-size-limit/add-max-file-size-rejection'; +} from '../../../user/file-size-limit/upload-size-limit-blocked-paths'; type Props = { path: string; processName: string; - container: Container; + findTemporalFileByPath: (path: string) => Promise; + deleteTemporalFile: (path: string) => Promise; + enqueueTemporalFile: (props: EnqueueProps) => Promise; }; // v.2.6.0 @@ -24,9 +24,15 @@ type Props = { // one for metadata and one for the actual content. // The issue is that when each descriptor closes, it triggers a release, // resulting in a duplicate request to create the file remotely. -export async function release({ path, processName, container }: Props): Promise> { +export async function release({ + path, + processName, + findTemporalFileByPath, + deleteTemporalFile, + enqueueTemporalFile, +}: Props): Promise> { try { - const temporalFile = await container.get(TemporalFileByPathFinder).run(path); + const temporalFile = await findTemporalFileByPath(path); if (!temporalFile) { logger.debug({ msg: '[Release] No temporal file found, nothing to upload', path, processName }); @@ -35,7 +41,7 @@ export async function release({ path, processName, container }: Props): Promise< if (temporalFile.isAuxiliary()) { logger.debug({ msg: '[Release] Auxiliary file detected, deleting without upload', path, processName }); - await container.get(TemporalFileDeleter).run(path); + await deleteTemporalFile(path); return { data: undefined }; } @@ -45,12 +51,12 @@ export async function release({ path, processName, container }: Props): Promise< path, processName, }); - await container.get(TemporalFileDeleter).run(path); + await deleteTemporalFile(path); return { data: undefined }; } try { - await container.get(TemporalFileUploadQueue).enqueue({ temporalFile, path, processName }); + await enqueueTemporalFile({ temporalFile, path, processName }); logger.debug({ msg: '[Release] Temporal file queued for upload', path, processName }); return { data: undefined }; } catch (uploadError) { @@ -65,7 +71,7 @@ export async function release({ path, processName, container }: Props): Promise< } logger.error({ msg: '[Release] Upload failed, deleting temporal file', error: uploadError, path, processName }); - await container.get(TemporalFileDeleter).run(path); + await deleteTemporalFile(path); return { error: new FuseIOError('Upload failed due to insufficient storage or network issues.') }; } } catch (err: unknown) { diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts deleted file mode 100644 index f522353b94..0000000000 --- a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { PATHS } from '../../../../../core/electron/paths'; -import { TemporalFileDeleter } from '../deletion/TemporalFileDeleter'; -import { TemporalFile } from '../../domain/TemporalFile'; -import { TemporalFileRepository } from '../../domain/TemporalFileRepository'; -import { TemporalFileUploader } from './TemporalFileUploader'; -import { FirstsFileSearcher } from '../../../../virtual-drive/files/application/search/FirstsFileSearcher'; -import { FileStatuses } from '../../../../virtual-drive/files/domain/FileStatus'; -import { preserveRejectedFileSizeTooBig } from '../../../../../backend/features/user/file-size-limit'; -import { UploadSizeLimitError } from '../../../../../backend/features/user/file-size-limit/upload-size-limit-error'; - -type Props = { - temporalFile: TemporalFile; - path: string; - processName: string; -}; - -type UploadTask = { - temporalFile: TemporalFile; - path: string; - processName: string; -}; - -type FactoryProps = { - repository: TemporalFileRepository; - uploader: TemporalFileUploader; - deleter: TemporalFileDeleter; - fileSearcher: FirstsFileSearcher; -}; - -export const TemporalFileUploadQueue = Symbol('TemporalFileUploadQueue'); - -export type TemporalFileUploadQueue = ReturnType; - -export function createTemporalFileUploadQueue({ repository, uploader, deleter, fileSearcher }: FactoryProps) { - const queuedPaths = new Set(); - const tasks: UploadTask[] = []; - let draining = false; - - async function enqueue({ temporalFile, path, processName }: Props) { - if (queuedPaths.has(path)) { - logger.debug({ msg: '[UploadQueue] Path already queued', path, processName }); - return; - } - - const stagedTemporalFile = await repository.stage(temporalFile.path, PATHS.UPLOAD_QUEUE); - - tasks.push({ - temporalFile: stagedTemporalFile, - path, - processName, - }); - queuedPaths.add(path); - - void drain(); - } - - async function drain() { - if (draining) { - return; - } - - draining = true; - - try { - while (tasks.length > 0) { - const task = tasks[0]; - - try { - await upload(task); - await deleter.run(task.path); - } catch (error) { - if (error instanceof UploadSizeLimitError) { - await preserveRejected(task); - } else { - logger.error({ - msg: '[UploadQueue] Upload failed, keeping staged file in queue folder', - error, - path: task.path, - }); - } - } finally { - tasks.shift(); - queuedPaths.delete(task.path); - } - } - } finally { - draining = false; - } - } - - async function upload(task: UploadTask) { - const existingFile = await fileSearcher.run({ path: task.path, status: FileStatuses.EXISTS }); - const replaces = existingFile - ? { contentsId: existingFile.contentsId, name: existingFile.name, extension: existingFile.type } - : undefined; - - await uploader.run(task.temporalFile, replaces); - - logger.debug({ msg: '[UploadQueue] Upload completed', path: task.path, processName: task.processName }); - } - - async function preserveRejected(task: UploadTask) { - if (!task.temporalFile.contentFilePath) { - logger.warn({ msg: '[UploadQueue] Rejected file missing content path', path: task.path }); - return; - } - - const { error } = await preserveRejectedFileSizeTooBig({ - originalPath: task.path, - temporalContentPath: task.temporalFile.contentFilePath, - size: task.temporalFile.size.value, - }); - - if (error) { - logger.error({ msg: '[UploadQueue] Failed to preserve oversized file', error, path: task.path }); - return; - } - - await deleter.run(task.path); - - logger.warn({ msg: '[UploadQueue] Preserved oversized file after queue rejection', path: task.path }); - } - - return { - enqueue, - }; -} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service.ts new file mode 100644 index 0000000000..9da24baf3b --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service.ts @@ -0,0 +1,14 @@ +import { createTemporalFileUploadQueueState } from './state'; +import type { FactoryProps } from './types'; +import { enqueueUpload } from './enqueue-upload'; + +export function createTemporalFileUploadQueueService(props: FactoryProps) { + const state = createTemporalFileUploadQueueState(); + + return { + enqueue: enqueueUpload.bind(null, { + ...props, + state, + }), + }; +} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.test.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.test.ts new file mode 100644 index 0000000000..c56f8a1b0e --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from 'vitest'; +import { UploadSizeLimitError } from '../../../../../../backend/features/user/file-size-limit/upload-size-limit-error'; +import { createTemporalFileUploadQueueState } from './state'; +import { drainUploadQueue } from './drain-upload-queue'; + +vi.mock('./upload-queued-task', () => ({ + uploadQueuedTask: vi.fn(), +})); + +vi.mock('./preserve-rejected-upload', () => ({ + preserveRejectedUpload: vi.fn(), +})); + +import { uploadQueuedTask } from './upload-queued-task'; +import { preserveRejectedUpload } from './preserve-rejected-upload'; + +describe('drain-upload-queue', () => { + it('should process queued tasks and clear state', async () => { + const state = createTemporalFileUploadQueueState(); + const task = { + temporalFile: { path: '/staged/file.txt' }, + path: '/target/file.txt', + processName: 'process', + }; + state.tasks.push(task as never); + state.queuedPaths.add(task.path); + + const deleter = { run: vi.fn().mockResolvedValue(undefined) }; + + vi.mocked(uploadQueuedTask).mockResolvedValue(undefined); + + await drainUploadQueue({ + repository: {} as never, + uploader: {} as never, + deleter, + fileSearcher: {} as never, + state, + } as never); + + expect(uploadQueuedTask).toHaveBeenCalledTimes(1); + expect(deleter.run).toHaveBeenCalledWith(task.path); + expect(state.tasks).toHaveLength(0); + expect(state.queuedPaths.has(task.path)).toBe(false); + expect(state.draining).toBe(false); + }); + + it('should preserve oversized uploads', async () => { + const state = createTemporalFileUploadQueueState(); + const task = { + temporalFile: { path: '/staged/file.txt' }, + path: '/target/file.txt', + processName: 'process', + }; + state.tasks.push(task as never); + state.queuedPaths.add(task.path); + + const deleter = { run: vi.fn().mockResolvedValue(undefined) }; + + vi.mocked(uploadQueuedTask).mockRejectedValue(new UploadSizeLimitError()); + + await drainUploadQueue({ + repository: {} as never, + uploader: {} as never, + deleter, + fileSearcher: {} as never, + state, + } as never); + + expect(preserveRejectedUpload).toHaveBeenCalledTimes(1); + expect(deleter.run).not.toHaveBeenCalled(); + expect(state.tasks).toHaveLength(0); + expect(state.queuedPaths.has(task.path)).toBe(false); + }); +}); diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.ts new file mode 100644 index 0000000000..845aa4c219 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/drain-upload-queue.ts @@ -0,0 +1,56 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { UploadSizeLimitError } from '../../../../../../backend/features/user/file-size-limit/upload-size-limit-error'; +import type { QueueContext, QueueState } from './types'; +import { preserveRejectedUpload } from './preserve-rejected-upload'; +import { uploadQueuedTask } from './upload-queued-task'; + +type Props = { + task: QueueState['tasks'][number]; + uploader: QueueContext['uploader']; + deleter: QueueContext['deleter']; + fileSearcher: QueueContext['fileSearcher']; + state: QueueState; +}; + +async function processTask({ task, uploader, deleter, fileSearcher, state }: Props) { + try { + await uploadQueuedTask({ task, uploader, fileSearcher }); + await deleter.run(task.path); + } catch (error) { + if (error instanceof UploadSizeLimitError) { + await preserveRejectedUpload({ task, deleter }); + return; + } + + logger.error({ + msg: '[UploadQueue] Upload failed, keeping staged file in queue folder', + error, + path: task.path, + }); + } finally { + state.tasks.shift(); + state.queuedPaths.delete(task.path); + } +} + +export async function drainUploadQueue({ + uploader, + deleter, + fileSearcher, + state, +}: QueueContext & { state: QueueState }) { + if (state.draining) { + return; + } + + state.draining = true; + + try { + while (state.tasks.length > 0) { + const task = state.tasks[0]; + await processTask({ task, uploader, deleter, fileSearcher, state }); + } + } finally { + state.draining = false; + } +} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.test.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.test.ts new file mode 100644 index 0000000000..df5a461eee --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { TemporalFile } from '../../../domain/TemporalFile'; +import { createTemporalFileUploadQueueState } from './state'; +import { enqueueUpload } from './enqueue-upload'; + +vi.mock('../../../../../../core/electron/paths', () => ({ + PATHS: { + UPLOAD_QUEUE: '/tmp/internxt-drive-tmp', + }, +})); + +vi.mock('./drain-upload-queue', () => ({ + drainUploadQueue: vi.fn(), +})); + +import { drainUploadQueue } from './drain-upload-queue'; + +describe('enqueue-upload', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should deduplicate concurrent enqueue calls for the same path', async () => { + const state = createTemporalFileUploadQueueState(); + let resolveStage: ((value: { path: string }) => void) | undefined; + const repository = { + stage: vi.fn().mockImplementation( + () => + new Promise<{ path: string }>((resolve) => { + resolveStage = resolve; + }), + ), + }; + + const props = { + repository, + uploader: {} as never, + deleter: {} as never, + fileSearcher: {} as never, + state, + temporalFile: { path: '/source/file.txt' } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }; + + const firstEnqueue = enqueueUpload(props); + const secondEnqueue = enqueueUpload(props); + + resolveStage?.({ path: '/staged/file.txt' }); + + await Promise.all([firstEnqueue, secondEnqueue]); + + expect(repository.stage).toHaveBeenCalledTimes(1); + expect(state.tasks).toHaveLength(1); + expect(state.tasks[0]).toStrictEqual({ + temporalFile: { path: '/staged/file.txt' }, + path: '/target/file.txt', + processName: 'process', + }); + }); + + it('should stage and queue a new upload', async () => { + const state = createTemporalFileUploadQueueState(); + const repository = { + stage: vi.fn().mockResolvedValue({ path: '/staged/file.txt' }), + }; + const context = { + repository, + uploader: {} as never, + deleter: {} as never, + fileSearcher: {} as never, + state, + }; + const temporalFile = { path: '/source/file.txt' } as unknown as TemporalFile; + + await enqueueUpload({ + ...context, + temporalFile, + path: '/target/file.txt', + processName: 'process', + }); + + expect(repository.stage).toHaveBeenCalledWith(temporalFile.path, '/tmp/internxt-drive-tmp'); + expect(state.queuedPaths.has('/target/file.txt')).toBe(true); + expect(state.tasks).toStrictEqual([ + { + temporalFile: { path: '/staged/file.txt' }, + path: '/target/file.txt', + processName: 'process', + }, + ]); + expect(drainUploadQueue).toHaveBeenCalledTimes(1); + }); + + it('should skip duplicated paths', async () => { + const state = createTemporalFileUploadQueueState(); + state.queuedPaths.add('/target/file.txt'); + + const repository = { + stage: vi.fn(), + }; + + await enqueueUpload({ + repository, + uploader: {} as never, + deleter: {} as never, + fileSearcher: {} as never, + state, + temporalFile: { path: '/source/file.txt' } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }); + + expect(repository.stage).not.toHaveBeenCalled(); + expect(drainUploadQueue).not.toHaveBeenCalled(); + }); +}); diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.ts new file mode 100644 index 0000000000..b7b18ef612 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/enqueue-upload.ts @@ -0,0 +1,41 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { PATHS } from '../../../../../../core/electron/paths'; +import type { EnqueueProps, QueueContext } from './types'; +import { drainUploadQueue } from './drain-upload-queue'; + +export async function enqueueUpload({ + temporalFile, + path, + processName, + repository, + uploader, + deleter, + fileSearcher, + state, +}: QueueContext & EnqueueProps) { + if (state.queuedPaths.has(path)) { + logger.debug({ msg: '[UploadQueue] Path already queued', path, processName }); + return; + } + + // Reserve path immediately to avoid duplicate staging when concurrent enqueues race. + state.queuedPaths.add(path); + + let stagedTemporalFile; + + try { + stagedTemporalFile = await repository.stage(temporalFile.path, PATHS.UPLOAD_QUEUE); + } catch (error) { + state.queuedPaths.delete(path); + logger.error({ msg: '[UploadQueue] Failed to stage temporal file before enqueue', error, path, processName }); + return Promise.reject(error); + } + + state.tasks.push({ + temporalFile: stagedTemporalFile, + path, + processName, + }); + + void drainUploadQueue({ repository, uploader, deleter, fileSearcher, state }); +} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.test.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.test.ts new file mode 100644 index 0000000000..00ebfd56b7 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { TemporalFile } from '../../../domain/TemporalFile'; +import { preserveRejectedUpload } from './preserve-rejected-upload'; + +vi.mock('../../../../../../backend/features/user/file-size-limit', () => ({ + preserveRejectedFileSizeTooBig: vi.fn(), +})); + +import { preserveRejectedFileSizeTooBig } from '../../../../../../backend/features/user/file-size-limit'; + +describe('preserve-rejected-upload', () => { + it('should skip preservation when the content path is missing', async () => { + const deleter = { run: vi.fn() }; + const task = { + temporalFile: { + contentFilePath: undefined, + size: { value: 123 }, + } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }; + + await preserveRejectedUpload({ task, deleter } as never); + + expect(preserveRejectedFileSizeTooBig).not.toHaveBeenCalled(); + expect(deleter.run).not.toHaveBeenCalled(); + }); + + it('should preserve and delete the rejected file when the copy succeeds', async () => { + const deleter = { run: vi.fn().mockResolvedValue(undefined) }; + const task = { + temporalFile: { + contentFilePath: '/tmp/content.bin', + size: { value: 123 }, + } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }; + + vi.mocked(preserveRejectedFileSizeTooBig).mockResolvedValue({ error: undefined } as never); + + await preserveRejectedUpload({ task, deleter } as never); + + expect(preserveRejectedFileSizeTooBig).toHaveBeenCalledWith({ + originalPath: '/target/file.txt', + temporalContentPath: '/tmp/content.bin', + size: 123, + }); + expect(deleter.run).toHaveBeenCalledWith('/target/file.txt'); + }); +}); diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.ts new file mode 100644 index 0000000000..53009a0db1 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/preserve-rejected-upload.ts @@ -0,0 +1,26 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { preserveRejectedFileSizeTooBig } from '../../../../../../backend/features/user/file-size-limit'; +import type { TemporalFileDeleter } from '../deletion/TemporalFileDeleter'; +import type { UploadTask } from './types'; + +export async function preserveRejectedUpload({ task, deleter }: { task: UploadTask; deleter: TemporalFileDeleter }) { + if (!task.temporalFile.contentFilePath) { + logger.warn({ msg: '[UploadQueue] Rejected file missing content path', path: task.path }); + return; + } + + const { error } = await preserveRejectedFileSizeTooBig({ + originalPath: task.path, + temporalContentPath: task.temporalFile.contentFilePath, + size: task.temporalFile.size.value, + }); + + if (error) { + logger.error({ msg: '[UploadQueue] Failed to preserve oversized file', error, path: task.path }); + return; + } + + await deleter.run(task.path); + + logger.warn({ msg: '[UploadQueue] Preserved oversized file after queue rejection', path: task.path }); +} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/state.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/state.ts new file mode 100644 index 0000000000..c4200094b6 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/state.ts @@ -0,0 +1,11 @@ +import type { QueueState } from './types'; + +export function createTemporalFileUploadQueueState() { + const state: QueueState = { + queuedPaths: new Set(), + tasks: [], + draining: false, + }; + + return state; +} diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types.ts new file mode 100644 index 0000000000..d76b9d65ef --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types.ts @@ -0,0 +1,40 @@ +import type { TemporalFileDeleter } from '../deletion/TemporalFileDeleter'; +import type { TemporalFile } from '../../../domain/TemporalFile'; +import type { TemporalFileRepository } from '../../../domain/TemporalFileRepository'; +import type { TemporalFileUploader } from '../TemporalFileUploader'; +import type { FirstsFileSearcher } from '../../../../../virtual-drive/files/application/search/FirstsFileSearcher'; + +export const TemporalFileUploadQueue = Symbol('TemporalFileUploadQueue'); + +export type UploadTask = { + temporalFile: TemporalFile; + path: string; + processName: string; +}; + +export type FactoryProps = { + repository: TemporalFileRepository; + uploader: TemporalFileUploader; + deleter: TemporalFileDeleter; + fileSearcher: FirstsFileSearcher; +}; + +export type QueueState = { + queuedPaths: Set; + tasks: Array; + draining: boolean; +}; + +export type QueueContext = FactoryProps & { + state: QueueState; +}; + +export type EnqueueProps = { + temporalFile: TemporalFile; + path: string; + processName: string; +}; + +export type TemporalFileUploadQueue = { + enqueue: (props: EnqueueProps) => Promise; +}; diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.test.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.test.ts new file mode 100644 index 0000000000..7d07f87a26 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { TemporalFile } from '../../../domain/TemporalFile'; +import { uploadQueuedTask } from './upload-queued-task'; + +describe('upload-queued-task', () => { + it('should pass replace info when the file already exists', async () => { + const uploader = { + run: vi.fn().mockResolvedValue('contents-id'), + }; + const fileSearcher = { + run: vi.fn().mockResolvedValue({ contentsId: 'contents-1', name: 'file', type: 'txt' }), + }; + const task = { + temporalFile: { path: '/tmp/file.txt' } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }; + + await uploadQueuedTask({ task, uploader, fileSearcher } as never); + + expect(fileSearcher.run).toHaveBeenCalledWith({ path: '/target/file.txt', status: 'EXISTS' }); + expect(uploader.run).toHaveBeenCalledWith(task.temporalFile, { + contentsId: 'contents-1', + name: 'file', + extension: 'txt', + }); + }); + + it('should upload without replacements when the file does not exist', async () => { + const uploader = { + run: vi.fn().mockResolvedValue('contents-id'), + }; + const fileSearcher = { + run: vi.fn().mockResolvedValue(undefined), + }; + const task = { + temporalFile: { path: '/tmp/file.txt' } as unknown as TemporalFile, + path: '/target/file.txt', + processName: 'process', + }; + + await uploadQueuedTask({ task, uploader, fileSearcher } as never); + + expect(uploader.run).toHaveBeenCalledWith(task.temporalFile, undefined); + }); +}); diff --git a/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.ts b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.ts new file mode 100644 index 0000000000..7d1b0bda62 --- /dev/null +++ b/src/context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/upload-queued-task.ts @@ -0,0 +1,22 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { FileStatuses } from '../../../../../virtual-drive/files/domain/FileStatus'; +import type { FirstsFileSearcher } from '../../../../../virtual-drive/files/application/search/FirstsFileSearcher'; +import type { TemporalFileUploader } from '../TemporalFileUploader'; +import type { UploadTask } from './types'; + +type QueueTaskDependencies = { + task: UploadTask; + uploader: TemporalFileUploader; + fileSearcher: FirstsFileSearcher; +}; + +export async function uploadQueuedTask({ task, uploader, fileSearcher }: QueueTaskDependencies) { + const existingFile = await fileSearcher.run({ path: task.path, status: FileStatuses.EXISTS }); + const replaces = existingFile + ? { contentsId: existingFile.contentsId, name: existingFile.name, extension: existingFile.type } + : undefined; + + await uploader.run(task.temporalFile, replaces); + + logger.debug({ msg: '[UploadQueue] Upload completed', path: task.path, processName: task.processName }); +} diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.test.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.test.ts index 709a8decca..f15a635029 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.test.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.test.ts @@ -457,4 +457,31 @@ describe('InMemoryFileRepository', () => { expect(await sut.searchByUuid('550e8400-e29b-41d4-a716-446655440018')).toBeDefined(); }); }); + + describe('deleteMatchingPartial', () => { + it('should delete files when contentsId matches after normalization', async () => { + const normalizedContentsId = 'aaaaaaaaaaaaaaaaaaaaaaa\u00E9'; + const decomposedContentsId = 'aaaaaaaaaaaaaaaaaaaaaaae\u0301'; + const uuid = '550e8400-e29b-41d4-a716-446655440099'; + + await sut.upsert( + File.from({ + id: 1, + uuid, + contentsId: normalizedContentsId, + folderId: 100, + createdAt: '10-03-2025', + modificationTime: '10-03-2025', + path: '/documents/file.txt', + size: 1000, + updatedAt: '10-03-2025', + status: FileStatuses.EXISTS, + }), + ); + + await sut.deleteMatchingPartial({ contentsId: decomposedContentsId }); + + expect(await sut.searchByUuid(uuid)).toBeUndefined(); + }); + }); }); diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts index b0cdb5c05e..25ff675870 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts @@ -18,6 +18,24 @@ export class InMemoryFileRepository implements FileRepository { this.filesByContentsId = new Map(); } + private matchesPartial({ + attributes, + partial, + keys, + }: { + attributes: FileAttributes; + partial: Partial; + keys: Array>; + }) { + return keys.every((key: keyof FileAttributes) => { + if (key === 'contentsId') { + return attributes[key].normalize() == (partial[key] as string).normalize(); + } + + return attributes[key] == partial[key]; + }); + } + async searchByUuid(uuid: File['uuid']): Promise { const attributes = this.filesByUuid.get(uuid); @@ -62,13 +80,7 @@ export class InMemoryFileRepository implements FileRepository { const keys = Object.keys(partial) as Array>; const filesAttributes = this.values.filter((attributes) => { - return keys.every((key: keyof FileAttributes) => { - if (key === 'contentsId') { - return attributes[key].normalize() == (partial[key] as string).normalize(); - } - - return attributes[key] == partial[key]; - }); + return this.matchesPartial({ attributes, partial, keys }); }); if (!filesAttributes) { @@ -108,7 +120,7 @@ export class InMemoryFileRepository implements FileRepository { const keys = Object.keys(partial) as Array>; for (const [uuid, attributes] of this.filesByUuid.entries()) { - const matches = keys.every((key: keyof FileAttributes) => attributes[key] === partial[key]); + const matches = this.matchesPartial({ attributes, partial, keys }); if (matches) { this.filesByUuid.delete(uuid); From b222c702e07f9038b8274c3adb50586630e3d609 Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Tue, 7 Jul 2026 21:58:24 -0500 Subject: [PATCH 6/6] refactor: format export statements for consistency --- .../user/file-size-limit/add-max-file-size-rejection.ts | 6 +----- .../create-lazy-virtual-drive-hydrator-service.ts | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts b/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts index f0e5a3977d..16d06c6529 100644 --- a/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts +++ b/src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts @@ -6,11 +6,7 @@ import { isUploadSizeLimitBlockedPath, markUploadSizeLimitBlockedPath, } from './upload-size-limit-blocked-paths'; -export { - clearUploadSizeLimitBlockedPath, - isUploadSizeLimitBlockedPath, - markUploadSizeLimitBlockedPath, -}; +export { clearUploadSizeLimitBlockedPath, isUploadSizeLimitBlockedPath, markUploadSizeLimitBlockedPath }; type MaxFileSizeRejectionModalState = { rejectedFilesCount: number; diff --git a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts index 367e278353..75f87861da 100644 --- a/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts +++ b/src/backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service.ts @@ -11,9 +11,9 @@ import { refreshChildrenIfNeeded } from './refresh-children-if-needed'; import { fetchAndStoreChildren } from './fetch-and-store-children'; import { readLocalDirectory } from './children'; -export const LazyVirtualDriveHydrator = Symbol( - 'LazyVirtualDriveHydrator', -) as unknown as Identifier>; +export const LazyVirtualDriveHydrator = Symbol('LazyVirtualDriveHydrator') as unknown as Identifier< + ReturnType +>; export type LazyVirtualDriveHydrator = ReturnType;