From a1e2d0ac5c6821a6b692ac4da35f9c78eee643d8 Mon Sep 17 00:00:00 2001 From: cyberinferno Date: Sun, 14 Jun 2026 19:43:49 +0530 Subject: [PATCH 1/2] feat: enhance agent UI and backend integration --- cmd/omnihance-a3-agent/docs/openapi.yml | 142 +++++ .../src/components/backup-page.tsx | 29 +- .../src/components/file-tree.tsx | 165 +++++- .../omnihance-a3-agent-ui/src/constants.ts | 1 + .../omnihance-a3-agent-ui/src/lib/api.ts | 65 +++ internal/config/config.go | 3 + internal/db/backup_jobs.go | 10 +- internal/db/directory_downloads.go | 190 +++++++ internal/db/file_downloads.go | 5 +- internal/db/internal_db.go | 141 +++++ internal/db/mock_InternalDB.go | 268 +++++++++ internal/server/file_download_routes.go | 127 +++++ internal/server/file_download_routes_test.go | 78 +++ internal/server/file_system_routes.go | 2 + internal/services/backup_service.go | 532 +++++++++++++++++- internal/services/backup_service_test.go | 135 +++++ internal/services/mock_BackupService.go | 148 +++++ 17 files changed, 2009 insertions(+), 32 deletions(-) create mode 100644 internal/db/directory_downloads.go create mode 100644 internal/services/backup_service_test.go diff --git a/cmd/omnihance-a3-agent/docs/openapi.yml b/cmd/omnihance-a3-agent/docs/openapi.yml index 8c0944c..141aefb 100644 --- a/cmd/omnihance-a3-agent/docs/openapi.yml +++ b/cmd/omnihance-a3-agent/docs/openapi.yml @@ -338,6 +338,104 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/directory-download-link: + post: + tags: + - file-system + summary: Create directory download link + description: Starts or resumes a background ZIP compression job for a directory. When an unchanged cached archive is available, returns a secure temp download link immediately. + security: + - ApiKeyAuth: [] + parameters: + - in: query + name: path + required: true + schema: + type: string + description: Directory path to compress and download. + responses: + '200': + description: Directory archive is ready and a secure download link was created or reused + content: + application/json: + schema: + $ref: '#/components/schemas/DirectoryDownloadResponse' + '202': + description: Directory compression started or the same directory is already in progress + content: + application/json: + schema: + $ref: '#/components/schemas/DirectoryDownloadResponse' + '400': + description: Bad Request - Missing path or path is not a directory + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Download files permission is required + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '409': + description: A different directory download job is already running + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/directory-downloads/{run_id}: + get: + tags: + - file-system + summary: Get directory download status + description: Polls a directory download backup run. When the archive is ready, returns a secure temp download link for the authenticated user. + security: + - ApiKeyAuth: [] + parameters: + - name: run_id + in: path + required: true + schema: + type: integer + format: int64 + responses: + '200': + description: Directory download is ready, failed, or cancelled + content: + application/json: + schema: + $ref: '#/components/schemas/DirectoryDownloadResponse' + '202': + description: Directory compression is still in progress + content: + application/json: + schema: + $ref: '#/components/schemas/DirectoryDownloadResponse' + '403': + description: Forbidden - Download files permission is required + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Directory download run or archive not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' /api/file-tree/download/{token}: get: tags: @@ -4521,6 +4619,46 @@ components: type: integer format: int64 description: Number of successful download requests already recorded for this link + DirectoryDownloadResponse: + type: object + properties: + status: + type: string + enum: [ready, started, in_progress, failed, cancelled] + description: Current directory download state + message: + type: string + description: User-facing status or failure message + job_id: + type: integer + format: int64 + description: Backup job used for the directory archive + run_id: + type: integer + format: int64 + description: Backup run used for the directory archive + file_id: + type: integer + format: int64 + description: Backup run file ID for the generated archive, present when ready + download_url: + type: string + description: Secure temp download URL, present when ready + example: '/api/file-tree/download/01HZX.temp-token' + expires_at: + type: string + format: date-time + description: Link expiration timestamp, present when ready + reused: + type: boolean + description: Whether an existing unexpired secure download link was reused + download_count: + type: integer + format: int64 + description: Number of successful download requests already recorded for the secure link + archive_reused: + type: boolean + description: Whether the directory ZIP archive was reused from cache NPCFileAPIData: type: object description: Parsed binary data from an NPC file (API request/response format). All fields are required when used as a request body. @@ -6314,6 +6452,10 @@ components: job_type: type: string enum: [file, sql_server] + tag: + type: string + nullable: true + example: directory_download name: type: string status: diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/backup-page.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/backup-page.tsx index 445e85d..a9fae8e 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/backup-page.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/backup-page.tsx @@ -130,6 +130,7 @@ const emptyForm: BackupFormState = { const backupRunPageSize = 10; const emptyBackupRuns: BackupRun[] = []; +const directoryDownloadBackupTag = 'directory_download'; export function BackupPage() { const queryClient = useQueryClient(); @@ -615,8 +616,8 @@ export function BackupPage() { - - {run.trigger_type} + + {formatBackupTriggerLabel(run.trigger_type)} + )} {item.kind === 'file' && item.is_viewable && ( ['directory-download', runId] as const, textFile: (path: string) => ['text-file', path] as const, npcFile: (path: string) => ['npc-file', path] as const, spawnFile: (path: string) => ['spawn-file', path] as const, diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts index bd89598..570e2d9 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts @@ -22,6 +22,9 @@ export const API_ROUTES = { DUPLICATE_FILE: '/api/file-tree/duplicate-file', REVISION_COUNT: '/api/file-tree/revision-summary', FILE_DOWNLOAD_LINK: '/api/file-tree/download-link', + DIRECTORY_DOWNLOAD_LINK: '/api/file-tree/directory-download-link', + DIRECTORY_DOWNLOAD_STATUS: (runId: number) => + `/api/file-tree/directory-downloads/${runId}`, METRICS_SUMMARY: '/api/metrics/summary', METRICS_CHARTS: '/api/metrics/charts', GAME_CLIENT_DATA_MONSTERS: '/api/game-client-data/monsters', @@ -380,6 +383,40 @@ const DownloadLinkResponseSchema = z.object({ export type DownloadLinkResponse = z.infer; +const DirectoryDownloadReadyResponseSchema = DownloadLinkResponseSchema.extend({ + status: z.literal('ready'), + job_id: z.number().int(), + run_id: z.number().int(), + file_id: z.number().int(), + archive_reused: z.boolean(), +}); + +const DirectoryDownloadPendingResponseSchema = z.object({ + status: z.enum(['started', 'in_progress']), + message: z.string(), + job_id: z.number().int(), + run_id: z.number().int(), + archive_reused: z.boolean().optional().default(false), +}); + +const DirectoryDownloadTerminalResponseSchema = z.object({ + status: z.enum(['failed', 'cancelled']), + message: z.string(), + job_id: z.number().int(), + run_id: z.number().int(), + archive_reused: z.boolean().optional().default(false), +}); + +const DirectoryDownloadResponseSchema = z.union([ + DirectoryDownloadReadyResponseSchema, + DirectoryDownloadPendingResponseSchema, + DirectoryDownloadTerminalResponseSchema, +]); + +export type DirectoryDownloadResponse = z.infer< + typeof DirectoryDownloadResponseSchema +>; + const TextFileAPIDataSchema = z.object({ content: z.string(), }); @@ -1138,6 +1175,33 @@ export async function createFileDownloadLink(params: { ); } +export async function createDirectoryDownloadLink(params: { + path: string; +}): Promise { + const response = await axiosInstance.post( + API_ROUTES.DIRECTORY_DOWNLOAD_LINK, + undefined, + { params }, + ); + return validateResponse( + DirectoryDownloadResponseSchema, + response.data, + API_ROUTES.DIRECTORY_DOWNLOAD_LINK, + ); +} + +export async function getDirectoryDownloadStatus( + runId: number, +): Promise { + const route = API_ROUTES.DIRECTORY_DOWNLOAD_STATUS(runId); + const response = await axiosInstance.get(route); + return validateResponse( + DirectoryDownloadResponseSchema, + response.data, + route, + ); +} + export async function getMetricsSummary(): Promise { const response = await axiosInstance.get(API_ROUTES.METRICS_SUMMARY); return validateResponse( @@ -2076,6 +2140,7 @@ export type BackupJobType = z.infer; const BackupJobSchema = z.object({ id: z.number().int(), job_type: BackupJobTypeSchema, + tag: z.string().nullable(), name: z.string(), status: z.string(), cron_expression: z.string().nullable(), diff --git a/internal/config/config.go b/internal/config/config.go index 2e54b30..f0b7edb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,6 +23,7 @@ type EnvVars struct { MetricsCleanupIntervalSeconds int RevisionsDirectory string BackupsDirectory string + DirectoryDownloadsDirectory string MetricsEnabled bool SessionTimeoutSeconds int CookieSecret string @@ -42,6 +43,7 @@ var defaultEnvVars = map[string]string{ "METRICS_CLEANUP_INTERVAL_SECONDS": "3600", "REVISIONS_DIRECTORY": ".revisions", "BACKUPS_DIRECTORY": ".backups", + "DIRECTORY_DOWNLOADS_DIRECTORY": ".directory-download", "METRICS_ENABLED": "true", "SESSION_TIMEOUT_SECONDS": fmt.Sprintf("%d", 60*60*24*30), "COOKIE_SECRET": externalUtils.GenerateRandomString(32), @@ -145,6 +147,7 @@ func New() *EnvVars { MetricsCleanupIntervalSeconds: metricsCleanupIntervalSeconds, RevisionsDirectory: os.Getenv("REVISIONS_DIRECTORY"), BackupsDirectory: os.Getenv("BACKUPS_DIRECTORY"), + DirectoryDownloadsDirectory: os.Getenv("DIRECTORY_DOWNLOADS_DIRECTORY"), MetricsEnabled: metricsEnabled, SessionTimeoutSeconds: sessionTimeoutSeconds, CookieSecret: cookieSecret, diff --git a/internal/db/backup_jobs.go b/internal/db/backup_jobs.go index d4b2551..028b94e 100644 --- a/internal/db/backup_jobs.go +++ b/internal/db/backup_jobs.go @@ -15,6 +15,8 @@ const ( BackupJobTypeFile = "file" BackupJobTypeSQLServer = "sql_server" + BackupJobTagDirectoryDownload = "directory_download" + BackupJobStatusActive = "active" BackupJobStatusInactive = "inactive" BackupJobStatusRunning = "running" @@ -26,13 +28,15 @@ const ( BackupRunStatusCancelled = "cancelled" BackupRunStatusSkipped = "skipped" - BackupRunTriggerManual = "manual" - BackupRunTriggerCron = "cron" + BackupRunTriggerManual = "manual" + BackupRunTriggerCron = "cron" + BackupRunTriggerDirectoryDownload = "directory_download" ) type BackupJob struct { ID int64 `db:"id" json:"id"` JobType string `db:"job_type" json:"job_type"` + Tag *string `db:"tag" json:"tag"` Name string `db:"name" json:"name"` Status string `db:"status" json:"status"` CronExpression *string `db:"cron_expression" json:"cron_expression"` @@ -80,6 +84,7 @@ type BackupRunFile struct { type BackupJobPayload struct { JobType string + Tag *string Name string Status string CronExpression *string @@ -599,6 +604,7 @@ func (s *sqliteInternalDB) MarkOrphanedBackupRunsFailed() error { func backupJobRecord(payload BackupJobPayload) goqu.Record { return goqu.Record{ "job_type": payload.JobType, + "tag": payload.Tag, "name": payload.Name, "status": payload.Status, "cron_expression": payload.CronExpression, diff --git a/internal/db/directory_downloads.go b/internal/db/directory_downloads.go new file mode 100644 index 0000000..74584b0 --- /dev/null +++ b/internal/db/directory_downloads.go @@ -0,0 +1,190 @@ +package db + +import ( + "database/sql" + "fmt" + "time" + + "github.com/doug-martin/goqu/v9" + "github.com/omnihance/omnihance-a3-agent/internal/constants" + "github.com/omnihance/omnihance-a3-agent/internal/logger" +) + +type DirectoryDownloadArchive struct { + ID int64 `db:"id" json:"id"` + NormalizedPath string `db:"normalized_path" json:"normalized_path"` + SourceFingerprint string `db:"source_fingerprint" json:"source_fingerprint"` + JobID int64 `db:"job_id" json:"job_id"` + RunID int64 `db:"run_id" json:"run_id"` + FileID int64 `db:"file_id" json:"file_id"` + ArchivePath string `db:"archive_path" json:"archive_path"` + ArchiveSize int64 `db:"archive_size" json:"archive_size"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +type DirectoryDownloadArchivePayload struct { + NormalizedPath string + SourceFingerprint string + JobID int64 + RunID int64 + FileID int64 + ArchivePath string + ArchiveSize int64 +} + +func (s *sqliteInternalDB) GetBackupJobByTagAndSourcePath(tag string, sourcePath string) (*BackupJob, error) { + var job BackupJob + found, err := s.goqu.From("backup_jobs"). + Prepared(true). + Where( + goqu.Ex{ + "tag": tag, + "source_path": sourcePath, + }, + goqu.C("status").Neq(BackupJobStatusDeleted), + ). + Order(goqu.C("created_at").Desc(), goqu.C("id").Desc()). + Limit(1). + ScanStruct(&job) + if err != nil { + s.logger.Error( + "failed to get backup job by tag and source path", + logger.Field{Key: "tag", Value: tag}, + logger.Field{Key: "source_path", Value: sourcePath}, + logger.Field{Key: "error", Value: err}, + ) + return nil, fmt.Errorf("failed to get backup job by tag and source path: %w", err) + } + + if !found { + return nil, fmt.Errorf("%w: backup job with tag %s and source path %s", ErrBackupNotFound, tag, sourcePath) + } + + return &job, nil +} + +func (s *sqliteInternalDB) GetRunningBackupRunForTag(tag string) (*BackupRun, *BackupJob, error) { + var run BackupRun + err := s.db.QueryRow(` + SELECT + r.id, + r.job_id, + r.trigger_type, + r.status, + r.previous_job_status, + r.started_at, + r.finished_at, + r.cancel_requested_at, + r.output, + r.error_details, + r.created_by, + r.created_at, + r.updated_at + FROM backup_runs r + INNER JOIN backup_jobs j ON j.id = r.job_id + WHERE j.tag = ? AND j.status <> ? AND r.status = ? + ORDER BY r.started_at DESC, r.id DESC + LIMIT 1 + `, tag, BackupJobStatusDeleted, BackupRunStatusRunning).Scan( + &run.ID, + &run.JobID, + &run.TriggerType, + &run.Status, + &run.PreviousJobStatus, + &run.StartedAt, + &run.FinishedAt, + &run.CancelRequestedAt, + &run.Output, + &run.ErrorDetails, + &run.CreatedBy, + &run.CreatedAt, + &run.UpdatedAt, + ) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + + s.logger.Error( + "failed to get running backup run by tag", + logger.Field{Key: "tag", Value: tag}, + logger.Field{Key: "error", Value: err}, + ) + return nil, nil, fmt.Errorf("failed to get running backup run by tag: %w", err) + } + + job, err := s.GetBackupJob(run.JobID) + if err != nil { + return nil, nil, err + } + + return &run, job, nil +} + +func (s *sqliteInternalDB) GetDirectoryDownloadArchive(normalizedPath string, sourceFingerprint string) (*DirectoryDownloadArchive, error) { + var archive DirectoryDownloadArchive + found, err := s.goqu.From("directory_download_archives"). + Prepared(true). + Where(goqu.Ex{ + "normalized_path": normalizedPath, + "source_fingerprint": sourceFingerprint, + }). + Order(goqu.C("updated_at").Desc(), goqu.C("id").Desc()). + Limit(1). + ScanStruct(&archive) + if err != nil { + s.logger.Error( + "failed to get directory download archive", + logger.Field{Key: "normalized_path", Value: normalizedPath}, + logger.Field{Key: "error", Value: err}, + ) + return nil, fmt.Errorf("failed to get directory download archive: %w", err) + } + + if !found { + return nil, constants.ErrNotFound + } + + return &archive, nil +} + +func (s *sqliteInternalDB) UpsertDirectoryDownloadArchive(payload DirectoryDownloadArchivePayload) (*DirectoryDownloadArchive, error) { + _, err := s.db.Exec(` + INSERT INTO directory_download_archives ( + normalized_path, + source_fingerprint, + job_id, + run_id, + file_id, + archive_path, + archive_size + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(normalized_path, source_fingerprint) DO UPDATE SET + job_id = excluded.job_id, + run_id = excluded.run_id, + file_id = excluded.file_id, + archive_path = excluded.archive_path, + archive_size = excluded.archive_size, + updated_at = CURRENT_TIMESTAMP + `, + payload.NormalizedPath, + payload.SourceFingerprint, + payload.JobID, + payload.RunID, + payload.FileID, + payload.ArchivePath, + payload.ArchiveSize, + ) + if err != nil { + s.logger.Error( + "failed to upsert directory download archive", + logger.Field{Key: "normalized_path", Value: payload.NormalizedPath}, + logger.Field{Key: "run_id", Value: payload.RunID}, + logger.Field{Key: "error", Value: err}, + ) + return nil, fmt.Errorf("failed to upsert directory download archive: %w", err) + } + + return s.GetDirectoryDownloadArchive(payload.NormalizedPath, payload.SourceFingerprint) +} diff --git a/internal/db/file_downloads.go b/internal/db/file_downloads.go index fbab58f..4394264 100644 --- a/internal/db/file_downloads.go +++ b/internal/db/file_downloads.go @@ -11,8 +11,9 @@ import ( ) const ( - FileDownloadSourceFileBrowser = "file_browser" - FileDownloadSourceBackup = "backup" + FileDownloadSourceFileBrowser = "file_browser" + FileDownloadSourceBackup = "backup" + FileDownloadSourceDirectoryDownload = "directory_download" ) type FileDownloadLink struct { diff --git a/internal/db/internal_db.go b/internal/db/internal_db.go index c48d84a..747689e 100644 --- a/internal/db/internal_db.go +++ b/internal/db/internal_db.go @@ -104,6 +104,8 @@ type InternalDB interface { GetBackupJobs() ([]BackupJob, error) GetSchedulableBackupJobs() ([]BackupJob, error) GetBackupJob(id int64) (*BackupJob, error) + GetBackupJobByTagAndSourcePath(tag string, sourcePath string) (*BackupJob, error) + GetRunningBackupRunForTag(tag string) (*BackupRun, *BackupJob, error) CreateBackupJob(payload BackupJobPayload, userID *int64) (*BackupJob, error) UpdateBackupJob(id int64, payload BackupJobPayload, userID *int64) (*BackupJob, error) UpdateBackupJobStatus(id int64, status string, userID *int64) error @@ -119,6 +121,8 @@ type InternalDB interface { GetBackupRunFiles(runID int64) ([]BackupRunFile, error) GetBackupRunFile(id int64) (*BackupRunFile, error) MarkOrphanedBackupRunsFailed() error + GetDirectoryDownloadArchive(normalizedPath string, sourceFingerprint string) (*DirectoryDownloadArchive, error) + UpsertDirectoryDownloadArchive(payload DirectoryDownloadArchivePayload) (*DirectoryDownloadArchive, error) CreateServerViewSyncRun(userID *int64) (*ServerViewSyncRun, error) FinishServerViewSyncRun(runID int64, status string, warningCount int, errorDetails *string) error GetLatestServerViewSyncRun() (*ServerViewSyncRun, error) @@ -290,10 +294,18 @@ func (s *sqliteInternalDB) MigrateUp() error { return err } + if err := s.migrate015DirectoryDownloadsTable(); err != nil { + return err + } + return nil } func (s *sqliteInternalDB) MigrateDown() error { + if err := s.rollback015DirectoryDownloadsTable(); err != nil { + return err + } + if err := s.rollback014FileDownloadLinksTable(); err != nil { return err } @@ -2134,3 +2146,132 @@ func (s *sqliteInternalDB) rollback014FileDownloadLinksTable() error { return nil } + +func (s *sqliteInternalDB) migrate015DirectoryDownloadsTable() error { + const migName = "015_directory_downloads" + + applied, err := s.isMigrationApplied(migName) + if err != nil { + s.logger.Error( + "failed to check migration status", + logger.Field{Key: "migration", Value: migName}, + logger.Field{Key: "error", Value: err}, + ) + return fmt.Errorf("failed to check migration status for %s: %w", migName, err) + } + + if applied { + return nil + } + + s.logger.Info("Applying migration", logger.Field{Key: "migration", Value: migName}) + + migrationSQL := ` + ALTER TABLE backup_jobs ADD COLUMN tag TEXT; + + CREATE INDEX IF NOT EXISTS idx_backup_jobs_tag ON backup_jobs (tag); + + CREATE INDEX IF NOT EXISTS idx_backup_jobs_tag_source_path ON backup_jobs (tag, source_path); + + CREATE TABLE IF NOT EXISTS directory_download_archives ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + normalized_path TEXT NOT NULL, + source_fingerprint TEXT NOT NULL, + job_id INTEGER NOT NULL REFERENCES backup_jobs(id) ON DELETE CASCADE, + run_id INTEGER NOT NULL REFERENCES backup_runs(id) ON DELETE CASCADE, + file_id INTEGER NOT NULL REFERENCES backup_run_files(id) ON DELETE CASCADE, + archive_path TEXT NOT NULL, + archive_size INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(normalized_path, source_fingerprint) + ); + + CREATE INDEX IF NOT EXISTS idx_directory_download_archives_path ON directory_download_archives (normalized_path); + + CREATE INDEX IF NOT EXISTS idx_directory_download_archives_run_id ON directory_download_archives (run_id); + ` + + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("failed to begin directory downloads migration: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + _, err = tx.Exec(migrationSQL) + if err != nil { + return fmt.Errorf("failed to create directory downloads tables: %w", err) + } + + if _, err := tx.Exec("INSERT INTO migrations (name) VALUES (?)", migName); err != nil { + s.logger.Error( + "failed to mark migration as applied", + logger.Field{Key: "migration", Value: migName}, + logger.Field{Key: "error", Value: err}, + ) + return fmt.Errorf("failed to mark migration as applied: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit directory downloads migration: %w", err) + } + + return nil +} + +func (s *sqliteInternalDB) rollback015DirectoryDownloadsTable() error { + const migName = "015_directory_downloads" + + applied, err := s.isMigrationApplied(migName) + if err != nil { + s.logger.Error( + "failed to check migration status", + logger.Field{Key: "migration", Value: migName}, + logger.Field{Key: "error", Value: err}, + ) + return fmt.Errorf("failed to check migration status for %s: %w", migName, err) + } + + if !applied { + return nil + } + + s.logger.Info("Rolling back migration", logger.Field{Key: "migration", Value: migName}) + + rollbackSQL := ` + DROP TABLE IF EXISTS directory_download_archives; + DROP INDEX IF EXISTS idx_backup_jobs_tag_source_path; + DROP INDEX IF EXISTS idx_backup_jobs_tag; + ALTER TABLE backup_jobs DROP COLUMN tag; + ` + + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("failed to begin directory downloads rollback: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + _, err = tx.Exec(rollbackSQL) + if err != nil { + return fmt.Errorf("failed to rollback directory downloads tables: %w", err) + } + + if _, err := tx.Exec("DELETE FROM migrations WHERE name = ?", migName); err != nil { + s.logger.Error( + "failed to mark migration as rolled back", + logger.Field{Key: "migration", Value: migName}, + logger.Field{Key: "error", Value: err}, + ) + return fmt.Errorf("failed to mark migration as rolled back: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit directory downloads rollback: %w", err) + } + + return nil +} diff --git a/internal/db/mock_InternalDB.go b/internal/db/mock_InternalDB.go index b8dfbf0..b8c088b 100644 --- a/internal/db/mock_InternalDB.go +++ b/internal/db/mock_InternalDB.go @@ -2451,6 +2451,74 @@ func (_c *MockInternalDB_GetBackupJob_Call) RunAndReturn(run func(id int64) (*Ba return _c } +// GetBackupJobByTagAndSourcePath provides a mock function for the type MockInternalDB +func (_mock *MockInternalDB) GetBackupJobByTagAndSourcePath(tag string, sourcePath string) (*BackupJob, error) { + ret := _mock.Called(tag, sourcePath) + + if len(ret) == 0 { + panic("no return value specified for GetBackupJobByTagAndSourcePath") + } + + var r0 *BackupJob + var r1 error + if returnFunc, ok := ret.Get(0).(func(string, string) (*BackupJob, error)); ok { + return returnFunc(tag, sourcePath) + } + if returnFunc, ok := ret.Get(0).(func(string, string) *BackupJob); ok { + r0 = returnFunc(tag, sourcePath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*BackupJob) + } + } + if returnFunc, ok := ret.Get(1).(func(string, string) error); ok { + r1 = returnFunc(tag, sourcePath) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockInternalDB_GetBackupJobByTagAndSourcePath_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBackupJobByTagAndSourcePath' +type MockInternalDB_GetBackupJobByTagAndSourcePath_Call struct { + *mock.Call +} + +// GetBackupJobByTagAndSourcePath is a helper method to define mock.On call +// - tag string +// - sourcePath string +func (_e *MockInternalDB_Expecter) GetBackupJobByTagAndSourcePath(tag interface{}, sourcePath interface{}) *MockInternalDB_GetBackupJobByTagAndSourcePath_Call { + return &MockInternalDB_GetBackupJobByTagAndSourcePath_Call{Call: _e.mock.On("GetBackupJobByTagAndSourcePath", tag, sourcePath)} +} + +func (_c *MockInternalDB_GetBackupJobByTagAndSourcePath_Call) Run(run func(tag string, sourcePath string)) *MockInternalDB_GetBackupJobByTagAndSourcePath_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockInternalDB_GetBackupJobByTagAndSourcePath_Call) Return(backupJob *BackupJob, err error) *MockInternalDB_GetBackupJobByTagAndSourcePath_Call { + _c.Call.Return(backupJob, err) + return _c +} + +func (_c *MockInternalDB_GetBackupJobByTagAndSourcePath_Call) RunAndReturn(run func(tag string, sourcePath string) (*BackupJob, error)) *MockInternalDB_GetBackupJobByTagAndSourcePath_Call { + _c.Call.Return(run) + return _c +} + // GetBackupJobs provides a mock function for the type MockInternalDB func (_mock *MockInternalDB) GetBackupJobs() ([]BackupJob, error) { ret := _mock.Called() @@ -2832,6 +2900,74 @@ func (_c *MockInternalDB_GetCompletedRevisionCount_Call) RunAndReturn(run func(f return _c } +// GetDirectoryDownloadArchive provides a mock function for the type MockInternalDB +func (_mock *MockInternalDB) GetDirectoryDownloadArchive(normalizedPath string, sourceFingerprint string) (*DirectoryDownloadArchive, error) { + ret := _mock.Called(normalizedPath, sourceFingerprint) + + if len(ret) == 0 { + panic("no return value specified for GetDirectoryDownloadArchive") + } + + var r0 *DirectoryDownloadArchive + var r1 error + if returnFunc, ok := ret.Get(0).(func(string, string) (*DirectoryDownloadArchive, error)); ok { + return returnFunc(normalizedPath, sourceFingerprint) + } + if returnFunc, ok := ret.Get(0).(func(string, string) *DirectoryDownloadArchive); ok { + r0 = returnFunc(normalizedPath, sourceFingerprint) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*DirectoryDownloadArchive) + } + } + if returnFunc, ok := ret.Get(1).(func(string, string) error); ok { + r1 = returnFunc(normalizedPath, sourceFingerprint) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockInternalDB_GetDirectoryDownloadArchive_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDirectoryDownloadArchive' +type MockInternalDB_GetDirectoryDownloadArchive_Call struct { + *mock.Call +} + +// GetDirectoryDownloadArchive is a helper method to define mock.On call +// - normalizedPath string +// - sourceFingerprint string +func (_e *MockInternalDB_Expecter) GetDirectoryDownloadArchive(normalizedPath interface{}, sourceFingerprint interface{}) *MockInternalDB_GetDirectoryDownloadArchive_Call { + return &MockInternalDB_GetDirectoryDownloadArchive_Call{Call: _e.mock.On("GetDirectoryDownloadArchive", normalizedPath, sourceFingerprint)} +} + +func (_c *MockInternalDB_GetDirectoryDownloadArchive_Call) Run(run func(normalizedPath string, sourceFingerprint string)) *MockInternalDB_GetDirectoryDownloadArchive_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockInternalDB_GetDirectoryDownloadArchive_Call) Return(directoryDownloadArchive *DirectoryDownloadArchive, err error) *MockInternalDB_GetDirectoryDownloadArchive_Call { + _c.Call.Return(directoryDownloadArchive, err) + return _c +} + +func (_c *MockInternalDB_GetDirectoryDownloadArchive_Call) RunAndReturn(run func(normalizedPath string, sourceFingerprint string) (*DirectoryDownloadArchive, error)) *MockInternalDB_GetDirectoryDownloadArchive_Call { + _c.Call.Return(run) + return _c +} + // GetDirectoryShortcut provides a mock function for the type MockInternalDB func (_mock *MockInternalDB) GetDirectoryShortcut(id int64) (*DirectoryShortcut, error) { ret := _mock.Called(id) @@ -3938,6 +4074,76 @@ func (_c *MockInternalDB_GetRunningBackupRunForJob_Call) RunAndReturn(run func(j return _c } +// GetRunningBackupRunForTag provides a mock function for the type MockInternalDB +func (_mock *MockInternalDB) GetRunningBackupRunForTag(tag string) (*BackupRun, *BackupJob, error) { + ret := _mock.Called(tag) + + if len(ret) == 0 { + panic("no return value specified for GetRunningBackupRunForTag") + } + + var r0 *BackupRun + var r1 *BackupJob + var r2 error + if returnFunc, ok := ret.Get(0).(func(string) (*BackupRun, *BackupJob, error)); ok { + return returnFunc(tag) + } + if returnFunc, ok := ret.Get(0).(func(string) *BackupRun); ok { + r0 = returnFunc(tag) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*BackupRun) + } + } + if returnFunc, ok := ret.Get(1).(func(string) *BackupJob); ok { + r1 = returnFunc(tag) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*BackupJob) + } + } + if returnFunc, ok := ret.Get(2).(func(string) error); ok { + r2 = returnFunc(tag) + } else { + r2 = ret.Error(2) + } + return r0, r1, r2 +} + +// MockInternalDB_GetRunningBackupRunForTag_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRunningBackupRunForTag' +type MockInternalDB_GetRunningBackupRunForTag_Call struct { + *mock.Call +} + +// GetRunningBackupRunForTag is a helper method to define mock.On call +// - tag string +func (_e *MockInternalDB_Expecter) GetRunningBackupRunForTag(tag interface{}) *MockInternalDB_GetRunningBackupRunForTag_Call { + return &MockInternalDB_GetRunningBackupRunForTag_Call{Call: _e.mock.On("GetRunningBackupRunForTag", tag)} +} + +func (_c *MockInternalDB_GetRunningBackupRunForTag_Call) Run(run func(tag string)) *MockInternalDB_GetRunningBackupRunForTag_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockInternalDB_GetRunningBackupRunForTag_Call) Return(backupRun *BackupRun, backupJob *BackupJob, err error) *MockInternalDB_GetRunningBackupRunForTag_Call { + _c.Call.Return(backupRun, backupJob, err) + return _c +} + +func (_c *MockInternalDB_GetRunningBackupRunForTag_Call) RunAndReturn(run func(tag string) (*BackupRun, *BackupJob, error)) *MockInternalDB_GetRunningBackupRunForTag_Call { + _c.Call.Return(run) + return _c +} + // GetRunningServerViewSyncRun provides a mock function for the type MockInternalDB func (_mock *MockInternalDB) GetRunningServerViewSyncRun() (*ServerViewSyncRun, error) { ret := _mock.Called() @@ -7017,3 +7223,65 @@ func (_c *MockInternalDB_UpdateUserStatus_Call) RunAndReturn(run func(userID int _c.Call.Return(run) return _c } + +// UpsertDirectoryDownloadArchive provides a mock function for the type MockInternalDB +func (_mock *MockInternalDB) UpsertDirectoryDownloadArchive(payload DirectoryDownloadArchivePayload) (*DirectoryDownloadArchive, error) { + ret := _mock.Called(payload) + + if len(ret) == 0 { + panic("no return value specified for UpsertDirectoryDownloadArchive") + } + + var r0 *DirectoryDownloadArchive + var r1 error + if returnFunc, ok := ret.Get(0).(func(DirectoryDownloadArchivePayload) (*DirectoryDownloadArchive, error)); ok { + return returnFunc(payload) + } + if returnFunc, ok := ret.Get(0).(func(DirectoryDownloadArchivePayload) *DirectoryDownloadArchive); ok { + r0 = returnFunc(payload) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*DirectoryDownloadArchive) + } + } + if returnFunc, ok := ret.Get(1).(func(DirectoryDownloadArchivePayload) error); ok { + r1 = returnFunc(payload) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockInternalDB_UpsertDirectoryDownloadArchive_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpsertDirectoryDownloadArchive' +type MockInternalDB_UpsertDirectoryDownloadArchive_Call struct { + *mock.Call +} + +// UpsertDirectoryDownloadArchive is a helper method to define mock.On call +// - payload DirectoryDownloadArchivePayload +func (_e *MockInternalDB_Expecter) UpsertDirectoryDownloadArchive(payload interface{}) *MockInternalDB_UpsertDirectoryDownloadArchive_Call { + return &MockInternalDB_UpsertDirectoryDownloadArchive_Call{Call: _e.mock.On("UpsertDirectoryDownloadArchive", payload)} +} + +func (_c *MockInternalDB_UpsertDirectoryDownloadArchive_Call) Run(run func(payload DirectoryDownloadArchivePayload)) *MockInternalDB_UpsertDirectoryDownloadArchive_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 DirectoryDownloadArchivePayload + if args[0] != nil { + arg0 = args[0].(DirectoryDownloadArchivePayload) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockInternalDB_UpsertDirectoryDownloadArchive_Call) Return(directoryDownloadArchive *DirectoryDownloadArchive, err error) *MockInternalDB_UpsertDirectoryDownloadArchive_Call { + _c.Call.Return(directoryDownloadArchive, err) + return _c +} + +func (_c *MockInternalDB_UpsertDirectoryDownloadArchive_Call) RunAndReturn(run func(payload DirectoryDownloadArchivePayload) (*DirectoryDownloadArchive, error)) *MockInternalDB_UpsertDirectoryDownloadArchive_Call { + _c.Call.Return(run) + return _c +} diff --git a/internal/server/file_download_routes.go b/internal/server/file_download_routes.go index 7185f25..98601d9 100644 --- a/internal/server/file_download_routes.go +++ b/internal/server/file_download_routes.go @@ -23,6 +23,7 @@ import ( "github.com/omnihance/omnihance-a3-agent/internal/constants" "github.com/omnihance/omnihance-a3-agent/internal/db" "github.com/omnihance/omnihance-a3-agent/internal/permissions" + "github.com/omnihance/omnihance-a3-agent/internal/services" "github.com/omnihance/omnihance-a3-agent/internal/utils" ) @@ -53,6 +54,68 @@ func (s *Server) handleCreateFileDownloadLink(w http.ResponseWriter, r *http.Req _ = utils.WriteJSONResponse(w, response) } +func (s *Server) handleCreateDirectoryDownloadLink(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionDownloadFiles) { + return + } + + pathParam := r.URL.Query().Get("path") + if pathParam == "" { + writeFileDownloadError(w, http.StatusBadRequest, constants.ErrorCodeBadRequest, fileDownloadErrorContext, "Path parameter is required") + return + } + + result, err := s.backupService.PrepareDirectoryDownload(r.Context(), filepath.Clean(pathParam), backupUserID(r)) + if err != nil { + writeDirectoryDownloadError(w, err) + return + } + + response, err := s.directoryDownloadResponse(r, result) + if err != nil { + writeDownloadLinkError(w, fileDownloadErrorContext, err) + return + } + + status := http.StatusOK + if response.Status != services.DirectoryDownloadStatusReady { + status = http.StatusAccepted + } + + _ = utils.WriteJSONResponseWithStatus(w, status, response) +} + +func (s *Server) handleGetDirectoryDownloadStatus(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionDownloadFiles) { + return + } + + runID, err := strconv.ParseInt(chi.URLParam(r, "run_id"), 10, 64) + if err != nil { + writeFileDownloadError(w, http.StatusBadRequest, constants.ErrorCodeBadRequest, fileDownloadErrorContext, "Invalid directory download run ID") + return + } + + result, err := s.backupService.GetDirectoryDownloadStatus(r.Context(), runID, backupUserID(r)) + if err != nil { + writeDirectoryDownloadError(w, err) + return + } + + response, err := s.directoryDownloadResponse(r, result) + if err != nil { + writeDownloadLinkError(w, fileDownloadErrorContext, err) + return + } + + status := http.StatusOK + if response.Status == services.DirectoryDownloadStatusInProgress || response.Status == services.DirectoryDownloadStatusStarted { + status = http.StatusAccepted + } + + _ = utils.WriteJSONResponseWithStatus(w, status, response) +} + func (s *Server) handleDownloadLinkedFile(w http.ResponseWriter, r *http.Request) { if !s.requireUserPermission(w, r, permissions.ActionDownloadFiles) { return @@ -344,6 +407,21 @@ func writeDownloadLinkError(w http.ResponseWriter, context string, err error) { writeFileDownloadError(w, http.StatusInternalServerError, constants.ErrorCodeInternalServerError, context, err.Error()) } +func writeDirectoryDownloadError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, services.ErrDirectoryDownloadConflict): + writeFileDownloadError(w, http.StatusConflict, constants.ErrorCodeBadRequest, fileDownloadErrorContext, err.Error()) + case errors.Is(err, services.ErrBackupNotFound), errors.Is(err, db.ErrBackupNotFound): + writeFileDownloadError(w, http.StatusNotFound, constants.ErrorCodeNotFound, fileDownloadErrorContext, err.Error()) + case errors.Is(err, services.ErrBackupInvalid): + writeFileDownloadError(w, http.StatusBadRequest, constants.ErrorCodeBadRequest, fileDownloadErrorContext, err.Error()) + case errors.Is(err, services.ErrBackupJobRunning): + writeFileDownloadError(w, http.StatusConflict, constants.ErrorCodeBadRequest, fileDownloadErrorContext, err.Error()) + default: + writeFileDownloadError(w, http.StatusInternalServerError, constants.ErrorCodeInternalServerError, fileDownloadErrorContext, err.Error()) + } +} + func writeFileDownloadError(w http.ResponseWriter, status int, errorCode string, context string, message string) { _ = utils.WriteJSONResponseWithStatus(w, status, map[string]interface{}{ "errorCode": errorCode, @@ -360,6 +438,42 @@ func newDownloadLinkError(status int, errorCode string, message string) error { } } +func (s *Server) directoryDownloadResponse(r *http.Request, result *services.DirectoryDownloadResult) (*DirectoryDownloadResponse, error) { + response := &DirectoryDownloadResponse{ + Status: result.Status, + Message: result.Message, + JobID: result.JobID, + RunID: result.RunID, + FileID: result.FileID, + ArchiveReused: result.ArchiveReused, + } + + if result.Status != services.DirectoryDownloadStatusReady { + return response, nil + } + + if result.FileID == nil || result.ArchivePath == "" { + return nil, newDownloadLinkError(http.StatusNotFound, constants.ErrorCodeNotFound, "Directory download archive is missing") + } + + backupFileID := *result.FileID + downloadLink, err := s.createDownloadLinkForPath(r, result.ArchivePath, downloadLinkSource{ + sourceType: db.FileDownloadSourceDirectoryDownload, + backupRunID: &result.RunID, + backupFileID: &backupFileID, + }) + if err != nil { + return nil, err + } + + response.DownloadURL = downloadLink.DownloadURL + response.ExpiresAt = &downloadLink.ExpiresAt + response.Reused = downloadLink.Reused + response.DownloadCount = downloadLink.DownloadCount + + return response, nil +} + func downloadRequestIP(r *http.Request) string { host, _, err := net.SplitHostPort(r.RemoteAddr) if err == nil { @@ -384,6 +498,19 @@ type DownloadLinkResponse struct { DownloadCount int64 `json:"download_count"` } +type DirectoryDownloadResponse struct { + Status string `json:"status"` + Message string `json:"message,omitempty"` + JobID int64 `json:"job_id"` + RunID int64 `json:"run_id"` + FileID *int64 `json:"file_id,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Reused bool `json:"reused"` + DownloadCount int64 `json:"download_count"` + ArchiveReused bool `json:"archive_reused"` +} + type downloadLinkSource struct { sourceType string backupRunID *int64 diff --git a/internal/server/file_download_routes_test.go b/internal/server/file_download_routes_test.go index d18daa1..9b5b171 100644 --- a/internal/server/file_download_routes_test.go +++ b/internal/server/file_download_routes_test.go @@ -17,6 +17,7 @@ import ( "github.com/omnihance/omnihance-a3-agent/internal/db" "github.com/omnihance/omnihance-a3-agent/internal/services" "github.com/omnihance/omnihance-a3-agent/internal/utils" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -92,6 +93,83 @@ func TestDownloadLinkedFileRejectsWrongUserExpiredChangedAndMissingFile(t *testi require.Equal(t, http.StatusGone, rr.Code) } +func TestCreateDirectoryDownloadLinkReturnsInProgress(t *testing.T) { + server := newFileDownloadTestServer(t) + backupService := services.NewMockBackupService(t) + server.backupService = backupService + sourceDir := t.TempDir() + runID := int64(77) + + backupService.EXPECT(). + PrepareDirectoryDownload(mock.Anything, filepath.Clean(sourceDir), mock.Anything). + Return(&services.DirectoryDownloadResult{ + Status: services.DirectoryDownloadStatusInProgress, + Message: "This directory download is already in progress. Keep this page open; the download will start when ready.", + JobID: 55, + RunID: runID, + }, nil) + + req := downloadRequest(http.MethodPost, "/api/file-tree/directory-download-link?path="+url.QueryEscape(sourceDir), nil, constants.RoleAdmin, 1) + rr := httptest.NewRecorder() + server.handleCreateDirectoryDownloadLink(rr, req) + + require.Equal(t, http.StatusAccepted, rr.Code, rr.Body.String()) + var response DirectoryDownloadResponse + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &response)) + require.Equal(t, services.DirectoryDownloadStatusInProgress, response.Status) + require.Equal(t, runID, response.RunID) +} + +func TestCreateDirectoryDownloadLinkUsesSecureDownloadTableWhenReady(t *testing.T) { + server := newFileDownloadTestServer(t) + backupService := services.NewMockBackupService(t) + server.backupService = backupService + dir := t.TempDir() + archivePath := writeDownloadTestFile(t, dir, "server.zip", []byte("zip data")) + sourcePath := filepath.Join(dir, "source") + + job, err := server.internalDB.CreateBackupJob(db.BackupJobPayload{ + JobType: db.BackupJobTypeFile, + Name: "Directory download: source", + Status: db.BackupJobStatusActive, + DestinationDirectory: dir, + SourcePath: &sourcePath, + }, nil) + require.NoError(t, err) + run, err := server.internalDB.CreateBackupRun(job.ID, db.BackupRunTriggerDirectoryDownload, db.BackupJobStatusActive, nil) + require.NoError(t, err) + file, err := server.internalDB.CreateBackupRunFile(run.ID, "source", archivePath, 8) + require.NoError(t, err) + + backupService.EXPECT(). + PrepareDirectoryDownload(mock.Anything, filepath.Clean(sourcePath), mock.Anything). + Return(&services.DirectoryDownloadResult{ + Status: services.DirectoryDownloadStatusReady, + JobID: job.ID, + RunID: run.ID, + FileID: &file.ID, + ArchivePath: archivePath, + }, nil) + + req := downloadRequest(http.MethodPost, "/api/file-tree/directory-download-link?path="+url.QueryEscape(sourcePath), nil, constants.RoleAdmin, 1) + rr := httptest.NewRecorder() + server.handleCreateDirectoryDownloadLink(rr, req) + + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + var response DirectoryDownloadResponse + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &response)) + require.Equal(t, services.DirectoryDownloadStatusReady, response.Status) + require.True(t, strings.HasPrefix(response.DownloadURL, "/api/file-tree/download/")) + + publicID, _, err := server.verifyDownloadToken(strings.TrimPrefix(response.DownloadURL, "/api/file-tree/download/")) + require.NoError(t, err) + link, err := server.internalDB.GetFileDownloadLinkByPublicID(publicID) + require.NoError(t, err) + require.Equal(t, db.FileDownloadSourceDirectoryDownload, link.SourceType) + require.Equal(t, run.ID, *link.BackupRunID) + require.Equal(t, file.ID, *link.BackupFileID) +} + func newFileDownloadTestServer(t *testing.T) *Server { t.Helper() diff --git a/internal/server/file_system_routes.go b/internal/server/file_system_routes.go index 1d72f4a..b474f6f 100644 --- a/internal/server/file_system_routes.go +++ b/internal/server/file_system_routes.go @@ -62,6 +62,8 @@ func (s *Server) InitializeFileSystemRoutes(r *chi.Mux) { r.Post("/duplicate-file", s.handleDuplicateFile) r.Get("/revision-summary", s.handleRevisionSummary) r.Post("/download-link", s.handleCreateFileDownloadLink) + r.Post("/directory-download-link", s.handleCreateDirectoryDownloadLink) + r.Get("/directory-downloads/{run_id}", s.handleGetDirectoryDownloadStatus) r.Get("/download/{token}", s.handleDownloadLinkedFile) }) } diff --git a/internal/services/backup_service.go b/internal/services/backup_service.go index 55a8cb2..499f73a 100644 --- a/internal/services/backup_service.go +++ b/internal/services/backup_service.go @@ -3,7 +3,9 @@ package services import ( "context" "crypto/rand" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" "io" @@ -34,14 +36,24 @@ const ( backupArchiveExtension = ".zip" backupSearchLimit = 10 backupSkipRunningMsg = "Skipped because backup job is already running." + directoryDownloadName = "Directory download" ) var ( - ErrBackupInvalid = errors.New("invalid backup job") - ErrBackupJobRunning = errors.New("backup job is currently running") - ErrBackupNotFound = errors.New("backup item not found") - ErrBackupRemoteSQLHost = errors.New("remote SQL Server backups are not supported") - ErrBackupNoRunningJob = errors.New("backup job is not running") + ErrBackupInvalid = errors.New("invalid backup job") + ErrBackupJobRunning = errors.New("backup job is currently running") + ErrBackupNotFound = errors.New("backup item not found") + ErrBackupRemoteSQLHost = errors.New("remote SQL Server backups are not supported") + ErrBackupNoRunningJob = errors.New("backup job is not running") + ErrDirectoryDownloadConflict = errors.New("another directory download job is in progress") +) + +const ( + DirectoryDownloadStatusReady = "ready" + DirectoryDownloadStatusStarted = "started" + DirectoryDownloadStatusInProgress = "in_progress" + DirectoryDownloadStatusFailed = "failed" + DirectoryDownloadStatusCancelled = "cancelled" ) type BackupService interface { @@ -57,6 +69,8 @@ type BackupService interface { GetRuns(jobID int64, page int, pageSize int) ([]db.BackupRun, int64, error) GetRunDetails(runID int64) (*BackupRunDetails, error) GetRunFile(fileID int64) (*db.BackupRunFile, error) + PrepareDirectoryDownload(ctx context.Context, path string, userID *int64) (*DirectoryDownloadResult, error) + GetDirectoryDownloadStatus(ctx context.Context, runID int64, userID *int64) (*DirectoryDownloadResult, error) SearchPaths(query string, kind string) ([]PathSearchResult, error) GetSQLServerDefaults() SQLServerBackupDefaults } @@ -66,6 +80,16 @@ type BackupRunDetails struct { Files []db.BackupRunFile `json:"files"` } +type DirectoryDownloadResult struct { + Status string + Message string + JobID int64 + RunID int64 + FileID *int64 + ArchivePath string + ArchiveReused bool +} + type PathSearchResult struct { Name string `json:"name"` Path string `json:"path"` @@ -234,6 +258,10 @@ func (s *backupService) UpdateJob(ctx context.Context, id int64, payload db.Back payload.SQLPassword = job.SQLPassword } + if payload.Tag == nil { + payload.Tag = job.Tag + } + normalizedPayload, err := s.validateJobPayload(ctx, payload) if err != nil { return nil, err @@ -304,44 +332,48 @@ func (s *backupService) RunJob(ctx context.Context, id int64, triggerType string s.mu.Lock() defer s.mu.Unlock() - if _, ok := s.runningCancels[id]; ok { + return s.runJobLocked(*job, triggerType, userID) +} + +func (s *backupService) runJobLocked(job db.BackupJob, triggerType string, userID *int64) (*db.BackupRun, error) { + if _, ok := s.runningCancels[job.ID]; ok { if triggerType == db.BackupRunTriggerCron { - return s.createSkippedCronRun(*job) + return s.createSkippedCronRun(job) } return nil, ErrBackupJobRunning } - if running, err := s.internalDB.GetRunningBackupRunForJob(id); err != nil { + if running, err := s.internalDB.GetRunningBackupRunForJob(job.ID); err != nil { return nil, err } else if running != nil { if triggerType == db.BackupRunTriggerCron { - return s.createSkippedCronRun(*job) + return s.createSkippedCronRun(job) } return nil, ErrBackupJobRunning } - lockPath, err := s.acquireJobLock(id) + lockPath, err := s.acquireJobLock(job.ID) if err != nil { if triggerType == db.BackupRunTriggerCron && errors.Is(err, ErrBackupJobRunning) { - return s.createSkippedCronRun(*job) + return s.createSkippedCronRun(job) } return nil, err } - run, err := s.internalDB.CreateBackupRun(id, triggerType, job.Status, userID) + run, err := s.internalDB.CreateBackupRun(job.ID, triggerType, job.Status, userID) if err != nil { s.releaseJobLock(lockPath) return nil, err } runCtx, cancel := context.WithCancel(context.Background()) - s.runningCancels[id] = cancel - s.runningRunIDs[id] = run.ID + s.runningCancels[job.ID] = cancel + s.runningRunIDs[job.ID] = run.ID s.wg.Add(1) - go s.executeRun(runCtx, *job, *run, lockPath) + go s.executeRun(runCtx, job, *run, lockPath) return run, nil } @@ -411,6 +443,143 @@ func (s *backupService) GetRunFile(fileID int64) (*db.BackupRunFile, error) { return file, nil } +func (s *backupService) PrepareDirectoryDownload(ctx context.Context, path string, userID *int64) (*DirectoryDownloadResult, error) { + normalizedPath, err := s.validateDirectoryDownloadPath(path) + if err != nil { + return nil, err + } + + runningResult, err := s.runningDirectoryDownloadResult(normalizedPath) + if err != nil { + return nil, err + } + + if runningResult != nil { + return runningResult, nil + } + + sourceFingerprint, err := s.buildDirectoryDownloadFingerprint(ctx, normalizedPath) + if err != nil { + return nil, err + } + + archive, err := s.internalDB.GetDirectoryDownloadArchive(normalizedPath, sourceFingerprint) + if err == nil { + if result := s.directoryDownloadArchiveResult(archive); result != nil { + return result, nil + } + } else if !errors.Is(err, constants.ErrNotFound) { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + runningRun, runningJob, err := s.internalDB.GetRunningBackupRunForTag(db.BackupJobTagDirectoryDownload) + if err != nil { + return nil, err + } + + if runningRun != nil && runningJob != nil { + if runningJob.SourcePath != nil && filepath.Clean(*runningJob.SourcePath) == normalizedPath { + return directoryDownloadInProgressResult(*runningJob, *runningRun), nil + } + + return nil, ErrDirectoryDownloadConflict + } + + job, err := s.internalDB.GetBackupJobByTagAndSourcePath(db.BackupJobTagDirectoryDownload, normalizedPath) + if err != nil { + if !errors.Is(err, db.ErrBackupNotFound) { + return nil, err + } + + tag := db.BackupJobTagDirectoryDownload + sourcePath := normalizedPath + payload := db.BackupJobPayload{ + JobType: db.BackupJobTypeFile, + Tag: &tag, + Name: directoryDownloadJobName(normalizedPath), + Status: db.BackupJobStatusActive, + DestinationDirectory: s.directoryDownloadsDirectory(), + SourcePath: &sourcePath, + } + + job, err = s.internalDB.CreateBackupJob(payload, userID) + if err != nil { + return nil, err + } + } + + run, err := s.runJobLocked(*job, db.BackupRunTriggerDirectoryDownload, userID) + if err != nil { + return nil, err + } + + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusStarted, + Message: "Compressing directory. Keep this page open and do not refresh; the download will start automatically when ready.", + JobID: job.ID, + RunID: run.ID, + }, nil +} + +func (s *backupService) GetDirectoryDownloadStatus(ctx context.Context, runID int64, userID *int64) (*DirectoryDownloadResult, error) { + _ = ctx + _ = userID + + run, err := s.internalDB.GetBackupRun(runID) + if err != nil { + return nil, backupNotFoundError(err) + } + + job, err := s.internalDB.GetBackupJob(run.JobID) + if err != nil { + return nil, backupNotFoundError(err) + } + + if !isDirectoryDownloadJob(*job) { + return nil, ErrBackupNotFound + } + + switch run.Status { + case db.BackupRunStatusRunning: + return directoryDownloadInProgressResult(*job, *run), nil + case db.BackupRunStatusFailed: + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusFailed, + Message: backupRunMessage(*run, "Directory download failed"), + JobID: job.ID, + RunID: run.ID, + }, nil + case db.BackupRunStatusCancelled: + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusCancelled, + Message: backupRunMessage(*run, "Directory download was cancelled"), + JobID: job.ID, + RunID: run.ID, + }, nil + case db.BackupRunStatusSucceeded: + files, err := s.internalDB.GetBackupRunFiles(run.ID) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, fmt.Errorf("%w: directory download archive missing", ErrBackupNotFound) + } + + file := files[0] + if info, err := s.fileEditor.Stat(file.FilePath); err != nil || info.IsDir() { + return nil, fmt.Errorf("%w: directory download archive missing", ErrBackupNotFound) + } + + return directoryDownloadReadyResult(*job, run.ID, file.ID, file.FilePath, false), nil + default: + return nil, fmt.Errorf("%w: unsupported directory download run status %s", ErrBackupInvalid, run.Status) + } +} + func backupNotFoundError(err error) error { if errors.Is(err, db.ErrBackupNotFound) || errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %w", ErrBackupNotFound, err) @@ -526,6 +695,7 @@ func (s *backupService) validateJobPayload(ctx context.Context, payload db.Backu payload.JobType = strings.TrimSpace(payload.JobType) payload.Status = strings.TrimSpace(payload.Status) payload.DestinationDirectory = filepath.Clean(strings.TrimSpace(payload.DestinationDirectory)) + payload.Tag = normalizeOptionalString(payload.Tag, true) payload.CronExpression = normalizeOptionalString(payload.CronExpression, true) payload.ArchivePassword = normalizeOptionalString(payload.ArchivePassword, false) payload.SourcePath = normalizeOptionalPath(payload.SourcePath) @@ -690,6 +860,10 @@ func (s *backupService) runFileBackup(ctx context.Context, job db.BackupJob, run return errors.New("source path is missing") } + if isDirectoryDownloadJob(job) { + return s.runDirectoryDownloadBackup(ctx, job, run, output) + } + sourceInfo, err := s.fileEditor.Stat(*job.SourcePath) if err != nil { return err @@ -725,6 +899,71 @@ func (s *backupService) runFileBackup(ctx context.Context, job db.BackupJob, run return nil } +func (s *backupService) runDirectoryDownloadBackup(ctx context.Context, job db.BackupJob, run db.BackupRun, output *strings.Builder) error { + normalizedPath, err := s.validateDirectoryDownloadPath(*job.SourcePath) + if err != nil { + return err + } + + sourceFingerprint, err := s.buildDirectoryDownloadFingerprint(ctx, normalizedPath) + if err != nil { + return err + } + + itemName := filepath.Base(normalizedPath) + if itemName == "." || itemName == string(filepath.Separator) { + itemName = "directory" + } + + archivePath, err := uniqueBackupPath(job.DestinationDirectory, itemName, job.ID) + if err != nil { + return err + } + + output.WriteString("Creating directory download archive " + archivePath + "\n") + if err := createZipArchiveWithExclusions(ctx, normalizedPath, archivePath, "", []string{job.DestinationDirectory}); err != nil { + _ = s.fileEditor.Remove(archivePath) + return err + } + + currentFingerprint, err := s.buildDirectoryDownloadFingerprint(ctx, normalizedPath) + if err != nil { + _ = s.fileEditor.Remove(archivePath) + return err + } + + if currentFingerprint != sourceFingerprint { + _ = s.fileEditor.Remove(archivePath) + return errors.New("directory changed while it was being compressed; please try again") + } + + archiveInfo, err := s.fileEditor.Stat(archivePath) + if err != nil { + return err + } + + file, err := s.internalDB.CreateBackupRunFile(run.ID, itemName, archivePath, archiveInfo.Size()) + if err != nil { + return err + } + + if _, err := s.internalDB.UpsertDirectoryDownloadArchive(db.DirectoryDownloadArchivePayload{ + NormalizedPath: normalizedPath, + SourceFingerprint: sourceFingerprint, + JobID: job.ID, + RunID: run.ID, + FileID: file.ID, + ArchivePath: archivePath, + ArchiveSize: archiveInfo.Size(), + }); err != nil { + return err + } + + _, _ = fmt.Fprintf(output, "Archived %s (%d bytes).\n", filepath.Base(normalizedPath), archiveInfo.Size()) + + return nil +} + func (s *backupService) runSQLServerBackup(ctx context.Context, job db.BackupJob, run db.BackupRun, output *strings.Builder) error { names := parseDatabaseNames(stringValue(job.SQLDatabaseNames)) if len(names) == 0 { @@ -963,6 +1202,10 @@ func openSQLServerDatabase(ctx context.Context, job db.BackupJob) (*sql.DB, erro } func createZipArchive(ctx context.Context, sourcePath string, archivePath string, password string) error { + return createZipArchiveWithExclusions(ctx, sourcePath, archivePath, password, nil) +} + +func createZipArchiveWithExclusions(ctx context.Context, sourcePath string, archivePath string, password string, excludedPaths []string) error { archiveFile, err := os.Create(archivePath) if err != nil { return err @@ -989,11 +1232,20 @@ func createZipArchive(ctx context.Context, sourcePath string, archivePath string baseDir = filepath.Dir(sourcePath) } + excludedAbsolutePaths := cleanDescendantAbsolutePaths(sourcePath, excludedPaths) err = filepath.WalkDir(sourcePath, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } + if shouldSkipZipPath(path, entry, sourcePath, excludedAbsolutePaths) { + if entry.IsDir() { + return filepath.SkipDir + } + + return nil + } + if err := ctx.Err(); err != nil { return err } @@ -1058,6 +1310,61 @@ func createZipArchive(ctx context.Context, sourcePath string, archivePath string return closeArchive() } +func cleanDescendantAbsolutePaths(sourcePath string, paths []string) []string { + results := make([]string, 0, len(paths)) + sourceAbsolutePath, err := filepath.Abs(filepath.Clean(sourcePath)) + if err != nil { + return results + } + + for _, path := range paths { + path = strings.TrimSpace(path) + if path == "" { + continue + } + + absolutePath, err := filepath.Abs(filepath.Clean(path)) + if err != nil { + continue + } + + if absolutePath == sourceAbsolutePath || !isPathWithin(sourceAbsolutePath, absolutePath) { + continue + } + + results = append(results, absolutePath) + } + + return results +} + +func shouldSkipZipPath(path string, entry os.DirEntry, sourcePath string, excludedAbsolutePaths []string) bool { + if len(excludedAbsolutePaths) == 0 { + return false + } + + if filepath.Clean(path) == filepath.Clean(sourcePath) { + return false + } + + absolutePath, err := filepath.Abs(filepath.Clean(path)) + if err != nil { + return false + } + + for _, excludedPath := range excludedAbsolutePaths { + if entry.IsDir() && absolutePath == excludedPath { + return true + } + + if isPathWithin(excludedPath, absolutePath) { + return true + } + } + + return false +} + func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { buffer := make([]byte, 1024*128) var written int64 @@ -1094,6 +1401,201 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, return written, nil } +func (s *backupService) validateDirectoryDownloadPath(path string) (string, error) { + path = filepath.Clean(strings.TrimSpace(path)) + if path == "" || path == "." { + return "", fmt.Errorf("%w: directory path is required", ErrBackupInvalid) + } + + info, err := s.fileEditor.Stat(path) + if err != nil { + if s.fileEditor.IsNotExist(err) { + return "", fmt.Errorf("%w: directory path does not exist", ErrBackupInvalid) + } + + return "", fmt.Errorf("%w: cannot access directory path: %v", ErrBackupInvalid, err) + } + + if !info.IsDir() { + return "", fmt.Errorf("%w: path is not a directory", ErrBackupInvalid) + } + + return path, nil +} + +func (s *backupService) buildDirectoryDownloadFingerprint(ctx context.Context, sourcePath string) (string, error) { + hash := sha256.New() + sourcePath = filepath.Clean(sourcePath) + excludedPath := s.directoryDownloadsDirectory() + excludedAbsolutePaths := cleanDescendantAbsolutePaths(sourcePath, []string{excludedPath}) + + err := filepath.WalkDir(sourcePath, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if shouldSkipZipPath(path, entry, sourcePath, excludedAbsolutePaths) { + if entry.IsDir() { + return filepath.SkipDir + } + + return nil + } + + if err := ctx.Err(); err != nil { + return err + } + + info, err := entry.Info() + if err != nil { + return err + } + + name, err := filepath.Rel(sourcePath, path) + if err != nil { + return err + } + + name = filepath.ToSlash(name) + _, _ = fmt.Fprintf( + hash, + "path=%s\x00dir=%t\x00mode=%s\x00size=%d\x00mod=%d\x00", + name, + entry.IsDir(), + info.Mode().String(), + info.Size(), + info.ModTime().UnixNano(), + ) + + if entry.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + + _, copyErr := copyWithContext(ctx, hash, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + + return closeErr + }) + if err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func (s *backupService) runningDirectoryDownloadResult(normalizedPath string) (*DirectoryDownloadResult, error) { + runningRun, runningJob, err := s.internalDB.GetRunningBackupRunForTag(db.BackupJobTagDirectoryDownload) + if err != nil { + return nil, err + } + + if runningRun == nil || runningJob == nil { + return nil, nil + } + + if runningJob.SourcePath != nil && filepath.Clean(*runningJob.SourcePath) == normalizedPath { + return directoryDownloadInProgressResult(*runningJob, *runningRun), nil + } + + return nil, ErrDirectoryDownloadConflict +} + +func (s *backupService) directoryDownloadArchiveResult(archive *db.DirectoryDownloadArchive) *DirectoryDownloadResult { + if archive == nil { + return nil + } + + info, err := s.fileEditor.Stat(archive.ArchivePath) + if err != nil || info.IsDir() { + return nil + } + + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusReady, + JobID: archive.JobID, + RunID: archive.RunID, + FileID: &archive.FileID, + ArchivePath: archive.ArchivePath, + ArchiveReused: true, + } +} + +func directoryDownloadReadyResult(job db.BackupJob, runID int64, fileID int64, archivePath string, archiveReused bool) *DirectoryDownloadResult { + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusReady, + JobID: job.ID, + RunID: runID, + FileID: &fileID, + ArchivePath: archivePath, + ArchiveReused: archiveReused, + } +} + +func directoryDownloadInProgressResult(job db.BackupJob, run db.BackupRun) *DirectoryDownloadResult { + return &DirectoryDownloadResult{ + Status: DirectoryDownloadStatusInProgress, + Message: "This directory download is already in progress. Keep this page open; the download will start when ready.", + JobID: job.ID, + RunID: run.ID, + } +} + +func backupRunMessage(run db.BackupRun, fallback string) string { + if run.ErrorDetails != nil && strings.TrimSpace(*run.ErrorDetails) != "" { + return *run.ErrorDetails + } + + if run.Output != nil && strings.TrimSpace(*run.Output) != "" { + return *run.Output + } + + return fallback +} + +func isDirectoryDownloadJob(job db.BackupJob) bool { + return job.Tag != nil && *job.Tag == db.BackupJobTagDirectoryDownload +} + +func directoryDownloadJobName(path string) string { + baseName := filepath.Base(filepath.Clean(path)) + if baseName == "." || baseName == string(filepath.Separator) { + return directoryDownloadName + } + + return directoryDownloadName + ": " + baseName +} + +func (s *backupService) directoryDownloadsDirectory() string { + if s.cfg == nil || strings.TrimSpace(s.cfg.DirectoryDownloadsDirectory) == "" { + return ".directory-download" + } + + return filepath.Clean(strings.TrimSpace(s.cfg.DirectoryDownloadsDirectory)) +} + +func isPathWithin(parentPath string, childPath string) bool { + parentPath = filepath.Clean(parentPath) + childPath = filepath.Clean(childPath) + if parentPath == childPath { + return true + } + + relativePath, err := filepath.Rel(parentPath, childPath) + if err != nil { + return false + } + + return relativePath != "." && relativePath != ".." && !strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) +} + func uniqueBackupPath(directory string, itemName string, jobID int64) (string, error) { timestamp := time.Now().Format("20060102150405") baseName := fmt.Sprintf("%s-%d-%s", safeBackupName(itemName), jobID, timestamp) diff --git a/internal/services/backup_service_test.go b/internal/services/backup_service_test.go new file mode 100644 index 0000000..294f6d6 --- /dev/null +++ b/internal/services/backup_service_test.go @@ -0,0 +1,135 @@ +package services + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/omnihance/omnihance-a3-agent/internal/config" + "github.com/omnihance/omnihance-a3-agent/internal/db" + "github.com/omnihance/omnihance-a3-agent/internal/logger" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestPrepareDirectoryDownloadCreatesTaggedJobAndReusesCache(t *testing.T) { + service, internalDB := newBackupServiceForTest(t) + sourceDir := writeDirectoryDownloadSource(t, "server", "maps/temoz.map", []byte("map data")) + + result, err := service.PrepareDirectoryDownload(context.Background(), sourceDir, nil) + require.NoError(t, err) + require.Equal(t, DirectoryDownloadStatusStarted, result.Status) + require.Contains(t, result.Message, "do not refresh") + + ready := waitDirectoryDownloadReady(t, service, result.RunID) + require.Equal(t, DirectoryDownloadStatusReady, ready.Status) + require.NotEmpty(t, ready.ArchivePath) + + job, err := internalDB.GetBackupJob(result.JobID) + require.NoError(t, err) + require.NotNil(t, job.Tag) + require.Equal(t, db.BackupJobTagDirectoryDownload, *job.Tag) + require.Nil(t, job.CronExpression) + require.Equal(t, filepath.Clean(sourceDir), *job.SourcePath) + + fingerprint, err := service.buildDirectoryDownloadFingerprint(context.Background(), filepath.Clean(sourceDir)) + require.NoError(t, err) + archive, err := internalDB.GetDirectoryDownloadArchive(filepath.Clean(sourceDir), fingerprint) + require.NoError(t, err) + require.Equal(t, ready.ArchivePath, archive.ArchivePath) + + reused, err := service.PrepareDirectoryDownload(context.Background(), sourceDir, nil) + require.NoError(t, err) + require.Equal(t, DirectoryDownloadStatusReady, reused.Status) + require.True(t, reused.ArchiveReused) + require.Equal(t, ready.ArchivePath, reused.ArchivePath) +} + +func TestPrepareDirectoryDownloadResumesSameRunningJobAndRejectsDifferentDirectory(t *testing.T) { + service, internalDB := newBackupServiceForTest(t) + sourceDir := writeDirectoryDownloadSource(t, "server", "maps/temoz.map", []byte("map data")) + otherDir := writeDirectoryDownloadSource(t, "other", "maps/quanato.map", []byte("map data")) + + tag := db.BackupJobTagDirectoryDownload + sourcePath := filepath.Clean(sourceDir) + job, err := internalDB.CreateBackupJob(db.BackupJobPayload{ + JobType: db.BackupJobTypeFile, + Tag: &tag, + Name: "Directory download: server", + Status: db.BackupJobStatusActive, + DestinationDirectory: service.directoryDownloadsDirectory(), + SourcePath: &sourcePath, + }, nil) + require.NoError(t, err) + + run, err := internalDB.CreateBackupRun(job.ID, db.BackupRunTriggerDirectoryDownload, db.BackupJobStatusActive, nil) + require.NoError(t, err) + + result, err := service.PrepareDirectoryDownload(context.Background(), sourceDir, nil) + require.NoError(t, err) + require.Equal(t, DirectoryDownloadStatusInProgress, result.Status) + require.Equal(t, run.ID, result.RunID) + + _, err = service.PrepareDirectoryDownload(context.Background(), otherDir, nil) + require.ErrorIs(t, err, ErrDirectoryDownloadConflict) +} + +func newBackupServiceForTest(t *testing.T) (*backupService, db.InternalDB) { + t.Helper() + + log := logger.NewZerologLogger(zerolog.New(io.Discard), "test", zerolog.Disabled) + internalDB := db.NewSQLiteDB(filepath.Join(t.TempDir(), "test.db"), log) + require.NoError(t, internalDB.Connect()) + require.NoError(t, internalDB.MigrateUp()) + t.Cleanup(func() { + require.NoError(t, internalDB.Close()) + }) + + baseDir := t.TempDir() + service := NewBackupService( + &config.EnvVars{ + BackupsDirectory: filepath.Join(baseDir, "backups"), + DirectoryDownloadsDirectory: filepath.Join(baseDir, "directory-downloads"), + }, + log, + internalDB, + NewFileEditorService(log), + ).(*backupService) + + return service, internalDB +} + +func writeDirectoryDownloadSource(t *testing.T, name string, fileName string, content []byte) string { + t.Helper() + + dir := filepath.Join(t.TempDir(), name) + path := filepath.Join(dir, fileName) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, content, 0600)) + return dir +} + +func waitDirectoryDownloadReady(t *testing.T, service *backupService, runID int64) *DirectoryDownloadResult { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + result, err := service.GetDirectoryDownloadStatus(context.Background(), runID, nil) + require.NoError(t, err) + + switch result.Status { + case DirectoryDownloadStatusReady: + return result + case DirectoryDownloadStatusFailed, DirectoryDownloadStatusCancelled: + t.Fatalf("directory download ended with %s: %s", result.Status, result.Message) + } + + time.Sleep(25 * time.Millisecond) + } + + t.Fatalf("directory download did not finish") + return nil +} diff --git a/internal/services/mock_BackupService.go b/internal/services/mock_BackupService.go index 9556725..5e2b0d1 100644 --- a/internal/services/mock_BackupService.go +++ b/internal/services/mock_BackupService.go @@ -243,6 +243,80 @@ func (_c *MockBackupService_DeleteJob_Call) RunAndReturn(run func(ctx context.Co return _c } +// GetDirectoryDownloadStatus provides a mock function for the type MockBackupService +func (_mock *MockBackupService) GetDirectoryDownloadStatus(ctx context.Context, runID int64, userID *int64) (*DirectoryDownloadResult, error) { + ret := _mock.Called(ctx, runID, userID) + + if len(ret) == 0 { + panic("no return value specified for GetDirectoryDownloadStatus") + } + + var r0 *DirectoryDownloadResult + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, int64, *int64) (*DirectoryDownloadResult, error)); ok { + return returnFunc(ctx, runID, userID) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, int64, *int64) *DirectoryDownloadResult); ok { + r0 = returnFunc(ctx, runID, userID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*DirectoryDownloadResult) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, int64, *int64) error); ok { + r1 = returnFunc(ctx, runID, userID) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockBackupService_GetDirectoryDownloadStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDirectoryDownloadStatus' +type MockBackupService_GetDirectoryDownloadStatus_Call struct { + *mock.Call +} + +// GetDirectoryDownloadStatus is a helper method to define mock.On call +// - ctx context.Context +// - runID int64 +// - userID *int64 +func (_e *MockBackupService_Expecter) GetDirectoryDownloadStatus(ctx interface{}, runID interface{}, userID interface{}) *MockBackupService_GetDirectoryDownloadStatus_Call { + return &MockBackupService_GetDirectoryDownloadStatus_Call{Call: _e.mock.On("GetDirectoryDownloadStatus", ctx, runID, userID)} +} + +func (_c *MockBackupService_GetDirectoryDownloadStatus_Call) Run(run func(ctx context.Context, runID int64, userID *int64)) *MockBackupService_GetDirectoryDownloadStatus_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 int64 + if args[1] != nil { + arg1 = args[1].(int64) + } + var arg2 *int64 + if args[2] != nil { + arg2 = args[2].(*int64) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockBackupService_GetDirectoryDownloadStatus_Call) Return(directoryDownloadResult *DirectoryDownloadResult, err error) *MockBackupService_GetDirectoryDownloadStatus_Call { + _c.Call.Return(directoryDownloadResult, err) + return _c +} + +func (_c *MockBackupService_GetDirectoryDownloadStatus_Call) RunAndReturn(run func(ctx context.Context, runID int64, userID *int64) (*DirectoryDownloadResult, error)) *MockBackupService_GetDirectoryDownloadStatus_Call { + _c.Call.Return(run) + return _c +} + // GetJob provides a mock function for the type MockBackupService func (_mock *MockBackupService) GetJob(id int64) (*db.BackupJob, error) { ret := _mock.Called(id) @@ -608,6 +682,80 @@ func (_c *MockBackupService_GetSQLServerDefaults_Call) RunAndReturn(run func() S return _c } +// PrepareDirectoryDownload provides a mock function for the type MockBackupService +func (_mock *MockBackupService) PrepareDirectoryDownload(ctx context.Context, path string, userID *int64) (*DirectoryDownloadResult, error) { + ret := _mock.Called(ctx, path, userID) + + if len(ret) == 0 { + panic("no return value specified for PrepareDirectoryDownload") + } + + var r0 *DirectoryDownloadResult + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, string, *int64) (*DirectoryDownloadResult, error)); ok { + return returnFunc(ctx, path, userID) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, string, *int64) *DirectoryDownloadResult); ok { + r0 = returnFunc(ctx, path, userID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*DirectoryDownloadResult) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, string, *int64) error); ok { + r1 = returnFunc(ctx, path, userID) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockBackupService_PrepareDirectoryDownload_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PrepareDirectoryDownload' +type MockBackupService_PrepareDirectoryDownload_Call struct { + *mock.Call +} + +// PrepareDirectoryDownload is a helper method to define mock.On call +// - ctx context.Context +// - path string +// - userID *int64 +func (_e *MockBackupService_Expecter) PrepareDirectoryDownload(ctx interface{}, path interface{}, userID interface{}) *MockBackupService_PrepareDirectoryDownload_Call { + return &MockBackupService_PrepareDirectoryDownload_Call{Call: _e.mock.On("PrepareDirectoryDownload", ctx, path, userID)} +} + +func (_c *MockBackupService_PrepareDirectoryDownload_Call) Run(run func(ctx context.Context, path string, userID *int64)) *MockBackupService_PrepareDirectoryDownload_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + var arg2 *int64 + if args[2] != nil { + arg2 = args[2].(*int64) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockBackupService_PrepareDirectoryDownload_Call) Return(directoryDownloadResult *DirectoryDownloadResult, err error) *MockBackupService_PrepareDirectoryDownload_Call { + _c.Call.Return(directoryDownloadResult, err) + return _c +} + +func (_c *MockBackupService_PrepareDirectoryDownload_Call) RunAndReturn(run func(ctx context.Context, path string, userID *int64) (*DirectoryDownloadResult, error)) *MockBackupService_PrepareDirectoryDownload_Call { + _c.Call.Return(run) + return _c +} + // RunJob provides a mock function for the type MockBackupService func (_mock *MockBackupService) RunJob(ctx context.Context, id int64, triggerType string, userID *int64) (*db.BackupRun, error) { ret := _mock.Called(ctx, id, triggerType, userID) From 228cab3a4a8e9cfcafd58997df7772d00b65f2f4 Mon Sep 17 00:00:00 2001 From: cyberinferno Date: Sun, 14 Jun 2026 20:13:55 +0530 Subject: [PATCH 2/2] feat: add directory download polling and caching --- README.md | 14 ++- cmd/omnihance-a3-agent/docs/openapi.yml | 17 ++++ .../omnihance-a3-agent-ui/README.md | 2 +- .../src/components/file-tree.tsx | 45 ++++++++-- internal/services/backup_service.go | 18 ++-- internal/services/backup_service_test.go | 85 +++++++++++++++++++ 6 files changed, 163 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 79db3ae..0c3f866 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ Omnihance A3 Agent is a full-stack application consisting of: - Text files (MIME type detection) - **File Viewing**: View NPC files, quest files, spawn files, drop files, item files, item combination data files, and text files in the browser - **File Duplication**: Right-click any file in the file explorer to duplicate it with a custom name +- **Directory Downloads**: Right-click a directory to request a ZIP download. The directory is compressed in the background through a tagged one-time backup job, stored under `DIRECTORY_DOWNLOADS_DIRECTORY`, and delivered through the same secure temp-link download flow used for files and backup outputs. - **File Editing**: - **NPC File Editor**: Edit NPC properties including: - ID, Name, Respawn Rate @@ -280,6 +281,11 @@ Omnihance A3 Agent is a full-stack application consisting of: - Create manual or scheduled jobs for a selected file or directory - Destination directories are created when missing, and destination paths must be directories - Source path and destination path fields support searchable path suggestions +- **Directory download jobs**: + - Directory-download requests create tagged no-repeat file backup jobs with the `directory_download` tag + - Only one directory-download compression job can run at a time; repeat requests for the same running directory resume polling, while requests for a different directory return a conflict + - Completed directory ZIPs are reused until the source directory fingerprint changes + - Directory-download backup jobs hide their destination in the Backups UI and show the run trigger as `Directory Download` - **Local SQL Server backups**: - Back up one or more comma-separated database names - SQL Server host, port, username, and password can be prefilled from settings @@ -517,6 +523,7 @@ The application uses environment variables for configuration. A `.env` file is a | `VERSION_CHECK_INTERVAL_SECONDS` | `3600` | How often to check GitHub for new releases | | `REVISIONS_DIRECTORY` | `.revisions` | Directory for file revision backups | | `BACKUPS_DIRECTORY` | `.backups` | Directory for internal backup lock files | +| `DIRECTORY_DOWNLOADS_DIRECTORY` | `.directory-download` | Directory for generated directory-download ZIP archives | | `MAX_FILE_UPLOAD_SIZE_MB` | `2` | Maximum multipart upload size in MB | | `DIRECTORY_SHORTCUTS_LIMIT` | `5` | Maximum pinned directories per user (`0` disables the limit) | | `RUNNING_IN_DOCKER` | `false` | Disable host metrics collection when running in Docker | @@ -578,6 +585,8 @@ Only stable GitHub releases are considered because GitHub's latest release endpo - `POST /api/file-tree/duplicate-file` - Duplicate a file in the same directory - `GET /api/file-tree/revision-summary` - Get revision count for a file - `POST /api/file-tree/download-link` - Create or reuse a one-day user-bound download link for a file (requires `download_files` permission) +- `POST /api/file-tree/directory-download-link` - Create, reuse, or start a background directory ZIP job and return either a secure download URL or polling metadata (requires `download_files` permission) +- `GET /api/file-tree/directory-downloads/{run_id}` - Poll a directory-download run until it returns a secure download URL or terminal status - `GET /api/file-tree/download/{token}` - Download a file through a valid user-bound temp link and record the download event ### Metrics @@ -687,9 +696,10 @@ The application uses SQLite with the following main tables: - Stores process name, file path, optional port, sequence order - Tracks start/end times for uptime calculation - Enforces unique paths to prevent duplicates -- **backup_jobs**: Backup job definitions, scheduling metadata, paths, SQL settings, and statuses +- **backup_jobs**: Backup job definitions, scheduling metadata, paths, SQL settings, statuses, and optional tags such as `directory_download` - **backup_runs**: Backup run history, trigger type, status, output logs, errors, and cancellation timestamps - **backup_run_files**: Output archive files created by each backup run +- **directory_download_archives**: Cached generated directory ZIP archives keyed by normalized directory path and recursive fingerprint - **file_download_links**: One-day user-bound file download links, file fingerprints, source context, and per-link download counts - **file_download_events**: Per-request file download audit events with user, link, source context, file fingerprint, IP address, and user agent - **server_view_svr_info_rows**: Raw `SvrInfo.ini` rows for Main, Account, Zone, and Battle servers @@ -722,7 +732,7 @@ The application uses SQLite with the following main tables: 5. **Upload Game Client Data**: Navigate to the Client Data section and upload MON.ull, MC.ull, and IT0.ull through IT3.ull files to populate monster, map, and item databases (requires admin or super admin role). -6. **Navigate Files**: Use the file tree sidebar to browse your server's file system (all authenticated users can view). Pin frequently used directories as shortcuts, right-click files to duplicate them, and download files with admin or super admin access. +6. **Navigate Files**: Use the file tree sidebar to browse your server's file system (all authenticated users can view). Pin frequently used directories as shortcuts, right-click files to duplicate them, and download files or directories with admin or super admin access. Directory downloads compress in the background; keep the page open and do not refresh so the download can start automatically when ready. If you refresh, click the same directory download again to resume polling for the in-progress job. 7. **Edit Files**: Click on editable files (NPC files, quest files, spawn files, drop files (monster drop configurations), item files, item combination data files, or text files) to view and edit them (requires admin or super admin role). - **Quest Files**: Edit quest configurations with type-aware objectives, add/remove controls for optional slots, and binary-safe padding preservation diff --git a/cmd/omnihance-a3-agent/docs/openapi.yml b/cmd/omnihance-a3-agent/docs/openapi.yml index 141aefb..46fc492 100644 --- a/cmd/omnihance-a3-agent/docs/openapi.yml +++ b/cmd/omnihance-a3-agent/docs/openapi.yml @@ -372,6 +372,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized - Authentication is required + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '403': description: Forbidden - Download files permission is required content: @@ -418,6 +424,12 @@ paths: application/json: schema: $ref: '#/components/schemas/DirectoryDownloadResponse' + '401': + description: Unauthorized - Authentication is required + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '403': description: Forbidden - Download files permission is required content: @@ -4621,6 +4633,11 @@ components: description: Number of successful download requests already recorded for this link DirectoryDownloadResponse: type: object + required: + - status + - message + - job_id + - run_id properties: status: type: string diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/README.md b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/README.md index bc1c3a1..88df105 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/README.md +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/README.md @@ -2,7 +2,7 @@ Omnihance A3 Agent frontend -File-browser and backup output downloads use the backend temp-link flow. Admin and super admin users can create one-day user-bound links; viewers see file-browser download actions but receive the denial toast. +File-browser, directory, and backup output downloads use the backend temp-link flow. Admin and super admin users can create one-day user-bound links; viewers see file-browser download actions but receive the denial toast. Directory downloads first compress the selected directory in the background, then poll and auto-start the secure download when the ZIP is ready. ## Adding Shadcn Components diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx index b498f41..df5ffdb 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx @@ -606,15 +606,23 @@ export function FileTree({ initialPath }: FileTreeProps) { }, }); - const { data: directoryDownloadStatus } = useQuery( - { + const { data: directoryDownloadStatus, error: directoryDownloadError } = + useQuery({ queryKey: directoryDownloadRun ? queryKeys.directoryDownload(directoryDownloadRun.runId) : queryKeys.directoryDownload(0), queryFn: () => getDirectoryDownloadStatus(directoryDownloadRun!.runId), enabled: directoryDownloadRun !== null, refetchInterval: (query) => { + if (query.state.error) { + return false; + } + const status = query.state.data?.status; + if (!status) { + return false; + } + if ( status === 'ready' || status === 'failed' || @@ -625,8 +633,7 @@ export function FileTree({ initialPath }: FileTreeProps) { return 2000; }, - }, - ); + }); useEffect(() => { if (!directoryDownloadStatus || directoryDownloadRun === null) { @@ -643,6 +650,28 @@ export function FileTree({ initialPath }: FileTreeProps) { handleDirectoryDownloadResponse, ]); + useEffect(() => { + if (!directoryDownloadError || directoryDownloadRun === null) { + return; + } + + clearDirectoryDownloadToast(); + setDirectoryDownloadRun(null); + + const errorMessage = + directoryDownloadError instanceof APIError + ? directoryDownloadError.getErrorMessage() + : directoryDownloadError instanceof Error + ? directoryDownloadError.message + : 'Failed to get directory download status'; + + toast.error(errorMessage); + }, [ + clearDirectoryDownloadToast, + directoryDownloadError, + directoryDownloadRun, + ]); + const generateSuggestedName = (): string => { if (!currentPath || currentPath === '') { return ''; @@ -889,13 +918,17 @@ export function FileTree({ initialPath }: FileTreeProps) { const existingProcess = isExecutable ? findProcessByPath(fullPath) : undefined; + const isDirectoryDownloadPolling = + item.kind === 'directory' && + directoryDownloadRun?.path === itemPath; const isDownloadPending = (item.kind === 'file' && downloadFileMutation.isPending && downloadFileMutation.variables === itemPath) || (item.kind === 'directory' && - downloadDirectoryMutation.isPending && - downloadDirectoryMutation.variables === itemPath); + ((downloadDirectoryMutation.isPending && + downloadDirectoryMutation.variables === itemPath) || + isDirectoryDownloadPolling)); const isInteractive = item.kind === 'directory' || (item.kind === 'file' && item.is_viewable); diff --git a/internal/services/backup_service.go b/internal/services/backup_service.go index 499f73a..fd2229e 100644 --- a/internal/services/backup_service.go +++ b/internal/services/backup_service.go @@ -449,15 +449,6 @@ func (s *backupService) PrepareDirectoryDownload(ctx context.Context, path strin return nil, err } - runningResult, err := s.runningDirectoryDownloadResult(normalizedPath) - if err != nil { - return nil, err - } - - if runningResult != nil { - return runningResult, nil - } - sourceFingerprint, err := s.buildDirectoryDownloadFingerprint(ctx, normalizedPath) if err != nil { return nil, err @@ -472,6 +463,15 @@ func (s *backupService) PrepareDirectoryDownload(ctx context.Context, path strin return nil, err } + runningResult, err := s.runningDirectoryDownloadResult(normalizedPath) + if err != nil { + return nil, err + } + + if runningResult != nil { + return runningResult, nil + } + s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/services/backup_service_test.go b/internal/services/backup_service_test.go index 294f6d6..eef9da9 100644 --- a/internal/services/backup_service_test.go +++ b/internal/services/backup_service_test.go @@ -77,6 +77,72 @@ func TestPrepareDirectoryDownloadResumesSameRunningJobAndRejectsDifferentDirecto require.ErrorIs(t, err, ErrDirectoryDownloadConflict) } +func TestPrepareDirectoryDownloadReusesCacheWhenDifferentDirectoryIsRunning(t *testing.T) { + service, internalDB := newBackupServiceForTest(t) + sourceDir := writeDirectoryDownloadSource(t, "server", "maps/temoz.map", []byte("map data")) + otherDir := writeDirectoryDownloadSource(t, "other", "maps/quanato.map", []byte("map data")) + normalizedPath := filepath.Clean(sourceDir) + + tag := db.BackupJobTagDirectoryDownload + sourcePath := normalizedPath + job, err := internalDB.CreateBackupJob(db.BackupJobPayload{ + JobType: db.BackupJobTypeFile, + Tag: &tag, + Name: "Directory download: server", + Status: db.BackupJobStatusActive, + DestinationDirectory: service.directoryDownloadsDirectory(), + SourcePath: &sourcePath, + }, nil) + require.NoError(t, err) + + run, err := internalDB.CreateBackupRun(job.ID, db.BackupRunTriggerDirectoryDownload, db.BackupJobStatusActive, nil) + require.NoError(t, err) + + archivePath := filepath.Join(service.directoryDownloadsDirectory(), "server.zip") + require.NoError(t, os.MkdirAll(filepath.Dir(archivePath), 0755)) + require.NoError(t, os.WriteFile(archivePath, []byte("zip"), 0600)) + + file, err := internalDB.CreateBackupRunFile(run.ID, "server", archivePath, 3) + require.NoError(t, err) + + otherSourcePath := filepath.Clean(otherDir) + otherJob, err := internalDB.CreateBackupJob(db.BackupJobPayload{ + JobType: db.BackupJobTypeFile, + Tag: &tag, + Name: "Directory download: other", + Status: db.BackupJobStatusActive, + DestinationDirectory: service.directoryDownloadsDirectory(), + SourcePath: &otherSourcePath, + }, nil) + require.NoError(t, err) + + _, err = internalDB.CreateBackupRun(otherJob.ID, db.BackupRunTriggerDirectoryDownload, db.BackupJobStatusActive, nil) + require.NoError(t, err) + + fingerprint := stableDirectoryDownloadFingerprint(t, service, normalizedPath) + + _, err = internalDB.UpsertDirectoryDownloadArchive(db.DirectoryDownloadArchivePayload{ + NormalizedPath: normalizedPath, + SourceFingerprint: fingerprint, + JobID: job.ID, + RunID: run.ID, + FileID: file.ID, + ArchivePath: archivePath, + ArchiveSize: 3, + }) + require.NoError(t, err) + + archive, err := internalDB.GetDirectoryDownloadArchive(normalizedPath, fingerprint) + require.NoError(t, err) + require.NotNil(t, service.directoryDownloadArchiveResult(archive)) + + result, err := service.PrepareDirectoryDownload(context.Background(), sourceDir, nil) + require.NoError(t, err) + require.Equal(t, DirectoryDownloadStatusReady, result.Status) + require.True(t, result.ArchiveReused) + require.Equal(t, archivePath, result.ArchivePath) +} + func newBackupServiceForTest(t *testing.T) (*backupService, db.InternalDB) { t.Helper() @@ -133,3 +199,22 @@ func waitDirectoryDownloadReady(t *testing.T, service *backupService, runID int6 t.Fatalf("directory download did not finish") return nil } + +func stableDirectoryDownloadFingerprint(t *testing.T, service *backupService, sourcePath string) string { + t.Helper() + + var previous string + for range 10 { + current, err := service.buildDirectoryDownloadFingerprint(context.Background(), sourcePath) + require.NoError(t, err) + + if current == previous { + return current + } + + previous = current + time.Sleep(10 * time.Millisecond) + } + + return previous +}