From 055d7e5f77cd3b2d0944bf2fb45a06f0249eb5b1 Mon Sep 17 00:00:00 2001 From: blankll Date: Fri, 29 May 2026 00:25:11 +0800 Subject: [PATCH 01/16] feat(transfer): implement scope-based architecture for Transfer module Add TransferScope enum (server/database/tables) across all transfer wizards with unified ScopeSelector component and backend scope branching. ## Frontend Changes - Add TransferScope type and update all request types (Export/Import/Migration/Ddl) - Create ScopeSelector.vue horizontal chip toggle component - Update TransferStepCard.vue to render ScopeSelector in header - ExportWizard: scope-based conditional selectors, wire startExport() - ImportWizard: scope state + createDatabaseIfNotExists checkbox - MigrationWizard: scope state + createTargetDatabaseIfNotExists checkbox - GenerateDdl/RunSqlFile: scope state integration - Update task config types (ImportTaskConfig, SqlFileTaskConfig, MigrationTaskConfig) - Remove dead TabbedColumnSelector.vue ## Backend Changes - Add TransferScope enum with serde serialization - export.rs: scope branching (Tables/Database/Server), ZIP creation, Chat2DB naming - import.rs: scope branching with database creation for database scope - migration.rs: scope branching with auto-discovery for server/database scopes - Add current_database field to TransferProgress - Add zip crate dependency ## Design Reference - docs/TRANSFER_SCOPE_DESIGN.md - complete architecture specification --- docs/TRANSFER_SCOPE_DESIGN.md | 797 ++++++++++++++++++ src-tauri/Cargo.lock | 129 ++- src-tauri/Cargo.toml | 1 + src-tauri/src/transfer/export.rs | 750 ++++++++++++++-- src-tauri/src/transfer/import.rs | 239 ++++-- src-tauri/src/transfer/migration.rs | 276 +++++- src-tauri/src/transfer/progress.rs | 1 + src-tauri/src/transfer/types.rs | 160 +++- .../transfer/export/ExportExecuteStep.vue | 7 +- .../transfer/export/ExportPreviewStep.vue | 7 +- .../transfer/export/ExportSourceStep.vue | 21 +- .../transfer/export/ExportWizard.vue | 177 ++-- .../transfer/import/ImportExecuteStep.vue | 4 +- .../transfer/import/ImportMappingStep.vue | 8 +- .../transfer/import/ImportWizard.vue | 75 +- .../transfer/migration/MigrationWizard.vue | 77 +- .../transfer/shared/ScopeSelector.vue | 53 ++ .../transfer/shared/TabbedColumnSelector.vue | 407 --------- .../transfer/shared/TransferStepCard.vue | 11 +- .../transfer/structure/GenerateDdl.vue | 29 +- .../transfer/structure/RunSqlFile.vue | 18 +- src/store/transferStore.ts | 3 +- src/types/transfer.ts | 37 +- 23 files changed, 2596 insertions(+), 691 deletions(-) create mode 100644 docs/TRANSFER_SCOPE_DESIGN.md create mode 100644 src/components/transfer/shared/ScopeSelector.vue delete mode 100644 src/components/transfer/shared/TabbedColumnSelector.vue diff --git a/docs/TRANSFER_SCOPE_DESIGN.md b/docs/TRANSFER_SCOPE_DESIGN.md new file mode 100644 index 00000000..774bb866 --- /dev/null +++ b/docs/TRANSFER_SCOPE_DESIGN.md @@ -0,0 +1,797 @@ +# Transfer Module Scope-Based Architecture Design + +> **Status**: Draft for review +> **Date**: 2026-05-28 +> **Author**: Architecture proposal based on user requirements + +## Executive Summary + +This design introduces a **Scope Selector** pattern across all transfer wizards (Export, Import, Migration, Structure) to provide a unified, simplified experience. The scope determines what level of database objects the operation targets: + +- **Server**: Operate on all databases within a connection +- **Database**: Operate on all tables/objects within a specific database +- **Tables**: Operate on specific selected tables (simplified - no column-level selection) + +--- + +## 1. Type Model Changes + +### 1.1 New Shared Enum + +```typescript +// Frontend: src/types/transfer.ts +export type TransferScope = 'server' | 'database' | 'tables' +``` + +```rust +// Backend: src-tauri/src/transfer/types.rs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub enum TransferScope { + #[default] + Tables, + Database, + Server, +} +``` + +### 1.2 Modified Request Types + +#### ExportRequest + +**Before**: +```typescript +export type ExportRequest = { + connectionId: string + database?: string + schema?: string + source: ExportSource // single table + format: ExportFormat + outputPath: string + // ...options +} +``` + +**After**: +```typescript +export type ExportRequest = { + scope: TransferScope // NEW (default: 'tables') + connectionId: string + database?: string // required for 'database'/'tables' scope + schema?: string + sources: ExportSource[] // CHANGED: array for multi-table + format: ExportFormat + outputPath: string // For 'tables': single file; for 'database/server': directory + // ...options unchanged +} + +export type ExportSource = { + table: string + columns: string[] // When scope='tables': user-selected; else: all columns + // whereClause, orderBy, limit removed for simplicity +} +``` + +#### ImportRequest + +**Before**: +```typescript +export type ImportRequest = { + connectionId: string + database?: string + table: string // single target table + filePath: string + // ... +} +``` + +**After**: +```typescript +export type ImportRequest = { + scope: TransferScope // NEW + connectionId: string + database?: string // required for 'database'/'tables' scope + createDatabaseIfNotExists?: boolean // NEW: for 'database' scope + tables: ImportTarget[] // CHANGED: array for multi-table + filePath: string + // ... +} + +export type ImportTarget = { + sourceTable?: string // From file (for multi-sheet Excel, multi-table SQL) + targetTable: string + columnMappings?: ColumnMapping[] +} +``` + +#### MigrationRequest + +**Before**: +```typescript +export type MigrationRequest = { + sourceConnectionId: string + sourceDatabase?: string + targetConnectionId: string + targetDatabase?: string + tablePlans: MigrationTablePlan[] + // ... +} +``` + +**After**: +```typescript +export type MigrationRequest = { + scope: TransferScope // NEW + sourceConnectionId: string + sourceDatabase?: string // required for 'database'/'tables' scope + targetConnectionId: string + targetDatabase?: string + createTargetDatabaseIfNotExists?: boolean // NEW: for 'database' scope + tablePlans: MigrationTablePlan[] // auto-populated for 'server'/'database' scope + // ... +} +``` + +#### DdlRequest + +**After**: +```typescript +export type DdlRequest = { + scope: TransferScope // NEW + connectionId: string + database?: string + objects: DdlObject[] // auto-populated for 'server'/'database' scope + options: DdlOptions +} +``` + +--- + +## 2. UI Component Changes + +### 2.1 New Component: ScopeSelector + +**Location**: `src/components/transfer/shared/ScopeSelector.vue` + +**Purpose**: Horizontal chip toggle for scope selection + +**Props**: +```typescript +defineProps<{ + scope: TransferScope + disabled?: boolean +}>() + +defineEmits<{ + 'update:scope': [value: TransferScope] +}>() +``` + +**Visual Design**: +``` +┌─────────────────────────────────────────────┐ +│ [Server] [Database] [Tables] │ ← chip buttons +│ gray gray primary │ ← selected styling +└─────────────────────────────────────────────┘ +``` + +**Styling**: +- Horizontal flex container +- Each chip: `px-3 py-1.5 rounded-md text-xs font-medium` +- Selected: `bg-primary/10 text-primary border border-primary/30` +- Unselected: `bg-muted/30 text-muted-foreground hover:bg-muted/50` +- Gap between chips: `gap-1.5` + +### 2.2 Modified Component: TransferStepCard + +**Current header layout** (line 39-48): +``` +[icon] [01] TITLE summary +``` + +**Proposed header layout**: +``` +[icon] [01] TITLE [ScopeSelector] summary +``` + +**New Props**: +```typescript +defineProps<{ + // ... existing props + scope?: TransferScope // NEW + scopeDisabled?: boolean // NEW +}>() + +defineEmits<{ + 'update:scope': [value: TransferScope] // NEW +}>() +``` + +**Header slot integration**: +```vue +
+ + + +
+``` + +--- + +## 3. Wizard Flow Changes + +### 3.1 ExportWizard (Priority 1) + +**Current Steps**: +1. Source (Connection → Database → Schema → Tables → Columns) +2. Format & Output + +**New Steps**: +1. Scope + Source (with scope chips in header) +2. Format & Output + +**Step 1: Scope + Source** + +``` +┌─ Step 1: Source ────────────────────────────────────────┐ +│ [icon] [01] SOURCE [Server][Database][Tables] 3 tables │ +│ │ +│ IF scope === 'server': │ +│ • ConnectionSelector only │ +│ • Badge: "All databases on this server" │ +│ • Summary: Auto-count all tables across all DBs │ +│ │ +│ IF scope === 'database': │ +│ • ConnectionSelector │ +│ • DatabaseSelector (required) │ +│ • Badge: "All tables in {database}" │ +│ • Summary: Auto-count tables in selected DB │ +│ │ +│ IF scope === 'tables': │ +│ • ConnectionSelector │ +│ • DatabaseSelector │ +│ • SchemaSelector (optional) │ +│ • MultiTableSelector (checkbox grid) │ +│ • Summary: "N tables selected" │ +│ │ +│ ❌ REMOVE: TabbedColumnSelector (no column selection) │ +└─────────────────────────────────────────────────────────┘ +``` + +**Step 2: Format & Output** (unchanged structure, but conditional output) + +``` +┌─ Step 2: Format & Output ───────────────────────────────┐ +│ Format selector (CSV/JSONL/Excel/SQL) │ +│ Format-specific options │ +│ │ +│ IF scope === 'tables': │ +│ • Output: Single file path │ +│ │ +│ IF scope === 'database': │ +│ • Output: Directory path │ +│ • Filename pattern: {database}_{table}.{ext} │ +│ │ +│ IF scope === 'server': │ +│ • Output: Directory path │ +│ • Filename pattern: {database}/{table}.{ext} │ +│ • Creates subdirectory per database │ +└─────────────────────────────────────────────────────────┘ +``` + +### 3.2 MigrationWizard (Priority 2) + +**Current Steps**: +1. Source Connection + Table Selection +2. Target Connection + Options +3. Preview & Execute + +**New Steps** (same count, scope in Step 1 header): + +``` +┌─ Step 1: Scope + Source ────────────────────────────────┐ +│ [01] SOURCE [Server][Database][Tables] │ +│ │ +│ IF scope === 'server': │ +│ • Source ConnectionSelector │ +│ • Badge: "All databases will be migrated" │ +│ │ +│ IF scope === 'database': │ +│ • Source ConnectionSelector + Database │ +│ • Badge: "All tables in {database}" │ +│ │ +│ IF scope === 'tables': │ +│ • Current behavior: multi-table checkbox grid │ +└─────────────────────────────────────────────────────────┘ + +┌─ Step 2: Target ────────────────────────────────────────┐ +│ Target ConnectionSelector │ +│ │ +│ IF scope === 'server': │ +│ • No target database selector │ +│ • Option: "Create databases if not exist" │ +│ │ +│ IF scope === 'database': │ +│ • Target database selector │ +│ • Option: "Create database if not exist" ✓ │ +│ │ +│ IF scope === 'tables': │ +│ • Current behavior │ +└─────────────────────────────────────────────────────────┘ + +┌─ Step 3: Preview & Execute ────────────────────────────┐ +│ (unchanged) │ +└─────────────────────────────────────────────────────────┘ +``` + +### 3.3 StructureWizard (Priority 3) + +**GenerateDdl sub-tab**: + +``` +┌─ Step 1: Scope + Source ────────────────────────────────┐ +│ [01] SOURCE [Server][Database][Tables] │ +│ │ +│ IF scope === 'server': │ +│ • ConnectionSelector only │ +│ • Auto-select: all databases, all objects │ +│ │ +│ IF scope === 'database': │ +│ • ConnectionSelector + Database │ +│ • Auto-select: all objects in DB │ +│ │ +│ IF scope === 'tables': │ +│ • Current object checkbox grid │ +└─────────────────────────────────────────────────────────┘ +``` + +**RunSqlFile sub-tab**: + +``` +┌─ Step 1: Scope + Target ────────────────────────────────┐ +│ [01] TARGET [Server][Database][Tables] │ +│ │ +│ IF scope === 'server': │ +│ • ConnectionSelector only │ +│ • SQL can CREATE DATABASE │ +│ │ +│ IF scope === 'database': │ +│ • ConnectionSelector + Database │ +│ • Option: "Create database if not exist" │ +│ │ +│ IF scope === 'tables': │ +│ • ConnectionSelector + Database │ +│ • SQL targets specific tables │ +└─────────────────────────────────────────────────────────┘ +``` + +### 3.4 ImportWizard (Priority 4) + +**Current Steps**: +1. Source File +2. Target & Mapping +3. Options & Execute + +**New Steps** (scope in Step 2 header): + +``` +┌─ Step 1: Source File ───────────────────────────────────┐ +│ (unchanged - file drop, detection, preview) │ +└─────────────────────────────────────────────────────────┘ + +┌─ Step 2: Scope + Target ────────────────────────────────┐ +│ [02] TARGET [Server][Database][Tables] │ +│ │ +│ IF scope === 'server': │ +│ • ConnectionSelector only │ +│ • SQL file can CREATE DATABASE │ +│ • No table mapping UI │ +│ │ +│ IF scope === 'database': │ +│ • ConnectionSelector + Database │ +│ • Checkbox: "Create database if not exists" ✓ │ +│ • Auto-create tables from file structure │ +│ • No manual column mapping │ +│ │ +│ IF scope === 'tables': │ +│ • Current behavior: single table + column mapping │ +└─────────────────────────────────────────────────────────┘ + +┌─ Step 3: Options & Execute ────────────────────────────┐ +│ (unchanged) │ +└─────────────────────────────────────────────────────────┘ +``` + +--- + +## 4. Backend Command Changes + +### 4.1 execute_export_data + +**Current signature**: +```rust +pub async fn execute_export_data( + request: ExportRequest, + app_state: State<'_, AppState>, + app_handle: AppHandle, +) -> Result +``` + +**Changes needed**: + +1. **Handle `scope` field**: +```rust +match request.scope { + TransferScope::Server => { + // 1. List all databases + let databases = adapter.list_databases()?; + // 2. For each database, list tables + // 3. Export each table to: {output_path}/{database}/{table}.{ext} + } + TransferScope::Database => { + // 1. List all tables in request.database + // 2. Export each table to: {output_path}/{database}_{table}.{ext} + } + TransferScope::Tables => { + // Current behavior: iterate request.sources + // Export each table to: {output_path} + } +} +``` + +2. **Multi-table support**: +```rust +// Change from single source to sources array +for source in request.sources.iter() { + export_table(adapter, source, &request.format, output_path)?; +} +``` + +3. **Progress events**: Include current database/table in progress for server/database scope + +### 4.2 execute_import_data + +**Changes needed**: + +1. **Handle `scope` field**: +```rust +match request.scope { + TransferScope::Server => { + // Execute SQL file directly at connection level + // File may contain CREATE DATABASE statements + } + TransferScope::Database => { + // 1. Check if database exists, create if request.create_database_if_not_exists + // 2. Auto-create tables from file structure (for CSV/Excel) + // 3. Import data + } + TransferScope::Tables => { + // Current behavior + } +} +``` + +### 4.3 execute_migration_data + +**Changes needed**: + +1. **Handle `scope` field**: +```rust +match request.scope { + TransferScope::Server => { + // Migrate all databases from source to target + // Auto-create target databases + } + TransferScope::Database => { + // 1. Check/create target database if request.create_target_database_if_not_exists + // 2. Migrate all tables + } + TransferScope::Tables => { + // Current behavior + } +} +``` + +### 4.4 generate_ddl_for_objects + +**Changes needed**: + +1. **Handle `scope` field**: +```rust +match request.scope { + TransferScope::Server => { + // Generate DDL for all databases, all objects + // Output: multiple DDL files per database + } + TransferScope::Database => { + // Generate DDL for all objects in database + } + TransferScope::Tables => { + // Current behavior + } +} +``` + +--- + +## 5. Visual Wireframe + +### ExportWizard with Scope Selector + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ TRANSFER [Task Manager] │ +│ Data import, export, and migration │ +├─────────────────────────────────────────────────────────────────────┤ +│ [Export] [Import] [Migration] [Structure] │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─ Step 1: Source ───────────────────────────────────────────────┐ │ +│ │ [📊] [01] SOURCE [Server][Database][Tables] 3 tables │ │ +│ │ │ │ +│ │ ┌────────────────────┐ ┌──────────────────────────────────┐ │ │ +│ │ │ Connection │ │ Tables │ │ │ +│ │ │ ┌────────────────┐ │ │ ┌──────────────────────────────┐ │ │ │ +│ │ │ │ localhost:5432 │ │ │ │ ☑ users 1,234 rows │ │ │ │ +│ │ │ └────────────────┘ │ │ │ ☐ products 5,678 rows │ │ │ │ +│ │ │ │ │ │ │ ☐ orders 2,345 rows │ │ │ │ +│ │ │ Database │ │ │ │ ... │ │ │ │ +│ │ │ ┌────────────────┐ │ │ │ └─────────────────────────── │ │ │ │ +│ │ │ │ mydb │ │ │ │ [Select All] [Deselect All] │ │ │ │ +│ │ │ └────────────────┘ │ │ └──────────────────────────────┘ │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ Schema (optional) │ │ │ │ │ +│ │ │ ┌────────────────┐ │ │ │ │ │ +│ │ │ │ public │ │ │ │ │ │ +│ │ │ └────────────────┘ │ │ │ │ │ +│ │ └────────────────────┘ │ │ │ │ +│ │ └────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ └────────────────────────────────────────────────────────────────│ │ +│ │ +│ ┌─ Step 2: Format & Output ──────────────────────────────────────┐ │ +│ │ [📄] [02] FORMAT & OUTPUT │ │ +│ │ │ │ +│ │ Format: │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │[CSV] │ │ JSONL │ │ Excel │ │ SQL │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ +│ │ │ │ +│ │ CSV Options: │ │ +│ │ Delimiter: [Comma (,) ▼] ☑ Include header row │ │ +│ │ │ │ +│ │ Output Path: │ │ +│ │ [/path/to/output.csv ] [Browse] │ │ +│ │ │ │ +│ │ ──────────────────────────────────────────────────────────── │ │ +│ │ [▶ Start Export] │ │ +│ │ │ │ +│ │ Summary: 3 tables | 4 cols | CSV │ │ +│ └────────────────────────────────────────────────────────────────│ │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 6. Implementation Checklist + +### Phase 1: Types & Components (Frontend + Backend) + +**Frontend**: +1. Add `TransferScope` to `src/types/transfer.ts` +2. Update `ExportRequest` (add `scope`, change `source` → `sources: ExportSource[]`) +3. Update `ImportRequest` (add `scope`, `createDatabaseIfNotExists`) +4. Update `MigrationRequest` (add `scope`, `createTargetDatabaseIfNotExists`) +5. Update `DdlRequest` (add `scope`) + +**Backend**: +1. Add `TransferScope` to `src-tauri/src/transfer/types.rs` +2. Update corresponding Rust request structs +3. Add timestamp generation helper function + +**Shared Components**: +1. Create `ScopeSelector.vue` component (horizontal chip toggle) +2. Modify `TransferStepCard.vue` to accept and render scope selector in header + +### Phase 2: ExportWizard (Frontend) + +1. Add scope state (default: `'tables'`) +2. Pass scope to TransferStepCard Step 1 header +3. Conditionally show/hide selectors based on scope: + - `server`: ConnectionSelector only, show summary badge + - `database`: ConnectionSelector + DatabaseSelector, show summary badge + - `tables`: ConnectionSelector + DatabaseSelector + MultiTableSelector +4. Remove `TabbedColumnSelector` usage +5. Change `source` to `sources` array in store sync +6. Wire `startExport()`: + - For `tables` scope: direct invoke with single/multiple sources + - For `database`/`server` scope: async task creation, poll for status +7. Handle output path: + - `tables` (single): file picker for `.csv/.sql/.xlsx` + - `tables` (multi): file picker for `.zip` + - `database`/`server`: directory picker (create ZIP inside) + +### Phase 3: Export Backend + +1. Update `execute_export_data` command signature +2. Add timestamp generator: `format_datetime(chrono::Local::now())` +3. Implement scope-based iteration: + - `server`: List databases → for each DB → list tables → export to ZIP nested + - `database`: List tables → export each to ZIP + - `tables`: Direct multi-table export to ZIP or single file +4. ZIP creation logic (use `zip` crate): + - Server scope: nested entries `{database}/{table}.{ext}` + - Database scope: flat entries `{table}.{ext}` +5. Progress events: Include `current_database`, `current_table`, `total_tables` +6. Async task pattern: + - For `database`/`server` scope: Return task ID immediately + - Background thread processes export + - Client polls `/task_status/{taskId}` (reuse existing BackgroundTask system) + +### Phase 4: MigrationWizard + Backend + +1. Add scope state to MigrationWizard (default: `'tables'`) +2. Conditionally show selectors: + - `server`: Source + Target ConnectionSelector, no DB selector + - `database`: ConnectionSelector + DatabaseSelector for both source/target + - `tables`: Current behavior +3. Add `createTargetDatabaseIfNotExists` checkbox for `database` scope +4. Update `execute_migration_data`: + - For `server` scope: Iterate all databases + - For `database` scope: Single DB migration (all tables or selected) +5. Handle target database creation + +### Phase 5: StructureWizard + Backend + +1. Add scope to GenerateDdl: + - `server`: Export DDL for all databases (ZIP output) + - `database`: Export DDL for all objects (single SQL or ZIP) + - `tables`: Current behavior (selected objects) +2. Add scope to RunSqlFile: + - `server`: SQL can CREATE DATABASE + - `database`: Add `createDatabaseIfNotExists` checkbox + - `tables`: Current behavior +3. Update backend commands accordingly + +### Phase 6: ImportWizard + Backend + +1. Add scope state to ImportWizard (default: `'tables'`) +2. Scope affects Step 2 (Target & Mapping): + - `server`: ConnectionSelector only, SQL file can CREATE DATABASE + - `database`: ConnectionSelector + DatabaseSelector, add `createDatabaseIfNotExists` checkbox, auto-create table from file + - `tables`: Current behavior (select target table + column mapping) +3. For `database` scope: Single file imports to ONE table (match Chat2DB) +4. Update `execute_import_data` for scope handling + +--- + +## 7. Design Decisions (Based on Chat2DB Research) + +> Reference: Chat2DB GitHub - https://github.com/codePhiliaX/Chat2DB + +### 7.1 Output Naming Convention (Adopt Chat2DB's Pattern) + +| Scope | Single Table | Multiple Tables | +|-------|--------------|-----------------| +| **Tables** | `{tableName}_{timestamp}.{ext}` | `export_{tables}_data_{timestamp}.zip` → `{tableName}.{ext}` inside | +| **Database** | — | `export_{databaseName}_data_{timestamp}.zip` → `{tableName}.{ext}` inside | +| **Server** | — | `export_{connectionName}_data_{timestamp}.zip` → `{database}/{tableName}.{ext}` nested | + +**Timestamp format**: `YYYYMMDDHHmmss` (pure datetime, matches Chat2DB) + +**Examples**: +``` +# Tables scope (single) +users_20240324153045.csv + +# Tables scope (multiple: users, orders) +export_users_orders_data_20240324153045.zip + → users.csv + → orders.csv + +# Database scope (mydb) +export_mydb_data_20240324153045.zip + → users.csv + → orders.csv + → products.csv + +# Server scope (localhost_5432) +export_localhost_5432_data_20240324153045.zip + → mydb/ + → users.csv + → orders.csv + → testdb/ + → test_table.csv +``` + +### 7.2 Import Database Scope (Match Chat2DB's Approach) + +**Decision**: Single file → creates ONE target table (same as Chat2DB) + +- For CSV/JSONL/Excel: User selects target database, file imports to one table +- For SQL files: Backend can auto-detect multiple CREATE TABLE statements and create accordingly +- The "create database if not exists" checkbox applies to the target database selection, not file parsing + +**Why**: Chat2DB has no multi-table import. This simplifies UX and aligns with common patterns. + +### 7.3 Scope Default + +**Decision**: `tables` scope for all wizards (default) + +- Safest (current behavior) +- Most common use case +- Matches Chat2DB's single-table focus + +### 7.4 Scope Persistence + +**Decision**: No persistence needed + +- Scope is per-operation (not remembered) +- Each wizard starts with `tables` scope +- Simpler implementation +- Matches Chat2DB's implicit scope approach (no state) + +### 7.5 Async Task Pattern for Bulk Operations + +**Decision**: Adopt async task pattern for database/server scope (like Chat2DB) + +- Database/server scope exports: Return task ID, poll for status +- Tables scope: Direct execution (smaller scope, faster) +- Use existing `BackgroundTask` system in `transferStore` + +### 7.6 Scope UI Approach (Keep Explicit Picker) + +**Decision**: Keep explicit scope selector (unlike Chat2DB's implicit approach) + +**Why SQLKit differs from Chat2DB**: +- SQLKit has a dedicated Transfer page (not tree-context driven) +- Better discoverability for users unfamiliar with database hierarchy +- Consistent experience across Export/Import/Migration/Structure tabs +- Chat2DB's approach works for tree-based UI; SQLKit's wizard-based UI needs explicit selection + +--- + +## 8. Acceptance Criteria + +- [ ] Scope selector appears in header of Step 1 for all wizards +- [ ] Scope selector uses chip toggle styling (horizontal, 3 options: Server/Database/Tables) +- [ ] `tables` scope is default for all wizards +- [ ] Scope is not persisted (resets to `tables` on wizard open) +- [ ] Selectors conditionally render based on selected scope +- [ ] `tables` scope behaves like current behavior (minus column selection for Export) +- [ ] `database` scope auto-selects all tables, shows count summary badge +- [ ] `server` scope auto-selects all databases, shows count summary badge +- [ ] **Output naming follows Chat2DB pattern**: + - [ ] Single table: `{tableName}_{timestamp}.{ext}` + - [ ] Multiple tables: ZIP with `{tableName}.{ext}` inside + - [ ] Database scope: ZIP with `{tableName}.{ext}` inside + - [ ] Server scope: ZIP nested with `{database}/{tableName}.{ext}` +- [ ] Backend handles all 3 scope levels correctly +- [ ] **Async task pattern** for database/server scope exports: + - [ ] Returns task ID immediately + - [ ] Background thread processes export + - [ ] Client can poll for status via BackgroundTask system +- [ ] Progress events include `current_database`, `current_table`, `total_tables` for bulk scopes +- [ ] Import `database` scope: Single file imports to one table, with `createDatabaseIfNotExists` option +- [ ] Migration `database` scope: Has `createTargetDatabaseIfNotExists` option +- [ ] Structure `database` scope: Auto-selects all objects for DDL generation + +--- + +## Next Steps + +Please review and provide feedback on: +1. Type model changes (section 1) +2. UI wireframe (section 5) +3. Open questions (section 7) +4. Any additional requirements or concerns + +Once approved, implementation will proceed in priority order: +Export → Migration → Structure → Import \ No newline at end of file diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 0e8e1344..6195b21d 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.7.8" @@ -282,6 +293,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -498,6 +515,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cairo-rs" version = "0.18.5" @@ -665,6 +702,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -751,6 +798,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "convert_case" version = "0.4.0" @@ -1151,6 +1204,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -2093,6 +2147,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -2412,6 +2475,15 @@ dependencies = [ "cfb", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -2961,7 +3033,7 @@ dependencies = [ "thiserror 1.0.69", "time", "uuid", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -3506,12 +3578,35 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "pathdiff" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", + "password-hash", + "sha2 0.10.9", +] + [[package]] name = "pem" version = "3.0.6" @@ -3832,7 +3927,7 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", + "hmac 0.13.0", "md-5", "memchr", "rand 0.10.0", @@ -5092,6 +5187,7 @@ dependencies = [ "tokio-util", "url", "uuid", + "zip 0.6.6", ] [[package]] @@ -7533,10 +7629,18 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ + "aes", "byteorder", + "bzip2", + "constant_time_eq", "crc32fast", "crossbeam-utils", "flate2", + "hmac 0.12.1", + "pbkdf2", + "sha1", + "time", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -7557,13 +7661,32 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.2.4", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3881abfb..c158feb7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -62,5 +62,6 @@ base64 = "0.22" chrono = "0.4" rust_decimal = { version = "1", features = [ "db-postgres" ] } hex = "0.4" +zip = "0.6" rust_xlsxwriter = "0.64" calamine = "0.22" diff --git a/src-tauri/src/transfer/export.rs b/src-tauri/src/transfer/export.rs index d5615894..b81eccca 100644 --- a/src-tauri/src/transfer/export.rs +++ b/src-tauri/src/transfer/export.rs @@ -1,4 +1,5 @@ //! Export implementation for CSV, JSONL, SQL, and Excel formats. +//! Supports scope-based export with ZIP creation for Server, Database, and Tables scopes. use std::fs::File; use std::io::{BufWriter, Write}; @@ -7,23 +8,188 @@ use std::time::Instant; use rust_xlsxwriter::{Workbook, Worksheet}; use serde_json::Value as JsonValue; +use zip::write::FileOptions; +use zip::ZipWriter; use super::defaults::*; use super::progress::*; use super::types::*; use crate::database::{DatabaseAdapter, QueryValue}; +const BATCH_SIZE: u64 = 1000; + +// ── Chat2DB-style naming ───────────────────────────────────────── + +/// Format a table filename with Chat2DB-style timestamp suffix. +fn format_chat2db_filename(table: &str, ext: &str) -> String { + let timestamp = chrono::Local::now().format("%Y%m%d%H%M%S"); + format!("{}_{}.{}", table, timestamp, ext) +} + +/// Get the file extension for an export format. +fn format_extension(format: &ExportFormat) -> &'static str { + match format { + ExportFormat::Csv => "csv", + ExportFormat::Jsonl => "jsonl", + ExportFormat::Sql => "sql", + ExportFormat::Excel => "xlsx", + } +} + +// ── Scope-based export entry point ─────────────────────────────── + /// Executes a data export operation. +/// +/// Supports three scopes: +/// - `Tables`: Export specified sources. Single source → single file, multiple → ZIP. +/// - `Database`: List all tables in the database, export each to ZIP. +/// - `Server`: List all databases + tables, export each with nested paths to ZIP. pub async fn execute_export( adapter: &A, request: ExportRequest, app_handle: &tauri::AppHandle, ) -> Result { let start_time = Instant::now(); - let _operation_id = uuid::Uuid::new_v4().to_string(); - let columns = request.source.columns.clone(); - let table = request.source.table.clone(); + match &request.scope { + TransferScope::Tables if request.sources.len() <= 1 => { + execute_single_table_export(adapter, request, app_handle, start_time).await + } + TransferScope::Tables => { + // Multiple sources → ZIP with flat paths + let sources_with_paths: Vec<(Option, ExportSource)> = request + .sources + .iter() + .map(|s| (None, s.clone())) + .collect(); + execute_zip_export( + adapter, + request, + sources_with_paths, + app_handle, + start_time, + ) + .await + } + TransferScope::Database => { + let db_name = request + .database + .clone() + .ok_or_else(|| "database name is required for Database scope".to_string())?; + + let mut progress = create_progress( + "export", + "discovering", + 0, + None, + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = Some(db_name.clone()); + emit_progress(app_handle, &progress); + + let tables = adapter + .list_tables(Some(&db_name), request.schema.as_deref()) + .await + .map_err(|e| format!("Failed to list tables in '{}': {}", db_name, e))?; + + let mut sources: Vec = Vec::with_capacity(tables.len()); + for table_info in &tables { + let columns = adapter + .list_columns(Some(&db_name), request.schema.as_deref(), &table_info.name) + .await + .map_err(|e| format!("Failed to list columns for '{}': {}", table_info.name, e))?; + sources.push(ExportSource { + table: table_info.name.clone(), + columns: columns.iter().map(|c| c.name.clone()).collect(), + }); + } + + // Database scope uses flat paths ({table}.{ext}) in ZIP, + // but we still pass the db name for progress tracking. + // The execute_zip_export uses the Option for path nesting, + // so we pass None here for flat entries. + let sources_flat: Vec = sources; + let sources_with_paths: Vec<(Option, ExportSource)> = + sources_flat.into_iter().map(|s| (None, s)).collect(); + + execute_zip_export(adapter, request, sources_with_paths, app_handle, start_time).await + } + TransferScope::Server => { + emit_progress( + app_handle, + &create_progress( + "export", + "discovering", + 0, + None, + start_time.elapsed().as_millis() as u64, + ), + ); + + let databases = adapter + .list_databases() + .await + .map_err(|e| format!("Failed to list databases: {}", e))?; + + let mut sources_with_paths: Vec<(Option, ExportSource)> = Vec::new(); + + for (db_idx, db) in databases.iter().enumerate() { + let mut progress = create_progress( + "export", + "discovering", + db_idx as u64, + Some(databases.len() as u64), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = Some(db.name.clone()); + emit_progress(app_handle, &progress); + + let tables = adapter + .list_tables(Some(&db.name), request.schema.as_deref()) + .await + .map_err(|e| format!("Failed to list tables in '{}': {}", db.name, e))?; + + for table_info in &tables { + let columns = adapter + .list_columns(Some(&db.name), request.schema.as_deref(), &table_info.name) + .await + .map_err(|e| { + format!( + "Failed to list columns for '{}' in '{}': {}", + table_info.name, db.name, e + ) + })?; + sources_with_paths.push(( + Some(db.name.clone()), + ExportSource { + table: table_info.name.clone(), + columns: columns.iter().map(|c| c.name.clone()).collect(), + }, + )); + } + } + + execute_zip_export(adapter, request, sources_with_paths, app_handle, start_time).await + } + } +} + +// ── Single-table export (original behavior) ───────────────────── + +/// Execute export for a single table to a single file. +async fn execute_single_table_export( + adapter: &A, + request: ExportRequest, + app_handle: &tauri::AppHandle, + start_time: Instant, +) -> Result { + let source = request + .sources + .first() + .ok_or("No export sources specified")? + .clone(); + let columns = source.columns.clone(); + let table = source.table.clone(); let schema = request.schema.clone(); let csv_opts = request @@ -43,9 +209,9 @@ pub async fn execute_export( .clone() .unwrap_or_else(excel_export_defaults); - let base_query = build_export_query(&schema, &table, &columns, &request.source); + let base_query = build_export_query(&schema, &table, &columns); - let count_query = build_count_query(&schema, &table, &request.source.where_clause); + let count_query = build_count_query(&schema, &table); let count_result = adapter .execute_query(&count_query) .await @@ -71,7 +237,6 @@ pub async fn execute_export( let mut processed_rows: u64 = 0; let mut errors: Vec = Vec::new(); - let batch_size = 1000u64; match request.format { ExportFormat::Csv => { @@ -82,7 +247,8 @@ pub async fn execute_export( let mut offset = 0u64; while offset < total_rows { - let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); let result = adapter .execute_query(&query) .await @@ -101,7 +267,7 @@ pub async fn execute_export( processed_rows += 1; } - offset += batch_size; + offset += BATCH_SIZE; emit_progress( app_handle, &create_progress( @@ -118,7 +284,8 @@ pub async fn execute_export( ExportFormat::Jsonl => { let mut offset = 0u64; while offset < total_rows { - let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); let result = adapter .execute_query(&query) .await @@ -126,7 +293,8 @@ pub async fn execute_export( for row in &result.rows { let json_obj = row_to_json_object(row, &jsonl_opts.date_format); - let json_line = serde_json::to_string(&json_obj).map_err(|e| e.to_string())?; + let json_line = + serde_json::to_string(&json_obj).map_err(|e| e.to_string())?; writer .write_all(json_line.as_bytes()) .map_err(|e| e.to_string())?; @@ -134,7 +302,7 @@ pub async fn execute_export( processed_rows += 1; } - offset += batch_size; + offset += BATCH_SIZE; emit_progress( app_handle, &create_progress( @@ -166,7 +334,8 @@ pub async fn execute_export( let mut offset = 0u64; while offset < total_rows { - let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); let result = adapter .execute_query(&query) .await @@ -191,7 +360,7 @@ pub async fn execute_export( } } - offset += batch_size; + offset += BATCH_SIZE; emit_progress( app_handle, &create_progress( @@ -239,7 +408,8 @@ pub async fn execute_export( let mut row_idx = header_row_offset; while offset < total_rows { - let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); let result = adapter .execute_query(&query) .await @@ -254,7 +424,7 @@ pub async fn execute_export( processed_rows += 1; } - offset += batch_size; + offset += BATCH_SIZE; emit_progress( app_handle, &create_progress( @@ -312,12 +482,410 @@ pub async fn execute_export( }) } -fn build_export_query( - schema: &Option, - table: &str, - columns: &[String], +// ── ZIP-based multi-table export ───────────────────────────────── + +/// Execute export for multiple tables into a single ZIP file. +/// +/// Each source is accompanied by an optional database name. +/// When `Some(db)`, the ZIP entry path is `{db}/{chat2db_name}`. +/// When `None`, the entry path is `{chat2db_name}` (flat). +async fn execute_zip_export( + adapter: &A, + request: ExportRequest, + sources_with_db: Vec<(Option, ExportSource)>, + app_handle: &tauri::AppHandle, + start_time: Instant, +) -> Result { + let csv_opts = request + .csv_options + .clone() + .unwrap_or_else(csv_export_defaults); + let jsonl_opts = request + .jsonl_options + .clone() + .unwrap_or_else(jsonl_export_defaults); + let excel_opts = request + .excel_options + .clone() + .unwrap_or_else(excel_export_defaults); + let ext = format_extension(&request.format); + let schema = request.schema.as_deref(); + + let output_path = Path::new(&request.output_path); + let file = File::create(output_path).map_err(|e| format!("Failed to create ZIP file: {}", e))?; + let mut zip = ZipWriter::new(file); + + let mut grand_total: u64 = 0; + let mut grand_processed: u64 = 0; + let mut errors: Vec = Vec::new(); + + let total_sources = sources_with_db.len() as u64; + + for (source_idx, (db_name_opt, source)) in sources_with_db.iter().enumerate() { + let table = &source.table; + + // Emit discovering progress for this table + { + let mut progress = create_progress( + "export", + if total_sources > 0 && source_idx == 0 { + "discovering" + } else { + "processing" + }, + source_idx as u64, + Some(total_sources), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = db_name_opt.clone(); + progress.current_table = Some(table.clone()); + progress.message = Some(format!("Exporting table: {}", table)); + emit_progress(app_handle, &progress); + } + + // Determine ZIP entry path + let chat2db_name = format_chat2db_filename(table, ext); + let zip_path = match db_name_opt { + Some(db) => format!("{}/{}", db, chat2db_name), + None => chat2db_name, + }; + + // Export this table to a byte buffer + // Build the SqlExportOptions for each table (some options reference the table name) + let sql_opts = request + .sql_options + .clone() + .unwrap_or_else(|| sql_export_defaults(table)); + + let data = export_table_to_bytes( + adapter, + source, + schema, + db_name_opt.as_deref(), + &request.format, + &csv_opts, + &jsonl_opts, + &sql_opts, + &excel_opts, + app_handle, + &mut grand_total, + &mut grand_processed, + &mut errors, + start_time, + ) + .await?; + + // Add entry to ZIP + let options = FileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + zip.start_file(&zip_path, options) + .map_err(|e| format!("ZIP error: {}", e))?; + zip.write_all(&data) + .map_err(|e| format!("ZIP write error: {}", e))?; + } + + // Finalize ZIP + let _zip_output = zip + .finish() + .map_err(|e| format!("ZIP finalize error: {}", e))?; + + let file_size = std::fs::metadata(output_path).map(|m| m.len()).unwrap_or(0); + + emit_progress( + app_handle, + &create_progress( + "export", + "finalizing", + grand_processed, + Some(grand_total), + start_time.elapsed().as_millis() as u64, + ), + ); + + Ok(TransferResult { + success: errors.is_empty(), + total_rows: grand_total, + processed_rows: grand_processed, + skipped_rows: 0, + error_count: errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: Some(request.output_path), + output_size_bytes: Some(file_size), + errors, + }) +} + +/// Export a single table's data to a byte vector, suitable for ZIP inclusion. +#[allow(clippy::too_many_arguments)] +async fn export_table_to_bytes( + adapter: &A, source: &ExportSource, -) -> String { + schema: Option<&str>, + database: Option<&str>, + format: &ExportFormat, + csv_opts: &CsvExportOptions, + jsonl_opts: &JsonlExportOptions, + sql_opts: &SqlExportOptions, + excel_opts: &ExcelExportOptions, + app_handle: &tauri::AppHandle, + accumulated_total: &mut u64, + accumulated_processed: &mut u64, + errors: &mut Vec, + start_time: Instant, +) -> Result, String> { + let columns = &source.columns; + let table = &source.table; + + let base_query = build_export_query(&schema.map(|s| s.to_string()), table, columns); + let count_query = build_count_query(&schema.map(|s| s.to_string()), table); + + let count_result = adapter + .execute_query(&count_query) + .await + .map_err(|e| e.to_string())?; + let total_rows = count_result + .rows + .first() + .and_then(|row| row.get("count")) + .and_then(|v| match v { + QueryValue::Int(n) => Some(*n as u64), + _ => None, + }) + .unwrap_or(0); + *accumulated_total += total_rows; + + let mut local_processed: u64 = 0; + + match format { + ExportFormat::Csv => { + let mut buffer = Vec::new(); + + if csv_opts.include_header { + let mut buf = BufWriter::new(&mut buffer); + write_csv_header(&mut buf, columns, csv_opts.delimiter) + .map_err(|e| e.to_string())?; + buf.flush().map_err(|e| e.to_string())?; + } + + let mut offset = 0u64; + while offset < total_rows { + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); + let result = adapter + .execute_query(&query) + .await + .map_err(|e| e.to_string())?; + + for row in &result.rows { + let mut buf = BufWriter::new(&mut buffer); + write_csv_row(&mut buf, columns, row, csv_opts).map_err(|e| { + errors.push(TransferError { + row_number: Some(*accumulated_processed + local_processed + 1), + statement_number: None, + message: e, + sql: None, + }); + String::new() + })?; + buf.flush().map_err(|e| e.to_string())?; + local_processed += 1; + } + + offset += BATCH_SIZE; + let mut progress = create_progress( + "export", + "processing", + *accumulated_processed + local_processed, + Some(*accumulated_total), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = database.map(|s| s.to_string()); + progress.current_table = Some(table.clone()); + emit_progress(app_handle, &progress); + } + + *accumulated_processed += local_processed; + Ok(buffer) + } + + ExportFormat::Jsonl => { + let mut buffer = Vec::new(); + + let mut offset = 0u64; + while offset < total_rows { + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); + let result = adapter + .execute_query(&query) + .await + .map_err(|e| e.to_string())?; + + for row in &result.rows { + let json_obj = row_to_json_object(row, &jsonl_opts.date_format); + let json_line = + serde_json::to_string(&json_obj).map_err(|e| e.to_string())?; + buffer.extend_from_slice(json_line.as_bytes()); + buffer.push(b'\n'); + local_processed += 1; + } + + offset += BATCH_SIZE; + let mut progress = create_progress( + "export", + "processing", + *accumulated_processed + local_processed, + Some(*accumulated_total), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = database.map(|s| s.to_string()); + progress.current_table = Some(table.clone()); + emit_progress(app_handle, &progress); + } + + *accumulated_processed += local_processed; + Ok(buffer) + } + + ExportFormat::Sql => { + let mut buffer = Vec::new(); + let schema_ref = schema.map(|s| s.to_string()); + + if sql_opts.include_create_table { + let table_info = adapter + .get_table_info(database, schema, table) + .await + .map_err(|e| e.to_string())?; + let create_stmt = + generate_create_table_sql(table, &table_info, sql_opts.include_drop_table); + buffer.extend_from_slice(create_stmt.as_bytes()); + buffer.extend_from_slice(b"\n\n"); + } + + let mut batch_rows: Vec> = Vec::new(); + let mut offset = 0u64; + + while offset < total_rows { + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); + let result = adapter + .execute_query(&query) + .await + .map_err(|e| e.to_string())?; + + for row in &result.rows { + let values: Vec = columns + .iter() + .map(|col| row.get(col).cloned().unwrap_or(QueryValue::Null)) + .collect(); + batch_rows.push(values); + local_processed += 1; + + if batch_rows.len() >= sql_opts.batch_size as usize { + let insert_stmt = + generate_insert_sql(&schema_ref, table, columns, &batch_rows); + buffer.extend_from_slice(insert_stmt.as_bytes()); + buffer.push(b'\n'); + batch_rows.clear(); + } + } + + offset += BATCH_SIZE; + let mut progress = create_progress( + "export", + "processing", + *accumulated_processed + local_processed, + Some(*accumulated_total), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = database.map(|s| s.to_string()); + progress.current_table = Some(table.clone()); + emit_progress(app_handle, &progress); + } + + if !batch_rows.is_empty() { + let insert_stmt = generate_insert_sql(&schema_ref, table, columns, &batch_rows); + buffer.extend_from_slice(insert_stmt.as_bytes()); + } + + *accumulated_processed += local_processed; + Ok(buffer) + } + + ExportFormat::Excel => { + let mut workbook = Workbook::new(); + let worksheet = workbook + .add_worksheet() + .set_name(&excel_opts.sheet_name) + .map_err(|e| e.to_string())?; + + if excel_opts.include_header { + for (col_idx, col_name) in columns.iter().enumerate() { + worksheet + .write_string(0, col_idx as u16, col_name) + .map_err(|e| e.to_string())?; + } + } + + if excel_opts.freeze_header && excel_opts.include_header { + worksheet + .set_freeze_panes(1, 0) + .map_err(|e| e.to_string())?; + } + + let header_row_offset = if excel_opts.include_header { 1 } else { 0 }; + let mut offset = 0u64; + let mut row_idx = header_row_offset; + + while offset < total_rows { + let query = + format!("{} LIMIT {} OFFSET {}", base_query, BATCH_SIZE, offset); + let result = adapter + .execute_query(&query) + .await + .map_err(|e| e.to_string())?; + + for row in &result.rows { + for (col_idx, col_name) in columns.iter().enumerate() { + let value = row.get(col_name).cloned().unwrap_or(QueryValue::Null); + write_excel_cell(worksheet, row_idx, col_idx as u16, &value)?; + } + row_idx += 1; + local_processed += 1; + } + + offset += BATCH_SIZE; + let mut progress = create_progress( + "export", + "processing", + *accumulated_processed + local_processed, + Some(*accumulated_total), + start_time.elapsed().as_millis() as u64, + ); + progress.current_database = database.map(|s| s.to_string()); + progress.current_table = Some(table.clone()); + emit_progress(app_handle, &progress); + } + + if excel_opts.auto_fit_columns { + let max_col = columns.len() as u16; + for col_idx in 0..max_col { + worksheet + .set_column_width(col_idx, 12.0) + .map_err(|e| e.to_string())?; + } + } + + *accumulated_processed += local_processed; + workbook + .save_to_buffer() + .map_err(|e| format!("Failed to save Excel to buffer: {}", e)) + } + } +} + +// ── Query builders ─────────────────────────────────────────────── + +fn build_export_query(schema: &Option, table: &str, columns: &[String]) -> String { let schema_prefix = schema .as_ref() .map(|s| format!("\"{}\".", s)) @@ -328,42 +896,24 @@ fn build_export_query( .collect::>() .join(", "); - let mut query = format!("SELECT {} FROM {}\"{}\"", cols, schema_prefix, table); - - if let Some(ref where_clause) = source.where_clause { - query.push_str(&format!(" WHERE {}", where_clause)); - } - - if let Some(ref order_by) = source.order_by { - query.push_str(&format!(" ORDER BY {}", order_by)); - } - - query + format!("SELECT {} FROM {}\"{}\"", cols, schema_prefix, table) } -fn build_count_query( - schema: &Option, - table: &str, - where_clause: &Option, -) -> String { +fn build_count_query(schema: &Option, table: &str) -> String { let schema_prefix = schema .as_ref() .map(|s| format!("\"{}\".", s)) .unwrap_or_default(); - let mut query = format!( + format!( "SELECT COUNT(*) AS count FROM {}\"{}\"", schema_prefix, table - ); - - if let Some(ref where_clause) = where_clause { - query.push_str(&format!(" WHERE {}", where_clause)); - } - - query + ) } -fn write_csv_header( - writer: &mut BufWriter, +// ── CSV helpers ────────────────────────────────────────────────── + +fn write_csv_header( + writer: &mut BufWriter, columns: &[String], delimiter: char, ) -> Result<(), std::io::Error> { @@ -377,8 +927,8 @@ fn write_csv_header( Ok(()) } -fn write_csv_row( - writer: &mut BufWriter, +fn write_csv_row( + writer: &mut BufWriter, columns: &[String], row: &crate::database::QueryRow, opts: &CsvExportOptions, @@ -397,7 +947,8 @@ fn write_csv_row( || s.contains('\n') { format!("\"{}\"", s.replace('"', "\"\"")) - } else { + } + else { s.clone() } } @@ -415,6 +966,8 @@ fn write_csv_row( Ok(()) } +// ── JSONL helpers ──────────────────────────────────────────────── + fn row_to_json_object(row: &crate::database::QueryRow, _date_format: &str) -> JsonValue { let mut obj = serde_json::Map::new(); for (key, value) in row { @@ -434,6 +987,8 @@ fn row_to_json_object(row: &crate::database::QueryRow, _date_format: &str) -> Js JsonValue::Object(obj) } +// ── SQL helpers ────────────────────────────────────────────────── + fn generate_create_table_sql( table: &str, _table_info: &crate::database::TableInfo, @@ -496,17 +1051,24 @@ fn query_value_to_sql_literal(value: &QueryValue) -> String { } } -/// Generates a preview of export data. +// ── Preview ────────────────────────────────────────────────────── + +/// Generates a preview of export data from the first export source. pub async fn preview_export( adapter: &A, request: ExportRequest, preview_rows: u32, ) -> Result { - let columns = request.source.columns.clone(); - let table = request.source.table.clone(); + let source = request + .sources + .first() + .ok_or("No export sources specified")? + .clone(); + let columns = source.columns.clone(); + let table = source.table.clone(); let schema = request.schema.clone(); - let base_query = build_export_query(&schema, &table, &columns, &request.source); + let base_query = build_export_query(&schema, &table, &columns); let query = format!("{} LIMIT {}", base_query, preview_rows); let result = adapter @@ -534,7 +1096,7 @@ pub async fn preview_export( }) .collect(); - let count_query = build_count_query(&schema, &table, &request.source.where_clause); + let count_query = build_count_query(&schema, &table); let count_result = adapter .execute_query(&count_query) .await @@ -598,7 +1160,8 @@ fn format_preview( .map(|v| { if v.is_empty() { "NULL".to_string() - } else { + } + else { format!("'{}'", v) } }) @@ -621,6 +1184,81 @@ fn format_preview( } } +// ── Excel helpers ──────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_chat2db_filename_pattern() { + let filename = format_chat2db_filename("users", "csv"); + // Pattern: users_YYYYMMDDHHMMSS.csv + assert!(filename.starts_with("users_")); + assert!(filename.ends_with(".csv")); + // The middle part should be 14 digits (YYYYMMDDHHMMSS) + let timestamp_part = &filename["users_".len()..filename.len() - ".csv".len()]; + assert_eq!(timestamp_part.len(), 14); + assert!(timestamp_part.chars().all(|c| c.is_ascii_digit())); + } + + #[test] + fn test_format_chat2db_filename_different_extensions() { + // Should work with any extension + let name_sql = format_chat2db_filename("orders", "sql"); + assert!(name_sql.starts_with("orders_")); + assert!(name_sql.ends_with(".sql")); + + let name_xlsx = format_chat2db_filename("orders", "xlsx"); + assert!(name_xlsx.starts_with("orders_")); + assert!(name_xlsx.ends_with(".xlsx")); + } + + #[test] + fn test_format_chat2db_filename_special_chars() { + // Table names with underscores should still produce valid filenames + let filename = format_chat2db_filename("my_table", "csv"); + assert!(filename.starts_with("my_table_")); + assert!(filename.ends_with(".csv")); + // Should only have one underscore before timestamp + let after_table = &filename["my_table".len()..]; + assert!(after_table.starts_with("_")); + } + + #[test] + fn test_format_extension_csv() { + assert_eq!(format_extension(&ExportFormat::Csv), "csv"); + } + + #[test] + fn test_format_extension_jsonl() { + assert_eq!(format_extension(&ExportFormat::Jsonl), "jsonl"); + } + + #[test] + fn test_format_extension_sql() { + assert_eq!(format_extension(&ExportFormat::Sql), "sql"); + } + + #[test] + fn test_format_extension_excel() { + assert_eq!(format_extension(&ExportFormat::Excel), "xlsx"); + } + + #[test] + fn test_format_extension_all_formats() { + let cases = [ + (ExportFormat::Csv, "csv"), + (ExportFormat::Jsonl, "jsonl"), + (ExportFormat::Sql, "sql"), + (ExportFormat::Excel, "xlsx"), + ]; + for (format, expected) in &cases { + assert_eq!(format_extension(format), *expected); + } + } +} + fn write_excel_cell( worksheet: &mut Worksheet, row: u32, @@ -655,7 +1293,7 @@ fn write_excel_cell( } QueryValue::Bytes(b) => { worksheet - .write_string(row, col, &hex::encode(b)) + .write_string(row, col, hex::encode(b)) .map_err(|e| e.to_string())?; Ok(()) } diff --git a/src-tauri/src/transfer/import.rs b/src-tauri/src/transfer/import.rs index b8055f55..582e6456 100644 --- a/src-tauri/src/transfer/import.rs +++ b/src-tauri/src/transfer/import.rs @@ -12,45 +12,199 @@ use super::progress::*; use super::types::*; /// Executes a data import operation. +/// Supports Tables scope (import to specific tables), Database scope (create database if needed), +/// and Server scope (execute SQL directly at connection level). pub async fn execute_import( adapter: &A, request: ImportRequest, app_handle: &tauri::AppHandle, ) -> Result { let start_time = Instant::now(); + let mut total_processed: u64 = 0; + let mut total_skipped: u64 = 0; + let mut total_errors: Vec = Vec::new(); emit_progress( app_handle, &create_progress("import", "preparing", 0, None, 0), ); - let csv_opts = request + match request.scope { + TransferScope::Server => { + // Server scope: execute SQL file directly at connection level + // SQL files may contain CREATE DATABASE statements + if let Some(target) = request.tables.first() { + if target.format == ImportFormat::Sql { + let target_result = + import_sql_at_server_level(adapter, &request, target, app_handle, &start_time) + .await?; + total_processed += target_result.processed_rows; + total_skipped += target_result.skipped_rows; + total_errors.extend(target_result.errors); + } else { + return Err("Server scope import only supports SQL files".to_string()); + } + } + } + TransferScope::Database => { + // Database scope: create database if needed, then import tables + if request.create_database_if_not_exists.unwrap_or(false) { + if let Some(ref db_name) = request.database { + let db_exists = adapter + .list_databases() + .await + .map_err(|e| e.to_string())? + .iter() + .any(|d| &d.name == db_name); + if !db_exists { + adapter + .execute_query(&format!("CREATE DATABASE \"{}\"", db_name)) + .await + .map_err(|e| format!("Failed to create database: {}", e))?; + } + } + } + for target in &request.tables { + let target_result = + import_target(adapter, &request, target, app_handle, &start_time).await?; + total_processed += target_result.processed_rows; + total_skipped += target_result.skipped_rows; + total_errors.extend(target_result.errors); + } + } + TransferScope::Tables => { + // Tables scope: import to specific tables (current behavior) + for target in &request.tables { + let target_result = + import_target(adapter, &request, target, app_handle, &start_time).await?; + total_processed += target_result.processed_rows; + total_skipped += target_result.skipped_rows; + total_errors.extend(target_result.errors); + } + } + } + + emit_progress( + app_handle, + &create_progress( + "import", + "finalizing", + total_processed, + None, + start_time.elapsed().as_millis() as u64, + ), + ); + + Ok(TransferResult { + success: total_errors.is_empty(), + total_rows: total_processed + total_skipped, + processed_rows: total_processed, + skipped_rows: total_skipped, + error_count: total_errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors: total_errors, + }) +} + +/// Import SQL file at server level (can contain CREATE DATABASE statements). +async fn import_sql_at_server_level( + adapter: &A, + request: &ImportRequest, + target: &ImportTarget, + app_handle: &tauri::AppHandle, + start_time: &Instant, +) -> Result { + let file_path = Path::new(&target.file_path); + let content = std::fs::read_to_string(file_path) + .map_err(|e| format!("Failed to read SQL file: {}", e))?; + + let mut processed_rows: u64 = 0; + let mut skipped_rows: u64 = 0; + let mut errors: Vec = Vec::new(); + + // Split by semicolon and execute each statement + for (idx, stmt) in content.split(';').filter(|s| !s.trim().is_empty()).enumerate() { + let stmt = stmt.trim(); + if stmt.is_empty() { + continue; + } + let result = adapter.execute_query(stmt).await; + match result { + Ok(_) => { + processed_rows += 1; + } + Err(e) => { + errors.push(TransferError { + row_number: None, + statement_number: Some(idx as u64 + 1), + message: e.to_string(), + sql: Some(stmt.to_string()), + }); + skipped_rows += 1; + } + } + + emit_progress( + app_handle, + &create_progress( + "import", + "processing", + processed_rows, + None, + start_time.elapsed().as_millis() as u64, + ), + ); + } + + Ok(TransferResult { + success: errors.is_empty(), + total_rows: processed_rows + skipped_rows, + processed_rows, + skipped_rows, + error_count: errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors, + }) +} + +async fn import_target( + adapter: &A, + request: &ImportRequest, + target: &ImportTarget, + app_handle: &tauri::AppHandle, + start_time: &Instant, +) -> Result { + let csv_opts = target .csv_options .clone() .unwrap_or_else(csv_import_defaults); - let file_path = Path::new(&request.file_path); + let file_path = Path::new(&target.file_path); let file = File::open(file_path).map_err(|e| format!("Failed to open file: {}", e))?; let mut processed_rows: u64 = 0; let mut skipped_rows: u64 = 0; let mut errors: Vec = Vec::new(); - match request.format { + match target.format { ImportFormat::Csv => { let reader = BufReader::new(file); let delimiter = csv_opts.delimiter; let mut lines = reader.lines().peekable(); - let header_line = if csv_opts.has_header { + let _header_line = if csv_opts.has_header { lines .next() .transpose() .map_err(|e| format!("Failed to read header: {}", e))? .unwrap_or_default() } else { - request + target .column_mappings .iter() .map(|m| m.source_column.clone()) @@ -59,9 +213,9 @@ pub async fn execute_import( }; let header_columns: Vec = if csv_opts.has_header { - parse_csv_line(&header_line, delimiter) + parse_csv_line(&_header_line, delimiter) } else { - request + target .column_mappings .iter() .map(|m| m.source_column.clone()) @@ -84,11 +238,7 @@ pub async fn execute_import( .iter() .enumerate() .filter_map(|(i, col)| { - let mapping = request - .column_mappings - .iter() - .find(|m| m.source_column == *col); - + let mapping = target.column_mappings.iter().find(|m| m.source_column == *col); if mapping.is_none() || mapping.and_then(|m| m.target_column.as_ref()).is_none() { @@ -103,9 +253,10 @@ pub async fn execute_import( processed_rows += 1; if batch_values.len() >= request.batch_size as usize { - let insert_result = execute_batch_insert( + let insert_result = execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -141,9 +292,10 @@ pub async fn execute_import( } if !batch_values.is_empty() { - let insert_result = execute_batch_insert( + let insert_result = execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -169,7 +321,7 @@ pub async fn execute_import( ImportFormat::Jsonl => { let reader = BufReader::new(file); let mut batch_values: Vec> = Vec::new(); - let _target_columns: Vec = request + let _target_columns: Vec = target .column_mappings .iter() .filter_map(|m| m.target_column.clone()) @@ -199,7 +351,7 @@ pub async fn execute_import( } let obj = json_obj.as_object().unwrap(); - let values: Vec = request + let values: Vec = target .column_mappings .iter() .filter_map(|m| { @@ -225,9 +377,10 @@ pub async fn execute_import( processed_rows += 1; if batch_values.len() >= request.batch_size as usize { - let insert_result = execute_batch_insert( + let insert_result = execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -263,9 +416,10 @@ pub async fn execute_import( } if !batch_values.is_empty() { - execute_batch_insert( + execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -349,7 +503,7 @@ pub async fn execute_import( let mut workbook: Xlsx<_> = open_workbook(file_path) .map_err(|e| format!("Failed to open Excel file: {}", e))?; - let sheet_name = request + let sheet_name = target .excel_options .as_ref() .map(|o| o.sheet_name.clone()) @@ -360,7 +514,7 @@ pub async fn execute_import( .ok_or_else(|| format!("Sheet '{}' not found", sheet_name))? .map_err(|e| format!("Failed to read sheet '{}': {:?}", sheet_name, e))?; - let has_header = request + let has_header = target .excel_options .as_ref() .map(|o| o.has_header) @@ -377,7 +531,7 @@ pub async fn execute_import( }) .unwrap_or_default() } else { - request + target .column_mappings .iter() .map(|m| m.source_column.clone()) @@ -391,11 +545,7 @@ pub async fn execute_import( .iter() .enumerate() .filter_map(|(col_idx, col)| { - let mapping = request - .column_mappings - .iter() - .find(|m| m.source_column == *col); - + let mapping = target.column_mappings.iter().find(|m| m.source_column == *col); if mapping.is_none() || mapping.and_then(|m| m.target_column.as_ref()).is_none() { @@ -414,9 +564,10 @@ pub async fn execute_import( processed_rows += 1; if batch_values.len() >= request.batch_size as usize { - let insert_result = execute_batch_insert( + let insert_result = execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -452,9 +603,10 @@ pub async fn execute_import( } if !batch_values.is_empty() { - let insert_result = execute_batch_insert( + let insert_result = execute_batch_insert_for_target( adapter, - &request, + request, + target, &batch_values, processed_rows - batch_values.len() as u64, ) @@ -478,17 +630,6 @@ pub async fn execute_import( } } - emit_progress( - app_handle, - &create_progress( - "import", - "finalizing", - processed_rows, - None, - start_time.elapsed().as_millis() as u64, - ), - ); - Ok(TransferResult { success: errors.is_empty(), total_rows: processed_rows + skipped_rows, @@ -522,9 +663,10 @@ fn parse_csv_line(line: &str, delimiter: char) -> Vec { values } -async fn execute_batch_insert( +async fn execute_batch_insert_for_target( adapter: &A, request: &ImportRequest, + target: &ImportTarget, batch: &[Vec], _start_row: u64, ) -> Result { @@ -537,7 +679,7 @@ async fn execute_batch_insert( .as_ref() .map(|s| format!("\"{}\".", s)) .unwrap_or_default(); - let target_columns: Vec = request + let target_columns: Vec = target .column_mappings .iter() .filter_map(|m| m.target_column.clone()) @@ -569,12 +711,11 @@ async fn execute_batch_insert( let sql = format!( "INSERT INTO {}\"{}\" ({}) VALUES {}", schema_prefix, - request.table, + target.table, col_list, values_list.join(", ") ); - // Dry-run: validate INSERT statement was built but skip the write. if request.dry_run { return Ok(batch.len() as u64); } diff --git a/src-tauri/src/transfer/migration.rs b/src-tauri/src/transfer/migration.rs index 209eba4d..3b766990 100644 --- a/src-tauri/src/transfer/migration.rs +++ b/src-tauri/src/transfer/migration.rs @@ -25,32 +25,258 @@ pub async fn execute_migration( &create_progress("migration", "preparing", 0, None, 0), ); - for (table_idx, table_plan) in request.table_plans.iter().enumerate() { + match request.scope { + TransferScope::Server => { + let source_databases = source_adapter + .list_databases() + .await + .map_err(|e| format!("Failed to list source databases: {}", e))?; + for source_db in source_databases { + if request.create_target_database_if_not_exists.unwrap_or(false) { + let target_databases = target_adapter + .list_databases() + .await + .map_err(|e| e.to_string())?; + if !target_databases.iter().any(|d| d.name == source_db.name) { + target_adapter + .execute_query(&format!("CREATE DATABASE \"{}\"", source_db.name)) + .await + .map_err(|e| format!("Failed to create target database: {}", e))?; + } + } + let tables = source_adapter + .list_tables(Some(&source_db.name), None) + .await + .map_err(|e| e.to_string())?; + for table_info in tables { + let columns = source_adapter + .list_columns(Some(&source_db.name), None, &table_info.name) + .await + .map_err(|e| e.to_string())?; + let mappings: Vec = columns + .iter() + .map(|c| MigrationMapping { + source_column: c.name.clone(), + source_type: c.data_type.clone(), + target_column: c.name.clone(), + target_type: c.data_type.clone(), + conversion: MigrationConversion::Direct, + }) + .collect(); + let table_plan = MigrationTablePlan { + source_table: table_info.name.clone(), + target_table: table_info.name.clone(), + column_mappings: mappings, + }; + let mut req = request.clone(); + req.source_database = Some(source_db.name.clone()); + req.target_database = Some(source_db.name.clone()); + req.table_plans = vec![table_plan]; + let result = migrate_single_table_plan( + source_adapter, + target_adapter, + &req, + app_handle, + start_time, + &mut total_processed, + &mut total_skipped, + &mut total_errors, + ) + .await?; + } + } + } + TransferScope::Database => { + if request.create_target_database_if_not_exists.unwrap_or(false) { + if let Some(ref target_db) = request.target_database { + let target_databases = target_adapter + .list_databases() + .await + .map_err(|e| e.to_string())?; + if !target_databases.iter().any(|d| d.name == *target_db) { + target_adapter + .execute_query(&format!("CREATE DATABASE \"{}\"", target_db)) + .await + .map_err(|e| format!("Failed to create target database: {}", e))?; + } + } + } + if request.table_plans.is_empty() { + let source_db = request.source_database.as_ref().ok_or_else(|| { + "Source database required for Database scope with empty table_plans" + })?; + let tables = source_adapter + .list_tables(Some(source_db), request.source_schema.as_deref()) + .await + .map_err(|e| e.to_string())?; + let mut auto_plans: Vec = Vec::new(); + for table_info in tables { + let columns = source_adapter + .list_columns(Some(source_db), request.source_schema.as_deref(), &table_info.name) + .await + .map_err(|e| e.to_string())?; + let mappings: Vec = columns + .iter() + .map(|c| MigrationMapping { + source_column: c.name.clone(), + source_type: c.data_type.clone(), + target_column: c.name.clone(), + target_type: c.data_type.clone(), + conversion: MigrationConversion::Direct, + }) + .collect(); + auto_plans.push(MigrationTablePlan { + source_table: table_info.name.clone(), + target_table: table_info.name.clone(), + column_mappings: mappings, + }); + } + let mut req = request.clone(); + req.table_plans = auto_plans; + for (table_idx, table_plan) in req.table_plans.iter().enumerate() { + let result = migrate_single_table_plan( + source_adapter, + target_adapter, + &req, + app_handle, + start_time, + &mut total_processed, + &mut total_skipped, + &mut total_errors, + ) + .await?; + } + } else { + for (table_idx, table_plan) in request.table_plans.iter().enumerate() { + let result = migrate_single_table_plan( + source_adapter, + target_adapter, + &request, + app_handle, + start_time, + &mut total_processed, + &mut total_skipped, + &mut total_errors, + ) + .await?; + } + } + } + TransferScope::Tables => { + for (table_idx, table_plan) in request.table_plans.iter().enumerate() { + emit_progress( + app_handle, + &TransferProgress { + operation: "migration".to_string(), + phase: "processing".to_string(), + current_database: None, + current_table: Some(table_plan.source_table.clone()), + total_rows: None, + processed_rows: total_processed, + skipped_rows: total_skipped, + error_count: total_errors.len() as u64, + percent: 0.0, + elapsed_ms: start_time.elapsed().as_millis() as u64, + estimated_remaining_ms: None, + message: Some(format!( + "Migrating table {} of {}", + table_idx + 1, + request.table_plans.len() + )), + }, + ); + + let table_result = migrate_table( + source_adapter, + target_adapter, + &request, + table_plan, + app_handle, + start_time, + ) + .await; + + match table_result { + Ok(result) => { + total_processed += result.processed_rows; + total_skipped += result.skipped_rows; + if !result.success { + total_errors.extend(result.errors); + } + } + Err(e) => { + total_errors.push(TransferError { + row_number: None, + statement_number: None, + message: format!("Table {} failed: {}", table_plan.source_table, e), + sql: None, + }); + if request.on_error == MigrationErrorStrategy::Abort { + break; + } + } + } + } + } + } + + emit_progress( + app_handle, + &create_progress( + "migration", + "finalizing", + total_processed, + None, + start_time.elapsed().as_millis() as u64, + ), + ); + + Ok(TransferResult { + success: total_errors.is_empty(), + total_rows: total_processed + total_skipped, + processed_rows: total_processed, + skipped_rows: total_skipped, + error_count: total_errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors: total_errors, + }) +} + +async fn migrate_single_table_plan( + source_adapter: &A1, + target_adapter: &A2, + request: &MigrationRequest, + app_handle: &tauri::AppHandle, + start_time: Instant, + total_processed: &mut u64, + total_skipped: &mut u64, + total_errors: &mut Vec, +) -> Result<(), String> { + if let Some(table_plan) = request.table_plans.first() { emit_progress( app_handle, &TransferProgress { operation: "migration".to_string(), phase: "processing".to_string(), + current_database: request.source_database.clone(), current_table: Some(table_plan.source_table.clone()), total_rows: None, - processed_rows: total_processed, - skipped_rows: total_skipped, + processed_rows: *total_processed, + skipped_rows: *total_skipped, error_count: total_errors.len() as u64, percent: 0.0, elapsed_ms: start_time.elapsed().as_millis() as u64, estimated_remaining_ms: None, - message: Some(format!( - "Migrating table {} of {}", - table_idx + 1, - request.table_plans.len() - )), + message: Some(format!("Migrating table {}", table_plan.source_table)), }, ); let table_result = migrate_table( source_adapter, target_adapter, - &request, + request, table_plan, app_handle, start_time, @@ -59,8 +285,8 @@ pub async fn execute_migration( match table_result { Ok(result) => { - total_processed += result.processed_rows; - total_skipped += result.skipped_rows; + *total_processed += result.processed_rows; + *total_skipped += result.skipped_rows; if !result.success { total_errors.extend(result.errors); } @@ -72,35 +298,10 @@ pub async fn execute_migration( message: format!("Table {} failed: {}", table_plan.source_table, e), sql: None, }); - if request.on_error == MigrationErrorStrategy::Abort { - break; - } } } } - - emit_progress( - app_handle, - &create_progress( - "migration", - "finalizing", - total_processed, - None, - start_time.elapsed().as_millis() as u64, - ), - ); - - Ok(TransferResult { - success: total_errors.is_empty(), - total_rows: total_processed + total_skipped, - processed_rows: total_processed, - skipped_rows: total_skipped, - error_count: total_errors.len() as u64, - duration_ms: start_time.elapsed().as_millis() as u64, - output_path: None, - output_size_bytes: None, - errors: total_errors, - }) + Ok(()) } async fn migrate_table( @@ -214,6 +415,7 @@ async fn migrate_table( &TransferProgress { operation: "migration".to_string(), phase: "processing".to_string(), + current_database: None, current_table: Some(table_plan.source_table.clone()), total_rows: Some(total_rows), processed_rows: processed_rows, diff --git a/src-tauri/src/transfer/progress.rs b/src-tauri/src/transfer/progress.rs index 0daa875d..bba10d19 100644 --- a/src-tauri/src/transfer/progress.rs +++ b/src-tauri/src/transfer/progress.rs @@ -37,6 +37,7 @@ pub fn create_progress( TransferProgress { operation: operation.to_string(), phase: phase.to_string(), + current_database: None, current_table: None, total_rows, processed_rows, diff --git a/src-tauri/src/transfer/types.rs b/src-tauri/src/transfer/types.rs index 1774076a..c9d4dfe2 100644 --- a/src-tauri/src/transfer/types.rs +++ b/src-tauri/src/transfer/types.rs @@ -16,6 +16,16 @@ pub enum ExportFormat { Excel, } +/// Scope of a transfer operation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub enum TransferScope { + Server, + Database, + #[default] + Tables, +} + /// CSV export options with sensible defaults. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -71,15 +81,12 @@ pub struct ExcelExportOptions { pub freeze_header: bool, } -/// Export source is always a table (Custom Query removed for simplicity). +/// Export source is always a table. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ExportSource { pub table: String, pub columns: Vec, - pub where_clause: Option, - pub order_by: Option, - pub limit: Option, } /// Export request payload. @@ -89,7 +96,9 @@ pub struct ExportRequest { pub connection_id: String, pub database: Option, pub schema: Option, - pub source: ExportSource, + #[serde(default)] + pub scope: TransferScope, + pub sources: Vec, pub format: ExportFormat, pub csv_options: Option, pub jsonl_options: Option, @@ -152,6 +161,18 @@ pub struct ExcelImportOptions { pub has_header: bool, } +/// Import target table configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportTarget { + pub table: String, + pub file_path: String, + pub format: ImportFormat, + pub column_mappings: Vec, + pub csv_options: Option, + pub excel_options: Option, +} + /// Import request payload. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -159,10 +180,9 @@ pub struct ImportRequest { pub connection_id: String, pub database: Option, pub schema: Option, - pub table: String, - pub file_path: String, - pub format: ImportFormat, - pub column_mappings: Vec, + #[serde(default)] + pub scope: TransferScope, + pub tables: Vec, #[serde(default)] pub conflict_strategy: ConflictStrategy, #[serde(default = "default_import_batch_size")] @@ -173,8 +193,8 @@ pub struct ImportRequest { pub truncate_before: bool, #[serde(default)] pub dry_run: bool, - pub csv_options: Option, - pub excel_options: Option, + #[serde(default)] + pub create_database_if_not_exists: Option, } // ── Progress & Results ────────────────────────────────────────── @@ -183,14 +203,15 @@ pub struct ImportRequest { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransferProgress { - pub operation: String, // "export" | "import" | "ddl" | "sql_file" | "migration" - pub phase: String, // "preparing" | "processing" | "finalizing" + pub operation: String, + pub phase: String, + pub current_database: Option, pub current_table: Option, pub total_rows: Option, pub processed_rows: u64, pub skipped_rows: u64, pub error_count: u64, - pub percent: f32, // 0.0–100.0 + pub percent: f32, pub elapsed_ms: u64, pub estimated_remaining_ms: Option, pub message: Option, @@ -312,6 +333,8 @@ pub struct DdlRequest { pub connection_id: String, pub database: Option, pub schema: Option, + #[serde(default)] + pub scope: TransferScope, pub objects: Vec, pub options: DdlOptions, } @@ -383,6 +406,8 @@ pub struct MigrationRequest { pub target_connection_id: String, pub target_database: Option, pub target_schema: Option, + #[serde(default)] + pub scope: TransferScope, pub table_plans: Vec, #[serde(default = "default_migration_batch_size")] pub batch_size: u32, @@ -400,6 +425,8 @@ pub struct MigrationRequest { pub migrate_constraints: bool, #[serde(default)] pub disable_fk_checks: bool, + #[serde(default)] + pub create_target_database_if_not_exists: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] @@ -432,3 +459,108 @@ pub struct MigrationTablePreview { fn default_migration_batch_size() -> u32 { 5000 } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_transfer_scope_serde_roundtrip_server() { + let scope = TransferScope::Server; + let json = serde_json::to_string(&scope).unwrap(); + assert_eq!(json, "\"server\""); + let deserialized: TransferScope = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized, TransferScope::Server); + } + + #[test] + fn test_transfer_scope_serde_roundtrip_database() { + let scope = TransferScope::Database; + let json = serde_json::to_string(&scope).unwrap(); + assert_eq!(json, "\"database\""); + let deserialized: TransferScope = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized, TransferScope::Database); + } + + #[test] + fn test_transfer_scope_serde_roundtrip_tables() { + let scope = TransferScope::Tables; + let json = serde_json::to_string(&scope).unwrap(); + assert_eq!(json, "\"tables\""); + let deserialized: TransferScope = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized, TransferScope::Tables); + } + + #[test] + fn test_transfer_scope_default_is_tables() { + let default_scope: TransferScope = Default::default(); + assert_eq!(default_scope, TransferScope::Tables); + } + + #[test] + fn test_transfer_scope_serde_default_on_missing() { + #[derive(Serialize, Deserialize)] + struct Container { + #[serde(default)] + scope: TransferScope, + } + let json = r#"{}"#; + let container: Container = serde_json::from_str(json).unwrap(); + assert_eq!(container.scope, TransferScope::Tables); + } + + #[test] + fn test_export_format_serde_roundtrip() { + let formats = [ + ExportFormat::Csv, + ExportFormat::Jsonl, + ExportFormat::Sql, + ExportFormat::Excel, + ]; + for format in &formats { + let json = serde_json::to_string(format).unwrap(); + let deserialized: ExportFormat = serde_json::from_str(&json).unwrap(); + assert_eq!(&deserialized, format); + } + } + + #[test] + fn test_export_request_serde_roundtrip() { + let request = ExportRequest { + connection_id: "test-conn".into(), + database: Some("test_db".into()), + schema: None, + scope: TransferScope::Tables, + sources: vec![ExportSource { + table: "users".into(), + columns: vec!["id".into(), "name".into()], + }], + format: ExportFormat::Csv, + csv_options: None, + jsonl_options: None, + sql_options: None, + excel_options: None, + output_path: "/tmp/export.csv".into(), + }; + let json = serde_json::to_string(&request).unwrap(); + let deserialized: ExportRequest = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.connection_id, request.connection_id); + assert_eq!(deserialized.database, request.database); + assert_eq!(deserialized.scope, TransferScope::Tables); + assert_eq!(deserialized.sources.len(), 1); + assert_eq!(deserialized.sources[0].table, "users"); + assert_eq!(deserialized.format, ExportFormat::Csv); + } + + #[test] + fn test_export_request_default_scope_is_tables() { + let json = r#"{ + "connectionId": "test", + "sources": [], + "format": "csv", + "outputPath": "/tmp/test.csv" + }"#; + let request: ExportRequest = serde_json::from_str(json).unwrap(); + assert_eq!(request.scope, TransferScope::Tables); + } +} diff --git a/src/components/transfer/export/ExportExecuteStep.vue b/src/components/transfer/export/ExportExecuteStep.vue index 434a827b..393a44c8 100644 --- a/src/components/transfer/export/ExportExecuteStep.vue +++ b/src/components/transfer/export/ExportExecuteStep.vue @@ -1,5 +1,5 @@ diff --git a/src/components/transfer/export/ExportWizard.vue b/src/components/transfer/export/ExportWizard.vue index b69aad9f..594b777f 100644 --- a/src/components/transfer/export/ExportWizard.vue +++ b/src/components/transfer/export/ExportWizard.vue @@ -1,5 +1,5 @@ @@ -163,8 +208,35 @@ function startExport() { icon-class="text-emerald-600 dark:text-emerald-500" :summary="sourceSummary" min-height="340px" + :scope="scope" + @update:scope="scope = $event" > -
+ +
+ + + All databases on this server + +
+ + +
+ + + All tables in {{ database }} + +
+ + +
- -
+ +
- -
@@ -331,7 +395,7 @@ function startExport() {
@@ -351,12 +415,9 @@ function startExport() {
-
- - {{ selectedTables.length }} {{ t('transfer.migration.tablesSelected', 'tables') }} - +
- {{ tableColumns.reduce((sum, tc) => sum + tc.selectedColumns.length, 0) }} cols + {{ scope === 'server' ? 'All databases' : scope === 'database' ? (database || 'Select database') : `${selectedTables.length} tables` }} {{ selectedFormat }} diff --git a/src/components/transfer/import/ImportExecuteStep.vue b/src/components/transfer/import/ImportExecuteStep.vue index d1979bfd..4b4ae3c5 100644 --- a/src/components/transfer/import/ImportExecuteStep.vue +++ b/src/components/transfer/import/ImportExecuteStep.vue @@ -55,10 +55,12 @@ function handleRunInBackground() { 'import', { connectionId: transferStore.importRequest.connectionId || '', - table: transferStore.importRequest.table || '', + scope: transferStore.importRequest.scope || 'tables', + tables: transferStore.importRequest.tables || [], filePath: transferStore.importRequest.filePath || '', format: transferStore.importRequest.format || 'csv', conflictStrategy: transferStore.importRequest.conflictStrategy, + createDatabaseIfNotExists: transferStore.importRequest.createDatabaseIfNotExists, }, progress.value?.totalRows || 0, ) diff --git a/src/components/transfer/import/ImportMappingStep.vue b/src/components/transfer/import/ImportMappingStep.vue index 61c325ea..f5065543 100644 --- a/src/components/transfer/import/ImportMappingStep.vue +++ b/src/components/transfer/import/ImportMappingStep.vue @@ -1,5 +1,5 @@ diff --git a/src/components/transfer/tasks/TaskManagerButton.vue b/src/components/transfer/tasks/TaskManagerButton.vue index 3393ef21..2cdf2522 100644 --- a/src/components/transfer/tasks/TaskManagerButton.vue +++ b/src/components/transfer/tasks/TaskManagerButton.vue @@ -13,7 +13,7 @@ const transferStore = useTransferStore()