diff --git a/docs/TRANSFER_DESIGN.md b/docs/TRANSFER_DESIGN.md new file mode 100644 index 00000000..dee1c1b6 --- /dev/null +++ b/docs/TRANSFER_DESIGN.md @@ -0,0 +1,2708 @@ +# Transfer Feature Design + +## Overview + +The **Transfer** feature replaces the placeholder "Import / Export" page with a comprehensive data transfer system supporting data export/import, database structure generation, SQL file execution, and cross-engine data migration. + +**Feature Name**: Transfer +**Route**: `/transfer` (replaces `/import-export`) +**Sidebar Label**: Transfer + +### Design Principles + +- **Step-based inline wizards** (not modal dialogs) for each operation +- **Streaming/chunked processing** for large datasets (100k+ rows) +- **Progress reporting** via Tauri events for real-time UI feedback +- **Background task system** — all long-running operations run as background tasks; users can navigate away and return to check progress or restore task state (modeled after dockit's pattern) +- **Best-practice defaults** — format options use sensible defaults (comma delimiter, UTF-8, include header, etc.) instead of exposing every knob to the user; advanced users can expand an optional "Advanced" section if needed +- **Cross-platform** file dialogs using Tauri's native dialog plugin +- **All strings use i18n** for internationalization + +--- + +## Information Architecture + +### Top-Level Tabs + +``` +┌────────────────────────────────────────────────────────────────────────────┐ +│ Transfer [🔔 Tasks] │ +├──────────┬──────────┬─────────────┬──────────────┬─────────────────────────┤ +│ Export │ Import │ Structure │ Migration │ │ +├──────────┴──────────┴─────────────┴──────────────┴─────────────────────────┤ +│ │ +│ (Tab content area — wizard steps render here) │ +│ │ +└────────────────────────────────────────────────────────────────────────────┘ +``` + +The **Tasks** button in the top-right opens a slide-out Task Manager panel showing all running and completed transfer tasks. + +| Tab | Purpose | Steps | +|-----|---------|-------| +| **Export** | Export table data to file | 4 steps | +| **Import** | Import file data into a table | 4 steps | +| **Structure** | Generate DDL / Run SQL files | 2 sub-tabs, 2-4 steps each | +| **Migration** | Cross-engine data migration | 5 steps | + +--- + +## Feature Specifications + +### 1. Data Export + +Export data from a table to a file in various formats. + +#### Supported Formats + +| Format | Extension | Crate | Best-Practice Defaults | +|--------|-----------|-------|------------------------| +| CSV | `.csv` | `csv` | Comma delimiter, double-quote, UTF-8, include header, LF line ending | +| JSONL | `.jsonl` | `serde_json` | One JSON object per line, compact, UTF-8, ISO 8601 dates | +| SQL | `.sql` | (built-in) | Auto-filled target table, batch size 1000, include CREATE TABLE | +| Excel | `.xlsx` | `rust_xlsxwriter` | Include header, auto-fit columns, freeze header row | + +> **Design decision**: Format options use best-practice defaults automatically. No per-format options panels are shown. An optional "Advanced Options" expandable section is available for power users who need to override defaults. + +#### Wizard Steps + +**Step 1 — Source Selection** + +``` +┌─ Step 1: Source ──────────────────────────────────────────────────────────┐ +│ │ +│ Connection: [▼ my-postgres-server ] │ +│ Database: [▼ mydb ] │ +│ Schema: [▼ public ] │ +│ │ +│ Table: [▼ users ] │ +│ Columns: ☑ id ☑ name ☑ email ☐ password ☑ created_at │ +│ [Select All] [Deselect All] │ +│ WHERE: [age > 18 ] (optional) │ +│ ORDER BY: [created_at DESC ] (optional) │ +│ LIMIT: [1000 ] (optional) │ +│ │ +│ [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 2 — Format Selection** + +``` +┌─ Step 2: Format ──────────────────────────────────────────────────────────┐ +│ │ +│ Format: ○ CSV ○ JSONL ○ SQL ○ Excel │ +│ │ +│ ┌─ Defaults Applied ────────────────────────────────────────────────┐ │ +│ │ ✓ Comma delimiter, double-quote, UTF-8 encoding │ │ +│ │ ✓ Header row included │ │ +│ │ ✓ LF line endings │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ▸ Advanced Options │ +│ ┌─ (expanded, only when clicked) ───────────────────────────────────┐ │ +│ │ Delimiter: [▼ Comma (,) ] │ │ +│ │ Encoding: [▼ UTF-8 ] │ │ +│ │ ☑ Include header row │ │ +│ │ (format-specific overrides shown based on selection) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [← Back] [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 3 — Preview** + +``` +┌─ Step 3: Preview ─────────────────────────────────────────────────────────┐ +│ │ +│ Source: my-postgres-server / mydb / public.users │ +│ Format: CSV | Rows: 1,000 (estimated) | Columns: 4 │ +│ │ +│ ┌─ Preview (first 10 rows) ─────────────────────────────────────────┐ │ +│ │ id,name,email,created_at │ │ +│ │ 1,"Alice","alice@example.com","2024-01-15T10:30:00Z" │ │ +│ │ 2,"Bob","bob@example.com","2024-01-16T14:22:00Z" │ │ +│ │ 3,"Charlie","charlie@example.com","2024-01-17T09:15:00Z" │ │ +│ │ ... │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Output File: [/Users/me/exports/users.csv ] [Browse...] │ +│ │ +│ [← Back] [Export →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 4 — Execution** + +``` +┌─ Step 4: Exporting ───────────────────────────────────────────────────────┐ +│ │ +│ Status: Exporting... │ +│ │ +│ ████████████████████░░░░░░░░░░ 65% │ +│ │ +│ Rows exported: 650 / 1,000 │ +│ Elapsed: 2.3s │ +│ Estimated remaining: 1.2s │ +│ │ +│ [Run in Background] [Cancel] │ +│ │ +│ ── After completion ── │ +│ │ +│ ✓ Export completed successfully │ +│ File: /Users/me/exports/users.csv (45.2 KB) │ +│ Rows exported: 1,000 │ +│ Duration: 3.5s │ +│ │ +│ [Open File] [Open Folder] [Export Again] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +Clicking **Run in Background** detaches the task from the wizard UI and adds it to the Task Manager. The user can navigate away and return later to check progress. + +--- + +### 2. Data Import + +Import data from a file into a database table. + +#### Wizard Steps + +**Step 1 — File Selection** + +``` +┌─ Step 1: Select File ─────────────────────────────────────────────────────┐ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ │ │ +│ │ ┌──────────┐ │ │ +│ │ │ 📄 ↑ │ Drag & drop a file here │ │ +│ │ └──────────┘ or click to browse │ │ +│ │ │ │ +│ │ Supported: CSV, JSONL, SQL, Excel (.xlsx) │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ── After file selected ── │ +│ │ +│ File: users_export.csv (45.2 KB) │ +│ Detected Format: CSV │ +│ Detected Encoding: UTF-8 │ +│ Rows (estimated): 1,000 │ +│ │ +│ Parse settings auto-detected. Adjust only if needed: │ +│ ▸ Advanced Parse Options │ +│ ┌─ (expanded, only when clicked) ───────────────────────────────────┐ │ +│ │ Delimiter: [▼ Comma (,) ] (auto-detected) │ │ +│ │ Encoding: [▼ UTF-8 ] (auto-detected) │ │ +│ │ ☑ First row is header │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 2 — Target & Column Mapping** + +``` +┌─ Step 2: Target & Mapping ────────────────────────────────────────────────┐ +│ │ +│ Connection: [▼ my-postgres-server ] │ +│ Database: [▼ mydb ] │ +│ Schema: [▼ public ] │ +│ Table: [▼ users ] │ +│ ☐ Create table if not exists │ +│ │ +│ ┌─ Column Mapping ──────────────────────────────────────────────────┐ │ +│ │ │ │ +│ │ Source Column → Target Column Type Status │ │ +│ │ ───────────────────────────────────────────────────────────── │ │ +│ │ id → [▼ id ] INTEGER ✓ Mapped │ │ +│ │ name → [▼ name ] VARCHAR(255) ✓ Mapped │ │ +│ │ email → [▼ email ] VARCHAR(255) ✓ Mapped │ │ +│ │ created_at → [▼ created_at ] TIMESTAMP ✓ Mapped │ │ +│ │ phone → [▼ (skip) ] — ⊘ Skipped │ │ +│ │ │ │ +│ │ [Auto-Map by Name] [Clear All] │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [← Back] [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 3 — Options & Preview** + +``` +┌─ Step 3: Options & Preview ───────────────────────────────────────────────┐ +│ │ +│ ┌─ Import Options ──────────────────────────────────────────────────┐ │ +│ │ On Conflict: [▼ Skip duplicates ] │ │ +│ │ ├─ Skip duplicates │ │ +│ │ ├─ Replace existing │ │ +│ │ ├─ Update existing (upsert) │ │ +│ │ └─ Abort on error │ │ +│ │ Batch Size: [5000 ] rows per transaction │ │ +│ │ ☐ Truncate table before import │ │ +│ │ ☐ Dry run (validate without inserting) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ Data Preview (first 5 rows as mapped) ───────────────────────────┐ │ +│ │ id │ name │ email │ created_at │ │ +│ │ ────┼───────────┼─────────────────────┼────────────────────── │ │ +│ │ 1 │ Alice │ alice@example.com │ 2024-01-15 10:30:00 │ │ +│ │ 2 │ Bob │ bob@example.com │ 2024-01-16 14:22:00 │ │ +│ │ 3 │ Charlie │ charlie@ex... │ 2024-01-17 09:15:00 │ │ +│ │ ... │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [← Back] [Import →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +> **Simplification**: "Disable indexes during import" option has been removed — the backend automatically handles index optimization for large imports (>10k rows) when the engine supports it. + +**Step 4 — Execution** + +``` +┌─ Step 4: Importing ───────────────────────────────────────────────────────┐ +│ │ +│ Status: Importing... │ +│ │ +│ ████████████████████░░░░░░░░░░ 65% │ +│ │ +│ Rows imported: 650 / 1,000 │ +│ Rows skipped: 3 (duplicates) │ +│ Errors: 0 │ +│ Elapsed: 4.1s │ +│ │ +│ [Run in Background] [Cancel] │ +│ │ +│ ── After completion ── │ +│ │ +│ ✓ Import completed successfully │ +│ Rows imported: 997 | Skipped: 3 | Errors: 0 │ +│ Duration: 6.2s │ +│ │ +│ ┌─ Error Log (if any) ─────────────────────────────────────────────┐ │ +│ │ Row 45: Duplicate key violation on column 'email' │ │ +│ │ Row 102: NULL value for non-nullable column 'name' │ │ +│ │ Row 339: Data truncation on column 'phone' (max 20 chars) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [View Table] [Import Again] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +### 3. Structure + +Two sub-tabs: **Generate DDL** and **Run SQL File**. + +``` +┌─ Structure ───────────────────────────────────────────────────────────────┐ +│ ┌──────────────┬────────────────┐ │ +│ │ Generate DDL │ Run SQL File │ │ +│ └──────────────┴────────────────┘ │ +│ (sub-tab content below) │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +#### 3A. Generate DDL + +Generate DDL (Data Definition Language) scripts from existing database objects. + +**Step 1 — Object Selection** + +``` +┌─ Step 1: Select Objects ──────────────────────────────────────────────────┐ +│ │ +│ Connection: [▼ my-postgres-server ] │ +│ Database: [▼ mydb ] │ +│ Schema: [▼ public ] │ +│ │ +│ ┌─ Objects ─────────────────────────────────────────────────────────┐ │ +│ │ ☑ users TABLE 12 columns 1,200 rows │ │ +│ │ ☑ orders TABLE 8 columns 45,000 rows │ │ +│ │ ☐ order_items TABLE 6 columns 120,000 rows │ │ +│ │ ☐ products TABLE 10 columns 500 rows │ │ +│ │ ☑ user_summary_view VIEW 5 columns — │ │ +│ │ │ │ +│ │ [Select All] [Deselect All] [Tables Only] [Views Only] │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 2 — DDL Options** + +``` +┌─ Step 2: DDL Options ─────────────────────────────────────────────────────┐ +│ │ +│ Target Engine: [▼ Same as source (PostgreSQL) ] │ +│ │ +│ ┌─ Include ─────────────────────────────────────────────────────────┐ │ +│ │ ☑ CREATE TABLE statements │ │ +│ │ ☑ Primary keys │ │ +│ │ ☑ Foreign keys │ │ +│ │ ☑ Indexes │ │ +│ │ ☑ Constraints (UNIQUE, CHECK, NOT NULL) │ │ +│ │ ☐ Comments / descriptions │ │ +│ │ ☐ Tablespace / storage options │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ Behavior ────────────────────────────────────────────────────────┐ │ +│ │ ☑ Include DROP IF EXISTS before CREATE │ │ +│ │ ☑ Include IF NOT EXISTS on CREATE │ │ +│ │ ☐ Include INSERT DATA (export structure + data) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [← Back] [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 3 — Preview & Export** + +``` +┌─ Step 3: Preview & Export ────────────────────────────────────────────────┐ +│ │ +│ Objects: 3 selected | Target: PostgreSQL │ +│ │ +│ ┌─ DDL Preview (Monaco editor, read-only) ──────────────────────────┐ │ +│ │ -- Generated by SQLKit on 2024-03-15 │ │ +│ │ -- Source: my-postgres-server / mydb / public │ │ +│ │ │ │ +│ │ DROP TABLE IF EXISTS "users" CASCADE; │ │ +│ │ CREATE TABLE IF NOT EXISTS "users" ( │ │ +│ │ "id" SERIAL PRIMARY KEY, │ │ +│ │ "name" VARCHAR(255) NOT NULL, │ │ +│ │ "email" VARCHAR(255) NOT NULL UNIQUE, │ │ +│ │ "created_at" TIMESTAMP WITH TIME ZONE DEFAULT NOW() │ │ +│ │ ); │ │ +│ │ │ │ +│ │ CREATE INDEX "idx_users_email" ON "users" ("email"); │ │ +│ │ ... │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Copy to Clipboard] [Save to File...] [Execute on Server] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +#### 3B. Run SQL File + +Open and execute SQL files against a database connection. + +**Step 1 — File & Connection** + +``` +┌─ Step 1: Select File & Connection ────────────────────────────────────────┐ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ ┌──────────┐ │ │ +│ │ │ 📄 ↑ │ Drag & drop a .sql file here │ │ +│ │ └──────────┘ or click to browse │ │ +│ │ │ │ +│ │ Supported: .sql files │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ File: schema_backup.sql (128 KB, 2,400 statements) │ +│ │ +│ Connection: [▼ my-postgres-server ] │ +│ Database: [▼ mydb ] │ +│ │ +│ ┌─ Execution Options ───────────────────────────────────────────────┐ │ +│ │ ☑ Wrap in transaction │ │ +│ │ On Error: [▼ Rollback all ] │ │ +│ │ ├─ Rollback all │ │ +│ │ ├─ Skip and continue │ │ +│ │ └─ Stop execution │ │ +│ │ ☐ Dry run (parse only, don't execute) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ File Preview (first 50 lines) ───────────────────────────────────┐ │ +│ │ -- Database schema backup │ │ +│ │ CREATE TABLE users ( ... ); │ │ +│ │ CREATE TABLE orders ( ... ); │ │ +│ │ INSERT INTO users VALUES (1, 'Alice', ...); │ │ +│ │ ... │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Execute →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 2 — Execution & Results** + +``` +┌─ Step 2: Execution ───────────────────────────────────────────────────────┐ +│ │ +│ Status: Executing... │ +│ │ +│ ████████████████████░░░░░░░░░░ 65% │ +│ │ +│ Statements: 1,560 / 2,400 │ +│ Succeeded: 1,558 | Failed: 2 │ +│ Elapsed: 12.4s │ +│ │ +│ [Run in Background] [Cancel] │ +│ │ +│ ── After completion ── │ +│ │ +│ ✓ Execution completed │ +│ Total: 2,400 | Succeeded: 2,398 | Failed: 2 │ +│ Duration: 19.1s │ +│ │ +│ ┌─ Execution Log ──────────────────────────────────────────────────┐ │ +│ │ ✓ Statement 1: CREATE TABLE users — OK │ │ +│ │ ✓ Statement 2: CREATE TABLE orders — OK │ │ +│ │ ✗ Statement 145: INSERT INTO ... — ERROR: duplicate key │ │ +│ │ ✗ Statement 892: ALTER TABLE ... — ERROR: column exists │ │ +│ │ ... │ │ +│ │ [Show Errors Only] [Copy Log] │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Run Again] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +### 4. Data Migration + +Cross-engine data migration (e.g., MySQL to PostgreSQL). + +#### Wizard Steps + +**Step 1 — Source Connection** + +``` +┌─ Step 1: Source ──────────────────────────────────────────────────────────┐ +│ │ +│ Source Connection: [▼ my-mysql-server ] │ +│ Database: [▼ production_db ] │ +│ Schema: [▼ (default) ] │ +│ │ +│ Available Tables: │ +│ ☑ users TABLE 12 columns 1,200 rows │ +│ ☑ orders TABLE 8 columns 45,000 rows │ +│ ☑ products TABLE 10 columns 500 rows │ +│ ☐ audit_log TABLE 6 columns 2,000,000 rows │ +│ ☐ temp_data TABLE 3 columns 50 rows │ +│ │ +│ [Select All] [Deselect All] │ +│ │ +│ Selected: 3 tables, ~46,700 rows │ +│ │ +│ [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 2 — Target Connection** + +``` +┌─ Step 2: Target ──────────────────────────────────────────────────────────┐ +│ │ +│ Target Connection: [▼ my-postgres-server ] │ +│ Database: [▼ new_production ] │ +│ Schema: [▼ public ] │ +│ │ +│ ☑ Create target tables if not exist │ +│ ☐ Drop target tables before migration │ +│ │ +│ Migration Direction: │ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ MySQL │ ───→ │ PostgreSQL │ │ +│ │ production │ │ new_prod │ │ +│ │ 3 tables │ │ public │ │ +│ └─────────────┘ └─────────────┘ │ +│ │ +│ [← Back] [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 3 — Schema & Type Mapping** + +``` +┌─ Step 3: Schema & Type Mapping ───────────────────────────────────────────┐ +│ │ +│ ┌─ Table: users ────────────────────────────────────────────────────┐ │ +│ │ │ │ +│ │ Source (MySQL) → Target (PostgreSQL) Status │ │ +│ │ ───────────────────────────────────────────────────────────── │ │ +│ │ id INT AUTO_INCREMENT → id SERIAL ✓ Auto │ │ +│ │ name VARCHAR(255) → name VARCHAR(255) ✓ Auto │ │ +│ │ email VARCHAR(255) → email VARCHAR(255) ✓ Auto │ │ +│ │ bio TEXT → bio TEXT ✓ Auto │ │ +│ │ data JSON → data JSONB ⚠ Mapped │ │ +│ │ created DATETIME → created TIMESTAMP ⚠ Mapped │ │ +│ │ active TINYINT(1) → active BOOLEAN ⚠ Mapped │ │ +│ │ │ │ +│ │ [Edit Mapping] [Reset to Auto] │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ Table: orders ───────────────────────────────────────────────────┐ │ +│ │ (similar mapping grid) │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ⚠ 5 columns require type conversion (auto-mapped) │ +│ ✓ 19 columns map directly │ +│ │ +│ [← Back] [Next →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 4 — Configure** + +``` +┌─ Step 4: Configure ───────────────────────────────────────────────────────┐ +│ │ +│ ┌─ Migration Options ───────────────────────────────────────────────┐ │ +│ │ Batch Size: [5000 ] rows per batch │ │ +│ │ On Error: [▼ Skip row and continue ] │ │ +│ │ ☑ Migrate indexes │ │ +│ │ ☑ Migrate foreign keys │ │ +│ │ ☑ Migrate constraints │ │ +│ │ ☐ Disable foreign key checks during migration │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ Migration Plan Summary ──────────────────────────────────────────┐ │ +│ │ Source: MySQL (my-mysql-server / production_db) │ │ +│ │ Target: PostgreSQL (my-postgres-server / new_production / public) │ │ +│ │ Tables: 3 │ │ +│ │ Total Rows: ~46,700 │ │ +│ │ Type Conversions: 5 │ │ +│ │ Estimated Time: ~30 seconds │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [← Back] [Start Migration →] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +**Step 5 — Execution** + +``` +┌─ Step 5: Migrating ───────────────────────────────────────────────────────┐ +│ │ +│ Overall Progress: │ +│ ████████████████░░░░░░░░░░░░░░ 52% │ +│ │ +│ ┌─ Per-Table Progress ──────────────────────────────────────────────┐ │ +│ │ ✓ users 1,200 / 1,200 ████████████████████ 100% 1.2s │ │ +│ │ ● orders 18,500 / 45,000 ████████░░░░░░░░░░░░ 41% 8.3s │ │ +│ │ ○ products 0 / 500 ░░░░░░░░░░░░░░░░░░░░ 0% — │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Phase: Migrating data (2/3 tables) │ +│ Errors: 0 | Elapsed: 9.5s | Remaining: ~8s │ +│ │ +│ [Run in Background] [Cancel] │ +│ │ +│ ── After completion ── │ +│ │ +│ ✓ Migration completed successfully │ +│ │ +│ ┌─ Results ─────────────────────────────────────────────────────────┐ │ +│ │ Table Rows Status Duration │ │ +│ │ ────────────────────────────────────────────── │ │ +│ │ users 1,200 ✓ OK 1.2s │ │ +│ │ orders 45,000 ✓ OK 14.8s │ │ +│ │ products 500 ✓ OK 0.3s │ │ +│ │ ────────────────────────────────────────────── │ │ +│ │ Total 46,700 16.3s │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ [Migrate Again] │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Background Task System + +All long-running transfer operations (export, import, SQL file execution, migration) run as **background tasks**. This allows users to navigate away from the Transfer page and return later to check progress or restore task state. + +**Architecture**: Task management is **frontend-only** (no Rust backend task registry). The Tauri commands block until completion, but the frontend wraps each invocation in an async call tracked by the Pinia store. Progress is synced from Tauri events to the corresponding task entry. + +### Task Lifecycle + +``` +User clicks "Export" / "Import" / etc. + ↓ +Frontend creates BackgroundTask { id, kind, status: 'running', config: snapshot } + ↓ +Frontend calls Tauri command (async, non-blocking from UI perspective) + ↓ +Tauri command emits progress events → store.syncProgressToTask(taskId) + ↓ +User clicks "Run in Background" → store.detachActiveTask(kind) + ↓ +User navigates away (task continues running in store) + ↓ +User opens Task Manager → sees task card with live progress + ↓ +User clicks "Go to task" → router navigates to /transfer?tab=export&taskId=xyz + ↓ +store.openTask(taskId) → restores full form state from task.config + ↓ +Tauri command completes → store.updateTaskStatus(taskId, 'completed') +``` + +### Task Manager Panel + +The Task Manager is a slide-out sidebar panel accessible from the Transfer page header. It shows all running and completed tasks. + +``` +┌─ Task Manager ──────────────────────────────────┐ +│ Tasks (3) [Clear Completed] │ +│ │ +│ ┌─ Export ─────────────────────────────────┐ │ +│ │ 📤 Export users → CSV │ │ +│ │ ● Running │ │ +│ │ ████████████████░░░░ 78% │ │ +│ │ 780 / 1,000 rows │ │ +│ │ Started: 2 min ago │ │ +│ │ [Go to Task →] │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ ┌─ Import ─────────────────────────────────┐ │ +│ │ 📥 Import orders.csv │ │ +│ │ ✓ Completed │ │ +│ │ ████████████████████ 100% │ │ +│ │ 45,000 rows | 3 skipped │ │ +│ │ Duration: 12.4s │ │ +│ │ [Dismiss] [Go to Task →] │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ ┌─ Migration ──────────────────────────────┐ │ +│ │ 🔄 MySQL → PostgreSQL (3 tables) │ │ +│ │ ✗ Failed │ │ +│ │ ████████████░░░░░░░░ 58% │ │ +│ │ Error: Connection lost to target │ │ +│ │ [Dismiss] [Go to Task →] │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────┘ +``` + +**Status colors**: Running = blue, Completed = green, Failed = red, Pending = yellow. + +**Task card actions**: +- **Go to Task**: Navigates to the Transfer page with the task's tab active and restores the full wizard state from the task's config snapshot +- **Dismiss**: Removes the task card (only shown for non-running tasks) +- **Clear Completed**: Removes all completed/failed task cards + +### Task Types + +```typescript +// src/types/transfer.ts (additions) + +export type TaskKind = 'export' | 'import' | 'sqlFile' | 'migration' + +export type TaskStatus = 'pending' | 'running' | 'completed' | 'failed' + +export type TaskRuntime = { + complete: number + total: number + skipped: number + errorCount: number +} + +export type ExportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + columns: string[] + whereClause?: string + orderBy?: string + limit?: number + format: ExportFormat + outputPath: string +} + +export type ImportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + filePath: string + format: ImportFormat + conflictStrategy: ConflictStrategy +} + +export type SqlFileTaskConfig = { + connectionId: string + database?: string + filePath: string + onError: SqlFileErrorStrategy +} + +export type MigrationTaskConfig = { + sourceConnectionId: string + sourceDatabase?: string + targetConnectionId: string + targetDatabase?: string + tables: string[] +} + +export type TaskConfig = ExportTaskConfig | ImportTaskConfig | SqlFileTaskConfig | MigrationTaskConfig + +export type BackgroundTask = { + id: string + kind: TaskKind + status: TaskStatus + progress: { complete: number; total: number } + config: TaskConfig + runtime: TaskRuntime + label: string + startTime: Date + endTime?: Date + error?: string +} +``` + +### Store Task Management Methods + +```typescript +// Added to useTransferStore (see Pinia Store section below) + +// ── Task State ────────────────────────────────── +const runningTasks = ref([]) +const activeExportTaskId = ref(null) +const activeImportTaskId = ref(null) +const activeSqlFileTaskId = ref(null) +const activeMigrationTaskId = ref(null) + +// ── Task Getters ──────────────────────────────── +const taskCount = computed(() => runningTasks.value.length) +const hasRunningTasks = computed(() => + runningTasks.value.some(t => t.status === 'running') +) + +// ── Task Actions ──────────────────────────────── +const addRunningTask = (task: BackgroundTask) => { + runningTasks.value = [...runningTasks.value, task] +} + +const updateTaskRuntime = (taskId: string, runtime: Partial) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { ...t, runtime: { ...t.runtime, ...runtime }, progress: { complete: runtime.complete ?? t.progress.complete, total: runtime.total ?? t.progress.total } } + : t + ) +} + +const updateTaskStatus = (taskId: string, status: TaskStatus, error?: string) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { ...t, status, endTime: status === 'completed' || status === 'failed' ? new Date() : undefined, error } + : t + ) +} + +const removeTask = (taskId: string) => { + runningTasks.value = runningTasks.value.filter(t => t.id !== taskId) +} + +const clearCompletedTasks = () => { + runningTasks.value = runningTasks.value.filter(t => + t.status === 'running' || t.status === 'pending' + ) +} + +const openTask = (taskId: string) => { + const task = runningTasks.value.find(t => t.id === taskId) + if (!task) return + // Restore form state from task.config back to the active wizard + // Implementation depends on task.kind — sets the appropriate step fields + activeTab.value = taskKindToTab(task.kind) + // ... restore config fields to wizard state +} + +const detachActiveTask = (kind: TaskKind) => { + // Clears the active task ID without stopping the operation + // Allows user to navigate away while task continues + switch (kind) { + case 'export': activeExportTaskId.value = null; break + case 'import': activeImportTaskId.value = null; break + case 'sqlFile': activeSqlFileTaskId.value = null; break + case 'migration': activeMigrationTaskId.value = null; break + } +} + +const syncProgressToTask = (taskId: string, progress: TransferProgress) => { + updateTaskRuntime(taskId, { + complete: progress.processedRows, + total: progress.totalRows ?? 0, + skipped: progress.skippedRows, + errorCount: progress.errorCount, + }) +} +``` + +### Task Creation Flow (Export Example) + +```typescript +// In ExportExecuteStep.vue + +import { ulid } from 'ulidx' + +const handleStartExport = async () => { + const taskId = ulid() + const configSnapshot: ExportTaskConfig = { + connectionId: transferStore.exportRequest.connectionId!, + database: transferStore.exportRequest.database, + schema: transferStore.exportRequest.schema, + table: transferStore.exportRequest.source.table, + columns: transferStore.exportRequest.source.columns, + format: transferStore.exportRequest.format!, + outputPath: transferStore.exportRequest.outputPath!, + } + + transferStore.addRunningTask({ + id: taskId, + kind: 'export', + status: 'running', + progress: { complete: 0, total: estimatedRows.value }, + config: configSnapshot, + runtime: { complete: 0, total: estimatedRows.value, skipped: 0, errorCount: 0 }, + label: `Export ${configSnapshot.table} → ${configSnapshot.format.toUpperCase()}`, + startTime: new Date(), + }) + transferStore.activeExportTaskId = taskId + + try { + const result = await executeExport(transferStore.exportRequest as ExportRequest) + transferStore.updateTaskStatus(taskId, 'completed') + transferStore.completeOperation(result) + } catch (err) { + transferStore.updateTaskStatus(taskId, 'failed', String(err)) + } +} +``` + +### Task Manager Navigation + +When the user clicks **"Go to Task"** in the Task Manager, the router navigates with query params: + +```typescript +router.push({ + path: '/transfer', + query: { tab: task.kind, taskId: task.id }, +}) +``` + +The `TransferPage.vue` watches for `taskId` in the route query and calls `transferStore.openTask(taskId)` to restore the wizard to the task's state (showing progress or results). + +--- + +## Cross-Engine Type Mapping Matrix + +The migration feature requires automatic type mapping between database engines. Below is the mapping matrix used for cross-engine translation. + +### PostgreSQL ↔ MySQL + +| PostgreSQL | MySQL | Notes | +|------------|-------|-------| +| `SERIAL` | `INT AUTO_INCREMENT` | Auto-increment PK | +| `BIGSERIAL` | `BIGINT AUTO_INCREMENT` | Large auto-increment PK | +| `SMALLINT` | `SMALLINT` | Direct | +| `INTEGER` | `INT` | Direct | +| `BIGINT` | `BIGINT` | Direct | +| `REAL` | `FLOAT` | Direct | +| `DOUBLE PRECISION` | `DOUBLE` | Direct | +| `NUMERIC(p,s)` | `DECIMAL(p,s)` | Direct | +| `BOOLEAN` | `TINYINT(1)` | MySQL lacks native boolean | +| `VARCHAR(n)` | `VARCHAR(n)` | Direct | +| `TEXT` | `TEXT` / `LONGTEXT` | TEXT if ≤64KB, LONGTEXT otherwise | +| `CHAR(n)` | `CHAR(n)` | Direct | +| `BYTEA` | `LONGBLOB` | Binary data | +| `TIMESTAMP` | `DATETIME` | MySQL DATETIME lacks timezone | +| `TIMESTAMPTZ` | `DATETIME` | Timezone info lost | +| `DATE` | `DATE` | Direct | +| `TIME` | `TIME` | Direct | +| `INTERVAL` | `VARCHAR(255)` | No MySQL equivalent | +| `JSON` | `JSON` | Direct | +| `JSONB` | `JSON` | MySQL lacks binary JSON | +| `UUID` | `CHAR(36)` | MySQL lacks native UUID | +| `INET` | `VARCHAR(45)` | No MySQL equivalent | +| `CIDR` | `VARCHAR(45)` | No MySQL equivalent | +| `MACADDR` | `VARCHAR(17)` | No MySQL equivalent | +| `ARRAY` | `JSON` | MySQL lacks native arrays | +| `POINT` | `POINT` | Spatial type (both support) | + +### PostgreSQL ↔ SQLite + +| PostgreSQL | SQLite | Notes | +|------------|--------|-------| +| `SERIAL` / `BIGSERIAL` | `INTEGER PRIMARY KEY` | SQLite auto-increment via ROWID | +| `SMALLINT` / `INTEGER` / `BIGINT` | `INTEGER` | SQLite has single integer type | +| `REAL` / `DOUBLE PRECISION` | `REAL` | Direct | +| `NUMERIC(p,s)` | `REAL` | SQLite lacks fixed-point | +| `BOOLEAN` | `INTEGER` | 0/1 convention | +| `VARCHAR(n)` / `TEXT` | `TEXT` | SQLite ignores length constraints | +| `BYTEA` | `BLOB` | Direct | +| `TIMESTAMP` / `TIMESTAMPTZ` | `TEXT` | ISO 8601 string | +| `DATE` / `TIME` | `TEXT` | ISO 8601 string | +| `JSON` / `JSONB` | `TEXT` | Plain text storage | +| `UUID` | `TEXT` | 36-char string | + +### PostgreSQL ↔ SQL Server + +| PostgreSQL | SQL Server | Notes | +|------------|------------|-------| +| `SERIAL` | `INT IDENTITY(1,1)` | Auto-increment | +| `BIGSERIAL` | `BIGINT IDENTITY(1,1)` | Large auto-increment | +| `SMALLINT` | `SMALLINT` | Direct | +| `INTEGER` | `INT` | Direct | +| `BIGINT` | `BIGINT` | Direct | +| `REAL` | `REAL` | Direct | +| `DOUBLE PRECISION` | `FLOAT` | Direct | +| `NUMERIC(p,s)` | `DECIMAL(p,s)` | Direct | +| `BOOLEAN` | `BIT` | 0/1 | +| `VARCHAR(n)` | `NVARCHAR(n)` | Unicode by default | +| `TEXT` | `NVARCHAR(MAX)` | Unicode by default | +| `CHAR(n)` | `NCHAR(n)` | Unicode by default | +| `BYTEA` | `VARBINARY(MAX)` | Binary data | +| `TIMESTAMP` | `DATETIME2` | Higher precision | +| `TIMESTAMPTZ` | `DATETIMEOFFSET` | With timezone | +| `DATE` | `DATE` | Direct | +| `TIME` | `TIME` | Direct | +| `JSON` / `JSONB` | `NVARCHAR(MAX)` | SQL Server lacks native JSON type | +| `UUID` | `UNIQUEIDENTIFIER` | Native support | +| `XML` | `XML` | Native support | + +### MySQL ↔ SQL Server + +| MySQL | SQL Server | Notes | +|-------|------------|-------| +| `INT AUTO_INCREMENT` | `INT IDENTITY(1,1)` | Auto-increment | +| `TINYINT(1)` | `BIT` | Boolean | +| `TINYINT` | `TINYINT` | Direct | +| `VARCHAR(n)` | `NVARCHAR(n)` | Unicode | +| `TEXT` | `NVARCHAR(MAX)` | Unicode | +| `LONGTEXT` | `NVARCHAR(MAX)` | Unicode | +| `BLOB` / `LONGBLOB` | `VARBINARY(MAX)` | Binary | +| `DATETIME` | `DATETIME2` | Higher precision | +| `TIMESTAMP` | `DATETIME2` | Meaning differs | +| `JSON` | `NVARCHAR(MAX)` | No native type | +| `ENUM(...)` | `NVARCHAR(255)` + CHECK | No native ENUM | +| `SET(...)` | `NVARCHAR(MAX)` | No native SET | + +--- + +## Backend Architecture + +### New Module Structure + +``` +src-tauri/src/ +├── transfer/ # New module +│ ├── mod.rs # Module exports +│ ├── types.rs # Transfer-specific types +│ ├── defaults.rs # Best-practice default configs per format +│ ├── export.rs # Export logic (CSV, JSONL, SQL, Excel) +│ ├── import.rs # Import logic (parse, validate, insert) +│ ├── ddl.rs # DDL generation trait + implementations +│ ├── migration.rs # Cross-engine migration orchestrator +│ ├── type_mapping.rs # Cross-engine type mapping matrix +│ └── progress.rs # Progress reporting via Tauri events +├── commands/ +│ ├── transfer.rs # New: Tauri commands for transfer +│ └── ... +└── ... +``` + +### Rust Types + +```rust +// transfer/types.rs + +use serde::{Deserialize, Serialize}; + +// ── Export ──────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExportFormat { + Csv, + Jsonl, + Sql, + Excel, +} + +/// CSV options — all fields have sensible defaults. +/// The frontend sends these only when the user explicitly overrides via "Advanced Options". +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CsvExportOptions { + #[serde(default = "default_comma")] + pub delimiter: char, + #[serde(default = "default_double_quote")] + pub quote_char: char, + #[serde(default = "default_utf8")] + pub encoding: String, + #[serde(default = "default_true")] + pub include_header: bool, + #[serde(default)] + pub quote_all: bool, + #[serde(default = "default_lf")] + pub line_ending: String, +} + +/// JSONL (JSON Lines) options — one JSON object per line, optimized for large datasets. +/// Simpler than JSON: no structure choice, no pretty print (always compact, one line per record). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JsonlExportOptions { + #[serde(default = "default_iso8601")] + pub date_format: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlExportOptions { + pub target_table: String, + #[serde(default = "default_batch_size")] + pub batch_size: u32, + #[serde(default = "default_true")] + pub include_create_table: bool, + #[serde(default)] + pub include_drop_table: bool, + pub target_engine: Option, +} + +/// Excel options — best-practice defaults, no user customization needed. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExcelExportOptions { + #[serde(default = "default_sheet_name")] + pub sheet_name: String, + #[serde(default = "default_true")] + pub include_header: bool, + #[serde(default = "default_true")] + pub auto_fit_columns: bool, + #[serde(default = "default_true")] + pub freeze_header: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub source: ExportSource, + pub format: ExportFormat, + pub csv_options: Option, + pub jsonl_options: Option, + pub sql_options: Option, + pub excel_options: Option, + pub output_path: String, +} + +/// Export source is always a table (Custom Query removed for simplicity). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportSource { + pub table: String, + pub columns: Vec, + pub where_clause: Option, + pub order_by: Option, + pub limit: Option, +} + +// ── Import ──────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ImportFormat { + Csv, + Jsonl, + Sql, + Excel, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnMapping { + pub source_column: String, + pub target_column: Option, // None = skip + pub target_type: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ConflictStrategy { + Skip, + Replace, + Upsert, + Abort, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub table: String, + pub file_path: String, + pub format: ImportFormat, + pub column_mappings: Vec, + pub conflict_strategy: ConflictStrategy, + #[serde(default = "default_batch_size")] + pub batch_size: u32, + pub create_table: bool, + pub truncate_before: bool, + pub dry_run: bool, + pub csv_options: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CsvImportOptions { + #[serde(default = "default_comma")] + pub delimiter: char, + #[serde(default = "default_utf8")] + pub encoding: String, + #[serde(default = "default_true")] + pub has_header: bool, +} + +// ── DDL ─────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DdlRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub tables: Vec, + pub target_engine: Option, + pub options: DdlOptions, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DdlOptions { + pub include_create_table: bool, + pub include_primary_keys: bool, + pub include_foreign_keys: bool, + pub include_indexes: bool, + pub include_constraints: bool, + pub include_comments: bool, + pub include_storage: bool, + pub include_drop_if_exists: bool, + pub include_if_not_exists: bool, + pub include_data: bool, +} + +// ── Run SQL File ────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RunSqlFileRequest { + pub connection_id: String, + pub database: Option, + pub file_path: String, + pub wrap_in_transaction: bool, + pub on_error: SqlFileErrorStrategy, + pub dry_run: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SqlFileErrorStrategy { + Rollback, + SkipAndContinue, + Stop, +} + +// ── Migration ───────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationMapping { + pub source_column: String, + pub source_type: String, + pub target_column: String, + pub target_type: String, + pub conversion: MigrationConversion, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MigrationConversion { + Direct, // Types are compatible, no conversion needed + Mapped, // Automatic type mapping applied + Custom, // User-defined mapping +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationTablePlan { + pub source_table: String, + pub target_table: String, + pub column_mappings: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationRequest { + pub source_connection_id: String, + pub source_database: Option, + pub source_schema: Option, + pub target_connection_id: String, + pub target_database: Option, + pub target_schema: Option, + pub table_plans: Vec, + pub batch_size: u32, + pub on_error: MigrationErrorStrategy, + pub create_tables: bool, + pub drop_tables: bool, + pub migrate_indexes: bool, + pub migrate_foreign_keys: bool, + pub migrate_constraints: bool, + pub disable_fk_checks: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MigrationErrorStrategy { + SkipRow, + SkipTable, + Abort, +} + +// ── Progress ────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferProgress { + pub operation: String, // "export" | "import" | "ddl" | "sql_file" | "migration" + pub phase: String, // "preparing" | "processing" | "finalizing" + pub current_table: Option, + pub total_rows: Option, + pub processed_rows: u64, + pub skipped_rows: u64, + pub error_count: u64, + pub percent: f32, // 0.0–100.0 + pub elapsed_ms: u64, + pub estimated_remaining_ms: Option, + pub message: Option, +} + +// ── Results ─────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferResult { + pub success: bool, + pub total_rows: u64, + pub processed_rows: u64, + pub skipped_rows: u64, + pub error_count: u64, + pub duration_ms: u64, + pub output_path: Option, + pub output_size_bytes: Option, + pub errors: Vec, + pub table_results: Option>, // For migration +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableTransferResult { + pub table: String, + pub rows: u64, + pub success: bool, + pub duration_ms: u64, + pub errors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferError { + pub row_number: Option, + pub statement_number: Option, + pub message: String, + pub sql: Option, +} + +// ── Preview / Detection ─────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileDetectionResult { + pub format: ImportFormat, + pub encoding: String, + pub estimated_rows: Option, + pub file_size_bytes: u64, + pub columns: Vec, + pub csv_delimiter: Option, + pub has_header: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportPreview { + pub columns: Vec, + pub sample_rows: Vec>, + pub total_rows_estimate: Option, + pub formatted_preview: String, // First N rows in target format +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TypeMappingSuggestion { + pub source_column: String, + pub source_type: String, + pub target_type: String, + pub conversion: MigrationConversion, + pub warning: Option, // e.g., "Precision loss possible" +} + +// ── Default value functions ─────────────────────────────── + +fn default_comma() -> char { ',' } +fn default_double_quote() -> char { '"' } +fn default_utf8() -> String { "UTF-8".to_string() } +fn default_lf() -> String { "lf".to_string() } +fn default_true() -> bool { true } +fn default_iso8601() -> String { "iso8601".to_string() } +fn default_batch_size() -> u32 { 1000 } +fn default_sheet_name() -> String { "Sheet1".to_string() } +``` + +### Best-Practice Defaults + +```rust +// transfer/defaults.rs + +use super::types::*; + +/// Returns default CSV export options (best-practice). +pub fn csv_defaults() -> CsvExportOptions { + CsvExportOptions { + delimiter: ',', + quote_char: '"', + encoding: "UTF-8".to_string(), + include_header: true, + quote_all: false, + line_ending: "lf".to_string(), + } +} + +/// Returns default JSONL export options (best-practice). +/// JSONL is always compact (one JSON object per line), no structure/pretty-print choice. +pub fn jsonl_defaults() -> JsonlExportOptions { + JsonlExportOptions { + date_format: "iso8601".to_string(), + } +} + +/// Returns default SQL export options (best-practice). +pub fn sql_defaults(table_name: &str) -> SqlExportOptions { + SqlExportOptions { + target_table: table_name.to_string(), + batch_size: 1000, + include_create_table: true, + include_drop_table: false, + target_engine: None, + } +} + +/// Returns default Excel export options (best-practice). +pub fn excel_defaults() -> ExcelExportOptions { + ExcelExportOptions { + sheet_name: "Sheet1".to_string(), + include_header: true, + auto_fit_columns: true, + freeze_header: true, + } +} + +/// Returns default CSV import options (best-practice). +pub fn csv_import_defaults() -> CsvImportOptions { + CsvImportOptions { + delimiter: ',', + encoding: "UTF-8".to_string(), + has_header: true, + } +} +``` + +### DDL Generator Trait + +```rust +// transfer/ddl.rs + +use async_trait::async_trait; +use crate::database::types::ColumnInfo; + +#[async_trait] +pub trait DdlGenerator: Send + Sync { + /// Generate CREATE TABLE statement for the given columns. + fn generate_create_table( + &self, + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, + ) -> String; + + /// Generate DROP TABLE statement. + fn generate_drop_table( + &self, + schema: Option<&str>, + table: &str, + cascade: bool, + ) -> String; + + /// Generate CREATE INDEX statements. + fn generate_indexes( + &self, + schema: Option<&str>, + table: &str, + indexes: &[IndexInfo], + ) -> Vec; + + /// Map a source column type to this engine's equivalent. + fn map_type(&self, source_type: &str, source_engine: &str) -> String; + + /// Generate INSERT statement for a batch of rows. + fn generate_insert( + &self, + schema: Option<&str>, + table: &str, + columns: &[String], + rows: &[Vec], + ) -> String; + + /// Get the engine name (e.g., "PostgreSQL", "MySQL"). + fn engine_name(&self) -> &str; + + /// Quote an identifier for this engine. + fn quote_identifier(&self, name: &str) -> String; +} +``` + +Implementations: +- `PostgresDdlGenerator` +- `MySqlDdlGenerator` +- `SqliteDdlGenerator` +- `SqlServerDdlGenerator` + +### Tauri Commands (12 total) + +```rust +// commands/transfer.rs + +use tauri::State; +use crate::state::AppState; +use crate::transfer::types::*; + +// ── Export Commands ─────────────────────────────────────── + +/// Preview export data (first N rows in target format). +#[tauri::command] +pub async fn preview_export( + request: ExportRequest, + preview_rows: u32, + state: State<'_, AppState>, +) -> Result { ... } + +/// Execute data export to file. +#[tauri::command] +pub async fn execute_export( + request: ExportRequest, + app_handle: tauri::AppHandle, + state: State<'_, AppState>, +) -> Result { ... } + +// ── Import Commands ────────────────────────────────────── + +/// Detect file format, encoding, columns, and delimiter. +#[tauri::command] +pub async fn detect_file( + file_path: String, +) -> Result { ... } + +/// Preview parsed file data (first N rows with column mapping applied). +#[tauri::command] +pub async fn preview_import( + file_path: String, + format: ImportFormat, + csv_options: Option, + preview_rows: u32, +) -> Result { ... } + +/// Execute data import from file into table. +#[tauri::command] +pub async fn execute_import( + request: ImportRequest, + app_handle: tauri::AppHandle, + state: State<'_, AppState>, +) -> Result { ... } + +// ── DDL Commands ───────────────────────────────────────── + +/// Generate DDL for selected objects. +#[tauri::command] +pub async fn generate_ddl( + request: DdlRequest, + state: State<'_, AppState>, +) -> Result { ... } + +/// Execute DDL/SQL against a connection. +#[tauri::command] +pub async fn execute_ddl( + connection_id: String, + database: Option, + sql: String, + state: State<'_, AppState>, +) -> Result { ... } + +// ── Run SQL File Commands ──────────────────────────────── + +/// Parse SQL file and return statement count + preview. +#[tauri::command] +pub async fn parse_sql_file( + file_path: String, +) -> Result { ... } + +/// Execute SQL file against a connection. +#[tauri::command] +pub async fn execute_sql_file( + request: RunSqlFileRequest, + app_handle: tauri::AppHandle, + state: State<'_, AppState>, +) -> Result { ... } + +// ── Migration Commands ─────────────────────────────────── + +/// Suggest type mappings for source→target migration. +#[tauri::command] +pub async fn suggest_type_mappings( + source_connection_id: String, + source_database: Option, + source_schema: Option, + source_tables: Vec, + target_engine: String, + state: State<'_, AppState>, +) -> Result, String> { ... } + +/// Execute cross-engine data migration. +#[tauri::command] +pub async fn execute_migration( + request: MigrationRequest, + app_handle: tauri::AppHandle, + state: State<'_, AppState>, +) -> Result { ... } + +// ── Shared Commands ────────────────────────────────────── + +/// Cancel a running transfer operation. +#[tauri::command] +pub async fn cancel_transfer( + operation_id: String, + state: State<'_, AppState>, +) -> Result<(), String> { ... } +``` + +### Progress Reporting + +Progress is reported via Tauri events so the frontend receives real-time updates without polling. + +```rust +// transfer/progress.rs + +use tauri::Emitter; + +pub const TRANSFER_PROGRESS_EVENT: &str = "transfer-progress"; + +pub fn emit_progress( + app_handle: &tauri::AppHandle, + progress: &TransferProgress, +) { + let _ = app_handle.emit(TRANSFER_PROGRESS_EVENT, progress); +} +``` + +### Recommended Rust Crates + +| Crate | Purpose | Version | +|-------|---------|---------| +| `csv` | CSV reading/writing | Latest | +| `serde_json` | JSONL reading/writing (one JSON object per line) | (already in deps) | +| `calamine` | Excel reading (.xlsx, .xls) | Latest | +| `rust_xlsxwriter` | Excel writing (.xlsx) | Latest | +| `encoding_rs` | Character encoding detection/conversion | Latest | +| `chardetng` | Automatic charset detection | Latest | + +### Streaming / Chunking Strategy + +For large datasets (100k+ rows), all operations use batched processing: + +1. **Export**: Query rows in pages (`LIMIT batch_size OFFSET n`), write each page to the output file incrementally. +2. **Import**: Read the file in chunks (e.g., 5,000 rows per batch), execute batched INSERTs within a transaction per batch. +3. **Migration**: Read from source in batches, type-convert in memory, write to target in batches. Per-table sequential, within-table batched. + +**Default batch size**: 5,000 rows. Configurable by the user. + +**Memory bound**: At most one batch worth of rows is held in memory at any time. + +--- + +## Frontend Architecture + +### Vue Component Hierarchy + +``` +src/ +├── pages/ +│ └── TransferPage.vue # Top-level page (replaces ImportExportPage.vue) +├── components/ +│ └── transfer/ +│ ├── index.ts # Module exports +│ │ +│ ├── TransferTabs.vue # Top-level tab container (Export|Import|Structure|Migration) +│ │ +│ ├── shared/ +│ │ ├── ConnectionSelector.vue # Connection + database + schema dropdowns +│ │ ├── TableSelector.vue # Table list with checkboxes +│ │ ├── ColumnSelector.vue # Column list with checkboxes +│ │ ├── WizardStepper.vue # Step indicator bar +│ │ ├── ProgressPanel.vue # Progress bar + stats + cancel + "Run in Background" +│ │ ├── ResultPanel.vue # Completion summary + error log +│ │ ├── FileDropZone.vue # Drag-and-drop file area +│ │ └── FormatPreview.vue # Preview formatted data (Monaco read-only) +│ │ +│ ├── export/ +│ │ ├── ExportWizard.vue # Export wizard container (4 steps) +│ │ ├── ExportSourceStep.vue # Step 1: Table + columns + filters +│ │ ├── ExportFormatStep.vue # Step 2: Format selection (defaults applied) +│ │ ├── ExportPreviewStep.vue # Step 3: Preview +│ │ └── ExportExecuteStep.vue # Step 4: Execute + results +│ │ +│ ├── import/ +│ │ ├── ImportWizard.vue # Import wizard container (4 steps) +│ │ ├── ImportFileStep.vue # Step 1: File selection (auto-detect) +│ │ ├── ImportMappingStep.vue # Step 2: Target & column mapping +│ │ ├── ImportOptionsStep.vue # Step 3: Options & preview +│ │ └── ImportExecuteStep.vue # Step 4: Execute + results +│ │ +│ ├── structure/ +│ │ ├── StructureTabs.vue # Sub-tab container (Generate DDL | Run SQL File) +│ │ ├── DdlWizard.vue # DDL wizard container (3 steps) +│ │ ├── DdlObjectStep.vue # Step 1: Object selection +│ │ ├── DdlOptionsStep.vue # Step 2: DDL options +│ │ ├── DdlPreviewStep.vue # Step 3: Preview & export +│ │ ├── SqlFileWizard.vue # SQL file wizard (2 steps) +│ │ ├── SqlFileSelectStep.vue # Step 1: File & connection +│ │ └── SqlFileExecuteStep.vue # Step 2: Execution & results +│ │ +│ ├── migration/ +│ │ ├── MigrationWizard.vue # Migration wizard container (5 steps) +│ │ ├── MigrationSourceStep.vue # Step 1: Source connection + tables +│ │ ├── MigrationTargetStep.vue # Step 2: Target connection +│ │ ├── MigrationMappingStep.vue # Step 3: Schema & type mapping +│ │ ├── MigrationConfigStep.vue # Step 4: Options & summary +│ │ └── MigrationExecuteStep.vue # Step 5: Execute + results +│ │ +│ └── tasks/ +│ ├── TaskManagerPanel.vue # Slide-out sidebar (400px) with task list +│ ├── TaskCard.vue # Individual task card with progress + actions +│ └── TaskManagerButton.vue # Header button showing task count badge +│ +├── store/ +│ └── transferStore.ts # Pinia store (wizard state + task management) +│ +├── datasources/ +│ └── transferApi.ts # Tauri invoke wrappers +│ +└── types/ + └── transfer.ts # TypeScript types +``` + +### TypeScript Types + +```typescript +// src/types/transfer.ts + +// ── Export ──────────────────────────────────────────────── + +export type ExportFormat = 'csv' | 'jsonl' | 'sql' | 'excel' + +export type ExportSource = { + table: string + columns: string[] + whereClause?: string + orderBy?: string + limit?: number +} + +export type ExportRequest = { + connectionId: string + database?: string + schema?: string + source: ExportSource + format: ExportFormat + csvOptions?: CsvExportOptions + jsonlOptions?: JsonlExportOptions + sqlOptions?: SqlExportOptions + excelOptions?: ExcelExportOptions + outputPath: string +} + +// Format options — only sent when user explicitly overrides via "Advanced Options" + +export type CsvExportOptions = { + delimiter?: string // default: ',' + quoteChar?: string // default: '"' + encoding?: string // default: 'UTF-8' + includeHeader?: boolean // default: true + quoteAll?: boolean // default: false + lineEnding?: 'lf' | 'crlf' // default: 'lf' +} + +export type JsonlExportOptions = { + dateFormat?: string // default: 'iso8601' +} + +export type SqlExportOptions = { + targetTable: string + batchSize?: number // default: 1000 + includeCreateTable?: boolean // default: true + includeDropTable?: boolean // default: false + targetEngine?: string +} + +export type ExcelExportOptions = { + sheetName?: string // default: 'Sheet1' + includeHeader?: boolean // default: true + autoFitColumns?: boolean // default: true + freezeHeader?: boolean // default: true +} + +// ── Import ──────────────────────────────────────────────── + +export type ImportFormat = 'csv' | 'jsonl' | 'sql' | 'excel' + +export type ColumnMapping = { + sourceColumn: string + targetColumn?: string + targetType?: string +} + +export type ConflictStrategy = 'skip' | 'replace' | 'upsert' | 'abort' + +export type CsvImportOptions = { + delimiter?: string // default: auto-detected or ',' + encoding?: string // default: auto-detected or 'UTF-8' + hasHeader?: boolean // default: true +} + +export type ImportRequest = { + connectionId: string + database?: string + schema?: string + table: string + filePath: string + format: ImportFormat + columnMappings: ColumnMapping[] + conflictStrategy: ConflictStrategy + batchSize: number + createTable: boolean + truncateBefore: boolean + dryRun: boolean + csvOptions?: CsvImportOptions +} + +// ── DDL ─────────────────────────────────────────────────── + +export type DdlOptions = { + includeCreateTable: boolean + includePrimaryKeys: boolean + includeForeignKeys: boolean + includeIndexes: boolean + includeConstraints: boolean + includeComments: boolean + includeStorage: boolean + includeDropIfExists: boolean + includeIfNotExists: boolean + includeData: boolean +} + +export type DdlRequest = { + connectionId: string + database?: string + schema?: string + tables: string[] + targetEngine?: string + options: DdlOptions +} + +// ── Run SQL File ────────────────────────────────────────── + +export type SqlFileErrorStrategy = 'rollback' | 'skipAndContinue' | 'stop' + +export type RunSqlFileRequest = { + connectionId: string + database?: string + filePath: string + wrapInTransaction: boolean + onError: SqlFileErrorStrategy + dryRun: boolean +} + +export type SqlFileInfo = { + filePath: string + fileSizeBytes: number + statementCount: number + previewLines: string[] +} + +// ── Migration ───────────────────────────────────────────── + +export type MigrationConversion = 'direct' | 'mapped' | 'custom' + +export type MigrationMapping = { + sourceColumn: string + sourceType: string + targetColumn: string + targetType: string + conversion: MigrationConversion +} + +export type MigrationTablePlan = { + sourceTable: string + targetTable: string + columnMappings: MigrationMapping[] +} + +export type MigrationErrorStrategy = 'skipRow' | 'skipTable' | 'abort' + +export type MigrationRequest = { + sourceConnectionId: string + sourceDatabase?: string + sourceSchema?: string + targetConnectionId: string + targetDatabase?: string + targetSchema?: string + tablePlans: MigrationTablePlan[] + batchSize: number + onError: MigrationErrorStrategy + createTables: boolean + dropTables: boolean + migrateIndexes: boolean + migrateForeignKeys: boolean + migrateConstraints: boolean + disableFkChecks: boolean +} + +// ── Progress ────────────────────────────────────────────── + +export type TransferProgress = { + operation: string + phase: string + currentTable?: string + totalRows?: number + processedRows: number + skippedRows: number + errorCount: number + percent: number + elapsedMs: number + estimatedRemainingMs?: number + message?: string +} + +// ── Results ─────────────────────────────────────────────── + +export type TransferError = { + rowNumber?: number + statementNumber?: number + message: string + sql?: string +} + +export type TableTransferResult = { + table: string + rows: number + success: boolean + durationMs: number + errors: TransferError[] +} + +export type TransferResult = { + success: boolean + totalRows: number + processedRows: number + skippedRows: number + errorCount: number + durationMs: number + outputPath?: string + outputSizeBytes?: number + errors: TransferError[] + tableResults?: TableTransferResult[] +} + +// ── Detection ───────────────────────────────────────────── + +export type FileDetectionResult = { + format: ImportFormat + encoding: string + estimatedRows?: number + fileSizeBytes: number + columns: string[] + csvDelimiter?: string + hasHeader?: boolean +} + +export type ExportPreview = { + columns: string[] + sampleRows: string[][] + totalRowsEstimate?: number + formattedPreview: string +} + +export type TypeMappingSuggestion = { + sourceColumn: string + sourceType: string + targetType: string + conversion: MigrationConversion + warning?: string +} + +// ── Background Tasks ────────────────────────────────────── + +export type TaskKind = 'export' | 'import' | 'sqlFile' | 'migration' + +export type TaskStatus = 'pending' | 'running' | 'completed' | 'failed' + +export type TaskRuntime = { + complete: number + total: number + skipped: number + errorCount: number +} + +export type ExportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + columns: string[] + whereClause?: string + orderBy?: string + limit?: number + format: ExportFormat + outputPath: string +} + +export type ImportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + filePath: string + format: ImportFormat + conflictStrategy: ConflictStrategy +} + +export type SqlFileTaskConfig = { + connectionId: string + database?: string + filePath: string + onError: SqlFileErrorStrategy +} + +export type MigrationTaskConfig = { + sourceConnectionId: string + sourceDatabase?: string + targetConnectionId: string + targetDatabase?: string + tables: string[] +} + +export type TaskConfig = ExportTaskConfig | ImportTaskConfig | SqlFileTaskConfig | MigrationTaskConfig + +export type BackgroundTask = { + id: string + kind: TaskKind + status: TaskStatus + progress: { complete: number; total: number } + config: TaskConfig + runtime: TaskRuntime + label: string + startTime: Date + endTime?: Date + error?: string +} +``` + +### Pinia Store + +```typescript +// src/store/transferStore.ts + +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import type { + ExportRequest, + ImportRequest, + MigrationRequest, + TransferProgress, + TransferResult, + BackgroundTask, + TaskKind, + TaskStatus, + TaskRuntime, +} from '@/types/transfer' + +export const useTransferStore = defineStore('transfer', () => { + // ── Active Tab ──────────────────────────────── + const activeTab = ref<'export' | 'import' | 'structure' | 'migration'>('export') + + // ── Progress Tracking ───────────────────────── + const isRunning = ref(false) + const progress = ref(null) + const lastResult = ref(null) + const operationId = ref(null) + + // ── Export State ────────────────────────────── + const exportStep = ref(0) + const exportRequest = ref>({}) + + // ── Import State ────────────────────────────── + const importStep = ref(0) + const importRequest = ref>({}) + + // ── Migration State ─────────────────────────── + const migrationStep = ref(0) + const migrationRequest = ref>({}) + + // ── Background Task State ───────────────────── + const runningTasks = ref([]) + const activeExportTaskId = ref(null) + const activeImportTaskId = ref(null) + const activeSqlFileTaskId = ref(null) + const activeMigrationTaskId = ref(null) + + // ── Computed ────────────────────────────────── + const progressPercent = computed(() => progress.value?.percent ?? 0) + const canCancel = computed(() => isRunning.value && operationId.value !== null) + const taskCount = computed(() => runningTasks.value.length) + const hasRunningTasks = computed(() => + runningTasks.value.some(t => t.status === 'running') + ) + const activeTaskId = computed(() => { + switch (activeTab.value) { + case 'export': return activeExportTaskId.value + case 'import': return activeImportTaskId.value + case 'structure': return activeSqlFileTaskId.value + case 'migration': return activeMigrationTaskId.value + default: return null + } + }) + + // ── Tab Actions ─────────────────────────────── + const setActiveTab = (tab: typeof activeTab.value) => { + activeTab.value = tab + } + + // ── Progress Actions ────────────────────────── + const updateProgress = (p: TransferProgress) => { + progress.value = p + } + + const startOperation = (id: string) => { + operationId.value = id + isRunning.value = true + progress.value = null + lastResult.value = null + } + + const completeOperation = (result: TransferResult) => { + isRunning.value = false + lastResult.value = result + progress.value = null + operationId.value = null + } + + // ── Reset Actions ───────────────────────────── + const resetExport = () => { + exportStep.value = 0 + exportRequest.value = {} + lastResult.value = null + } + + const resetImport = () => { + importStep.value = 0 + importRequest.value = {} + lastResult.value = null + } + + const resetMigration = () => { + migrationStep.value = 0 + migrationRequest.value = {} + lastResult.value = null + } + + // ── Task Management Actions ─────────────────── + + const addRunningTask = (task: BackgroundTask) => { + runningTasks.value = [...runningTasks.value, task] + } + + const updateTaskRuntime = (taskId: string, runtime: Partial) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { + ...t, + runtime: { ...t.runtime, ...runtime }, + progress: { + complete: runtime.complete ?? t.progress.complete, + total: runtime.total ?? t.progress.total, + }, + } + : t + ) + } + + const updateTaskStatus = (taskId: string, status: TaskStatus, error?: string) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { + ...t, + status, + endTime: status === 'completed' || status === 'failed' ? new Date() : undefined, + error, + } + : t + ) + } + + const removeTask = (taskId: string) => { + runningTasks.value = runningTasks.value.filter(t => t.id !== taskId) + } + + const clearCompletedTasks = () => { + runningTasks.value = runningTasks.value.filter(t => + t.status === 'running' || t.status === 'pending' + ) + } + + const syncProgressToTask = (taskId: string, p: TransferProgress) => { + updateTaskRuntime(taskId, { + complete: p.processedRows, + total: p.totalRows ?? 0, + skipped: p.skippedRows, + errorCount: p.errorCount, + }) + } + + const detachActiveTask = (kind: TaskKind) => { + switch (kind) { + case 'export': activeExportTaskId.value = null; break + case 'import': activeImportTaskId.value = null; break + case 'sqlFile': activeSqlFileTaskId.value = null; break + case 'migration': activeMigrationTaskId.value = null; break + } + } + + const openTask = (taskId: string) => { + const task = runningTasks.value.find(t => t.id === taskId) + if (!task) return + + // Map task kind to tab + const tabMap: Record = { + export: 'export', + import: 'import', + sqlFile: 'structure', + migration: 'migration', + } + activeTab.value = tabMap[task.kind] + + // Restore form state from task.config + // The specific restoration logic depends on task.kind + // and populates the corresponding wizard step fields + switch (task.kind) { + case 'export': + activeExportTaskId.value = taskId + // Restore exportRequest from task.config + break + case 'import': + activeImportTaskId.value = taskId + // Restore importRequest from task.config + break + case 'sqlFile': + activeSqlFileTaskId.value = taskId + break + case 'migration': + activeMigrationTaskId.value = taskId + // Restore migrationRequest from task.config + break + } + } + + return { + // Tab + activeTab, + setActiveTab, + + // Progress + isRunning, + progress, + lastResult, + operationId, + progressPercent, + canCancel, + updateProgress, + startOperation, + completeOperation, + + // Wizard state + exportStep, + exportRequest, + importStep, + importRequest, + migrationStep, + migrationRequest, + resetExport, + resetImport, + resetMigration, + + // Task management + runningTasks, + activeExportTaskId, + activeImportTaskId, + activeSqlFileTaskId, + activeMigrationTaskId, + taskCount, + hasRunningTasks, + activeTaskId, + addRunningTask, + updateTaskRuntime, + updateTaskStatus, + removeTask, + clearCompletedTasks, + syncProgressToTask, + detachActiveTask, + openTask, + } +}) +``` + +### Frontend API Wrapper + +```typescript +// src/datasources/transferApi.ts + +import { invoke } from '@tauri-apps/api/core' +import type { + ExportRequest, + ExportPreview, + ImportRequest, + CsvImportOptions, + ImportFormat, + DdlRequest, + RunSqlFileRequest, + SqlFileInfo, + MigrationRequest, + MigrationTablePlan, + FileDetectionResult, + TransferResult, +} from '@/types/transfer' + +// ── Export ──────────────────────────────────────────────── + +export const previewExport = (request: ExportRequest, previewRows = 10) => + invoke('preview_export', { request, previewRows }) + +export const executeExport = (request: ExportRequest) => + invoke('execute_export', { request }) + +// ── Import ──────────────────────────────────────────────── + +export const detectFile = (filePath: string) => + invoke('detect_file', { filePath }) + +export const previewImport = ( + filePath: string, + format: ImportFormat, + csvOptions?: CsvImportOptions, + previewRows = 10, +) => invoke('preview_import', { filePath, format, csvOptions, previewRows }) + +export const executeImport = (request: ImportRequest) => + invoke('execute_import', { request }) + +// ── DDL ─────────────────────────────────────────────────── + +export const generateDdl = (request: DdlRequest) => + invoke('generate_ddl', { request }) + +export const executeDdl = (connectionId: string, database: string | undefined, sql: string) => + invoke('execute_ddl', { connectionId, database, sql }) + +// ── Run SQL File ────────────────────────────────────────── + +export const parseSqlFile = (filePath: string) => + invoke('parse_sql_file', { filePath }) + +export const executeSqlFile = (request: RunSqlFileRequest) => + invoke('execute_sql_file', { request }) + +// ── Migration ───────────────────────────────────────────── + +export const suggestTypeMappings = ( + sourceConnectionId: string, + sourceDatabase: string | undefined, + sourceSchema: string | undefined, + sourceTables: string[], + targetEngine: string, +) => invoke('suggest_type_mappings', { + sourceConnectionId, + sourceDatabase, + sourceSchema, + sourceTables, + targetEngine, +}) + +export const executeMigration = (request: MigrationRequest) => + invoke('execute_migration', { request }) + +// ── Shared ──────────────────────────────────────────────── + +export const cancelTransfer = (operationId: string) => + invoke('cancel_transfer', { operationId }) +``` + +### Progress Event Listener + +```typescript +// Usage in component (e.g., ExportExecuteStep.vue) + +import { listen } from '@tauri-apps/api/event' +import { useTransferStore } from '@/store/transferStore' +import type { TransferProgress } from '@/types/transfer' + +const transferStore = useTransferStore() + +const unlisten = await listen('transfer-progress', event => { + transferStore.updateProgress(event.payload) + + // Sync progress to background task if one is active + const taskId = transferStore.activeExportTaskId + if (taskId) { + transferStore.syncProgressToTask(taskId, event.payload) + } +}) + +// Cleanup on unmount +onUnmounted(() => { + unlisten() +}) +``` + +--- + +## i18n Keys + +```json +{ + "transfer.title": "Transfer", + "transfer.subtitle": "Import, export, and migrate your data", + + "transfer.tabs.export": "Export", + "transfer.tabs.import": "Import", + "transfer.tabs.structure": "Structure", + "transfer.tabs.migration": "Migration", + + "transfer.export.step.source": "Source", + "transfer.export.step.format": "Format", + "transfer.export.step.preview": "Preview", + "transfer.export.step.execute": "Export", + "transfer.export.columns.selectAll": "Select All", + "transfer.export.columns.deselectAll": "Deselect All", + "transfer.export.where": "WHERE clause (optional)", + "transfer.export.orderBy": "ORDER BY (optional)", + "transfer.export.limit": "LIMIT (optional)", + + "transfer.format.csv": "CSV (.csv)", + "transfer.format.jsonl": "JSONL (.jsonl)", + "transfer.format.sql": "SQL (.sql)", + "transfer.format.excel": "Excel (.xlsx)", + "transfer.format.defaults.csv": "Comma delimiter, double-quote, UTF-8, include header, LF line ending", + "transfer.format.defaults.jsonl": "One JSON object per line, compact, UTF-8, ISO 8601 dates", + "transfer.format.defaults.sql": "Auto-filled target table, batch size 1000, include CREATE TABLE", + "transfer.format.defaults.excel": "Include header, auto-fit columns, freeze header row", + "transfer.format.advancedOptions": "Advanced Options", + + "transfer.import.step.file": "Select File", + "transfer.import.step.mapping": "Target & Mapping", + "transfer.import.step.options": "Options & Preview", + "transfer.import.step.execute": "Import", + "transfer.import.dropzone.title": "Drag & drop a file here", + "transfer.import.dropzone.subtitle": "or click to browse", + "transfer.import.dropzone.supported": "Supported: CSV, JSONL, SQL, Excel (.xlsx)", + "transfer.import.detected.format": "Detected Format", + "transfer.import.detected.encoding": "Detected Encoding", + "transfer.import.detected.rows": "Rows (estimated)", + "transfer.import.advancedParseOptions": "Advanced Parse Options", + "transfer.import.createTable": "Create table if not exists", + "transfer.import.autoMap": "Auto-Map by Name", + "transfer.import.clearAll": "Clear All", + "transfer.import.conflict": "On Conflict", + "transfer.import.conflict.skip": "Skip duplicates", + "transfer.import.conflict.replace": "Replace existing", + "transfer.import.conflict.upsert": "Update existing (upsert)", + "transfer.import.conflict.abort": "Abort on error", + "transfer.import.truncateBefore": "Truncate table before import", + "transfer.import.dryRun": "Dry run (validate without inserting)", + + "transfer.structure.tabs.ddl": "Generate DDL", + "transfer.structure.tabs.sqlFile": "Run SQL File", + "transfer.structure.ddl.step.objects": "Select Objects", + "transfer.structure.ddl.step.options": "DDL Options", + "transfer.structure.ddl.step.preview": "Preview & Export", + "transfer.structure.ddl.targetEngine": "Target Engine", + "transfer.structure.ddl.sameAsSource": "Same as source", + "transfer.structure.ddl.includeCreate": "CREATE TABLE statements", + "transfer.structure.ddl.includePk": "Primary keys", + "transfer.structure.ddl.includeFk": "Foreign keys", + "transfer.structure.ddl.includeIndexes": "Indexes", + "transfer.structure.ddl.includeConstraints": "Constraints (UNIQUE, CHECK, NOT NULL)", + "transfer.structure.ddl.includeComments": "Comments / descriptions", + "transfer.structure.ddl.includeStorage": "Tablespace / storage options", + "transfer.structure.ddl.includeDrop": "Include DROP IF EXISTS before CREATE", + "transfer.structure.ddl.includeIfNotExists": "Include IF NOT EXISTS on CREATE", + "transfer.structure.ddl.includeData": "Include INSERT DATA", + "transfer.structure.ddl.copyClipboard": "Copy to Clipboard", + "transfer.structure.ddl.saveFile": "Save to File", + "transfer.structure.ddl.executeServer": "Execute on Server", + "transfer.structure.sqlFile.step.select": "Select File & Connection", + "transfer.structure.sqlFile.step.execute": "Execution", + "transfer.structure.sqlFile.dropzone.title": "Drag & drop a .sql file here", + "transfer.structure.sqlFile.dropzone.supported": "Supported: .sql files", + "transfer.structure.sqlFile.wrapTransaction": "Wrap in transaction", + "transfer.structure.sqlFile.onError": "On Error", + "transfer.structure.sqlFile.onError.rollback": "Rollback all", + "transfer.structure.sqlFile.onError.skip": "Skip and continue", + "transfer.structure.sqlFile.onError.stop": "Stop execution", + "transfer.structure.sqlFile.dryRun": "Dry run (parse only)", + + "transfer.migration.step.source": "Source", + "transfer.migration.step.target": "Target", + "transfer.migration.step.mapping": "Mapping", + "transfer.migration.step.configure": "Configure", + "transfer.migration.step.execute": "Migrate", + "transfer.migration.createTables": "Create target tables if not exist", + "transfer.migration.dropTables": "Drop target tables before migration", + "transfer.migration.batchSize": "Batch Size", + "transfer.migration.onError": "On Error", + "transfer.migration.onError.skipRow": "Skip row and continue", + "transfer.migration.onError.skipTable": "Skip table and continue", + "transfer.migration.onError.abort": "Abort migration", + "transfer.migration.migrateIndexes": "Migrate indexes", + "transfer.migration.migrateFk": "Migrate foreign keys", + "transfer.migration.migrateConstraints": "Migrate constraints", + "transfer.migration.disableFkChecks": "Disable foreign key checks during migration", + "transfer.migration.editMapping": "Edit Mapping", + "transfer.migration.resetAuto": "Reset to Auto", + "transfer.migration.conversion.direct": "Direct", + "transfer.migration.conversion.mapped": "Auto-mapped", + "transfer.migration.conversion.custom": "Custom", + + "transfer.progress.exporting": "Exporting...", + "transfer.progress.importing": "Importing...", + "transfer.progress.migrating": "Migrating...", + "transfer.progress.executing": "Executing...", + "transfer.progress.elapsed": "Elapsed", + "transfer.progress.remaining": "Estimated remaining", + "transfer.progress.rows": "Rows", + "transfer.progress.exported": "exported", + "transfer.progress.imported": "imported", + "transfer.progress.skipped": "skipped", + "transfer.progress.errors": "errors", + "transfer.progress.statements": "Statements", + "transfer.progress.succeeded": "Succeeded", + "transfer.progress.failed": "Failed", + "transfer.progress.cancel": "Cancel", + "transfer.progress.runInBackground": "Run in Background", + + "transfer.result.success": "completed successfully", + "transfer.result.partial": "completed with errors", + "transfer.result.failed": "failed", + "transfer.result.duration": "Duration", + "transfer.result.file": "File", + "transfer.result.openFile": "Open File", + "transfer.result.openFolder": "Open Folder", + "transfer.result.exportAgain": "Export Again", + "transfer.result.importAgain": "Import Again", + "transfer.result.migrateAgain": "Migrate Again", + "transfer.result.runAgain": "Run Again", + "transfer.result.viewTable": "View Table", + "transfer.result.showErrorsOnly": "Show Errors Only", + "transfer.result.copyLog": "Copy Log", + + "transfer.tasks.title": "Tasks", + "transfer.tasks.clearCompleted": "Clear Completed", + "transfer.tasks.goToTask": "Go to Task", + "transfer.tasks.dismiss": "Dismiss", + "transfer.tasks.noTasks": "No transfer tasks", + "transfer.tasks.status.pending": "Pending", + "transfer.tasks.status.running": "Running", + "transfer.tasks.status.completed": "Completed", + "transfer.tasks.status.failed": "Failed", + "transfer.tasks.startedAgo": "Started {time} ago", + + "transfer.common.connection": "Connection", + "transfer.common.database": "Database", + "transfer.common.schema": "Schema", + "transfer.common.table": "Table", + "transfer.common.selectAll": "Select All", + "transfer.common.deselectAll": "Deselect All", + "transfer.common.tablesOnly": "Tables Only", + "transfer.common.viewsOnly": "Views Only", + "transfer.common.back": "Back", + "transfer.common.next": "Next", + "transfer.common.browse": "Browse..." +} +``` + +--- + +## Router & Sidebar Changes + +### Router Update + +```typescript +// src/router/index.ts — change: +// { path: '/import-export', ... } +// to: +{ + path: '/transfer', + name: 'transfer', + component: () => import('@/pages/TransferPage.vue'), +} +``` + +### Sidebar Update + +```typescript +// In AppSidebar.vue — change: +// { label: t('sidebar.importExport'), icon: ArrowLeftRight, to: '/import-export' } +// to: +{ label: t('sidebar.transfer'), icon: ArrowLeftRight, to: '/transfer' } +``` + +### i18n Sidebar Key + +```json +{ + "sidebar.transfer": "Transfer" +} +``` + +--- + +## Implementation Phases + +### Phase 1 — MVP (Data Export + Import + Task System) + +**Scope**: Export and Import tabs with CSV and JSONL support, plus background task infrastructure. + +**Deliverables**: +- `src-tauri/src/transfer/` module: `mod.rs`, `types.rs`, `defaults.rs`, `export.rs`, `import.rs`, `progress.rs` +- `src-tauri/src/commands/transfer.rs`: `preview_export`, `execute_export`, `detect_file`, `preview_import`, `execute_import`, `cancel_transfer` +- `src/pages/TransferPage.vue` replacing `ImportExportPage.vue` +- `src/components/transfer/` — all shared components + export/ + import/ wizards + tasks/ +- `src/store/transferStore.ts` (including full task management) +- `src/datasources/transferApi.ts` +- `src/types/transfer.ts` +- Router + sidebar updates +- i18n keys for export/import/tasks +- Formats: CSV, JSONL only (SQL, Excel deferred) +- Background task system with Task Manager panel + +**Estimated effort**: 3-4 weeks + +### Phase 2 — Full Export/Import + Structure + +**Scope**: Complete format support + Structure tab. + +**Deliverables**: +- SQL, Excel export/import support (using best-practice defaults) +- `src-tauri/src/transfer/ddl.rs` — `DdlGenerator` trait + per-engine implementations +- `src-tauri/src/commands/transfer.rs`: `generate_ddl`, `execute_ddl`, `parse_sql_file`, `execute_sql_file` +- `src/components/transfer/structure/` — DDL wizard + SQL file wizard +- i18n keys for structure tab +- New Cargo dependencies: `rust_xlsxwriter`, `calamine`, `encoding_rs`, `chardetng` + +**Estimated effort**: 2-3 weeks + +### Phase 3 — Cross-Engine Migration + +**Scope**: Migration tab with full cross-engine support. + +**Deliverables**: +- `src-tauri/src/transfer/migration.rs` — migration orchestrator +- `src-tauri/src/transfer/type_mapping.rs` — cross-engine type mapping matrix +- `src-tauri/src/commands/transfer.rs`: `suggest_type_mappings`, `execute_migration` +- `src/components/transfer/migration/` — all 5 migration wizard steps +- i18n keys for migration tab +- Per-table progress tracking with rollback support + +**Estimated effort**: 2-3 weeks + +--- + +## UI Components Used + +All UI components come from the existing shadcn-vue component library in `src/components/ui/`: + +| Component | Usage | +|-----------|-------| +| `Button` | Wizard navigation, actions, "Run in Background" | +| `Card` | Step containers, summary panels, task cards | +| `Tabs`, `TabsList`, `TabsTrigger`, `TabsContent` | Top-level tabs, structure sub-tabs | +| `Select` | Connection, database, schema, format dropdowns | +| `Input` | Text inputs (table name, file path, etc.) | +| `Checkbox` | Column selection, boolean options | +| `RadioGroup` | Format selection | +| `Label` | Form labels | +| `Table` | Column mapping grid, results table | +| `Progress` | Progress bar during execution, task cards | +| `Badge` | Status indicators (mapped, skipped, error), task status | +| `Spinner` | Loading states | +| `Dialog` | Confirmation dialogs (cancel, truncate) | +| `AlertDialog` | Destructive action confirmations | +| `Tooltip` | Help text on options | +| `DropdownMenu` | Additional actions menus | +| `Notification` | Success/error toast notifications | +| `Sheet` | Task Manager slide-out panel | +| `Collapsible` | "Advanced Options" expandable sections | + +--- + +## Revision History + +| Date | Change | Reason | +|------|--------|--------| +| v1.0 | Initial design | Comprehensive Transfer feature spec | +| v2.0 | Added Background Task System | Modeled after dockit's frontend-only task management pattern | +| v2.0 | Removed Custom Query export source | Simplification — table-only export covers primary use cases | +| v2.0 | Removed XML options panel | Simplification — XML uses fixed best-practice defaults | +| v2.0 | Simplified Format & Options step | Best-practice defaults applied automatically; "Advanced Options" collapsible for power users | +| v2.0 | Removed "Disable indexes during import" option | Backend handles automatically for large imports | +| v2.0 | Added Task Manager UI | Slide-out panel for tracking background tasks with progress, status, and navigation | +| v2.0 | Added `tasks/` component directory | TaskManagerPanel, TaskCard, TaskManagerButton | +| v2.0 | Updated Pinia store | Added full task lifecycle management (add, update, sync, detach, open, clear) | +| v2.0 | Phase 1 scope expanded | Includes background task infrastructure from the start | +| v3.0 | Removed XML format | Dropped XML from both export and import — reduces complexity and crate dependencies | +| v3.0 | Replaced JSON with JSONL | JSON Lines format (`.jsonl`) for better streaming and size efficiency with large datasets | +| v3.0 | Removed SQL-specific options panel | SQL export uses auto-filled defaults (target table from source, batch size 1000, include CREATE TABLE) — no dedicated UI section | diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 40de11e7..0e8e1344 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -523,6 +523,21 @@ dependencies = [ "system-deps", ] +[[package]] +name = "calamine" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0ba51a659bb6c8bffd6f7c1c5ffafcafa0c97e4769411d841c3cc5c154ab47" +dependencies = [ + "byteorder", + "codepage", + "encoding_rs", + "log", + "quick-xml 0.30.0", + "serde", + "zip 0.6.6", +] + [[package]] name = "camino" version = "1.2.2" @@ -676,6 +691,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "combine" version = "4.6.7" @@ -3754,7 +3778,7 @@ checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" dependencies = [ "base64 0.22.1", "indexmap 2.13.0", - "quick-xml", + "quick-xml 0.38.4", "serde", "time", ] @@ -3987,6 +4011,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quick-xml" version = "0.38.4" @@ -4363,6 +4397,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "rust_xlsxwriter" +version = "0.64.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f47f5318c1e512e57c07781559367577b1eb9618325cf1574cd30d38b112c5" +dependencies = [ + "lazy_static", + "regex", + "zip 0.6.6", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -5019,13 +5064,16 @@ version = "0.2.0" dependencies = [ "async-trait", "base64 0.22.1", + "calamine", "chrono", "deadpool-postgres", + "hex", "mysql_async", "native-tls", "postgres-native-tls", "rusqlite", "rust_decimal", + "rust_xlsxwriter", "serde", "serde_json", "tauri", @@ -5565,7 +5613,7 @@ dependencies = [ "tokio", "url", "windows-sys 0.60.2", - "zip", + "zip 4.6.1", ] [[package]] @@ -7479,6 +7527,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] + [[package]] name = "zip" version = "4.6.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 391b0442..3881abfb 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -61,3 +61,6 @@ url = "2" base64 = "0.22" chrono = "0.4" rust_decimal = { version = "1", features = [ "db-postgres" ] } +hex = "0.4" +rust_xlsxwriter = "0.64" +calamine = "0.22" diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 75b495ee..5bb35f3a 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -1,15 +1,3 @@ -//! Tauri command handlers. -//! -//! This module contains all Tauri commands organized by domain: -//! - `server`: Server connection testing -//! - `connection`: Connection lifecycle management -//! - `query`: SQL query execution -//! - `browse`: Database metadata browsing -//! - `store`: Key-value store management -//! - `file_operations`: File save/load operations for queries -//! - `converter`: Data conversion utilities for JSON serialization -//! - `helpers`: Shared utilities to reduce code duplication - pub mod browse; pub mod connection; pub mod converter; @@ -18,11 +6,12 @@ pub mod helpers; pub mod query; pub mod server; pub mod store; +pub mod transfer; -// Re-export all command functions for convenience pub use browse::*; pub use connection::*; pub use file_operations::*; pub use query::*; pub use server::*; pub use store::*; +pub use transfer::*; diff --git a/src-tauri/src/commands/transfer.rs b/src-tauri/src/commands/transfer.rs new file mode 100644 index 00000000..f31fec79 --- /dev/null +++ b/src-tauri/src/commands/transfer.rs @@ -0,0 +1,299 @@ +use crate::state::{ActiveConnection, AppState}; +use crate::transfer::{ + auto_map_columns, detect_file, execute_export, execute_import, execute_migration, + preview_export, preview_import, preview_migration, + ExportPreview, ExportRequest, FileDetectionResult, ImportFormat, ImportRequest, + MigrationPreview, MigrationRequest, TransferResult, +}; +use crate::database::DatabaseType; +use tauri::{AppHandle, State}; + +#[tauri::command] +pub async fn preview_export_data( + request: ExportRequest, + preview_rows: u32, + state: State<'_, AppState>, +) -> Result { + let connections = state.connections.lock().await; + let connection = connections + .get(&request.connection_id) + .ok_or_else(|| "No active connection found".to_string())?; + + match connection { + ActiveConnection::Postgres(adapter) => { + let adapter = adapter.lock().await; + preview_export(&*adapter, request, preview_rows).await + } + ActiveConnection::MySQL(adapter) => { + let adapter = adapter.lock().await; + preview_export(&*adapter, request, preview_rows).await + } + ActiveConnection::SQLServer(adapter) => { + let adapter = adapter.lock().await; + preview_export(&*adapter, request, preview_rows).await + } + ActiveConnection::SQLite(adapter) => { + let adapter = adapter.lock().await; + preview_export(&*adapter, request, preview_rows).await + } + } +} + +#[tauri::command] +pub async fn execute_export_data( + request: ExportRequest, + app_handle: AppHandle, + state: State<'_, AppState>, +) -> Result { + let connections = state.connections.lock().await; + let connection = connections + .get(&request.connection_id) + .ok_or_else(|| "No active connection found".to_string())?; + + match connection { + ActiveConnection::Postgres(adapter) => { + let adapter = adapter.lock().await; + execute_export(&*adapter, request, &app_handle).await + } + ActiveConnection::MySQL(adapter) => { + let adapter = adapter.lock().await; + execute_export(&*adapter, request, &app_handle).await + } + ActiveConnection::SQLServer(adapter) => { + let adapter = adapter.lock().await; + execute_export(&*adapter, request, &app_handle).await + } + ActiveConnection::SQLite(adapter) => { + let adapter = adapter.lock().await; + execute_export(&*adapter, request, &app_handle).await + } + } +} + +#[tauri::command] +pub fn detect_file_format(file_path: String) -> Result { + detect_file(&file_path) +} + +#[tauri::command] +pub fn preview_import_data( + file_path: String, + format: ImportFormat, + preview_rows: u32, +) -> Result { + preview_import(&file_path, format, preview_rows) +} + +#[tauri::command] +pub async fn execute_import_data( + request: ImportRequest, + app_handle: AppHandle, + state: State<'_, AppState>, +) -> Result { + let connections = state.connections.lock().await; + let connection = connections + .get(&request.connection_id) + .ok_or_else(|| "No active connection found".to_string())?; + + match connection { + ActiveConnection::Postgres(adapter) => { + let adapter = adapter.lock().await; + execute_import(&*adapter, request, &app_handle).await + } + ActiveConnection::MySQL(adapter) => { + let adapter = adapter.lock().await; + execute_import(&*adapter, request, &app_handle).await + } + ActiveConnection::SQLServer(adapter) => { + let adapter = adapter.lock().await; + execute_import(&*adapter, request, &app_handle).await + } + ActiveConnection::SQLite(adapter) => { + let adapter = adapter.lock().await; + execute_import(&*adapter, request, &app_handle).await + } + } +} + +#[tauri::command] +pub async fn preview_migration_data( + request: MigrationRequest, + state: State<'_, AppState>, +) -> Result { + let connections = state.connections.lock().await; + + let source_connection = connections + .get(&request.source_connection_id) + .ok_or_else(|| "No source connection found".to_string())?; + + match source_connection { + ActiveConnection::Postgres(adapter) => { + let adapter = adapter.lock().await; + preview_migration(&*adapter, &request).await + } + ActiveConnection::MySQL(adapter) => { + let adapter = adapter.lock().await; + preview_migration(&*adapter, &request).await + } + ActiveConnection::SQLServer(adapter) => { + let adapter = adapter.lock().await; + preview_migration(&*adapter, &request).await + } + ActiveConnection::SQLite(adapter) => { + let adapter = adapter.lock().await; + preview_migration(&*adapter, &request).await + } + } +} + +#[tauri::command] +pub async fn execute_migration_data( + request: MigrationRequest, + app_handle: AppHandle, + state: State<'_, AppState>, +) -> Result { + let connections = state.connections.lock().await; + + let source_connection = connections + .get(&request.source_connection_id) + .ok_or_else(|| "No source connection found".to_string())?; + + let target_connection = connections + .get(&request.target_connection_id) + .ok_or_else(|| "No target connection found".to_string())?; + + macro_rules! run_migration { + ($source_adapter:expr, $target_adapter:expr) => { + execute_migration(&*$source_adapter, &*$target_adapter, request, &app_handle).await + }; + } + + match (source_connection, target_connection) { + (ActiveConnection::Postgres(src), ActiveConnection::Postgres(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::Postgres(src), ActiveConnection::MySQL(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::Postgres(src), ActiveConnection::SQLServer(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::Postgres(src), ActiveConnection::SQLite(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::MySQL(src), ActiveConnection::Postgres(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::MySQL(src), ActiveConnection::MySQL(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::MySQL(src), ActiveConnection::SQLServer(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::MySQL(src), ActiveConnection::SQLite(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLServer(src), ActiveConnection::Postgres(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLServer(src), ActiveConnection::MySQL(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLServer(src), ActiveConnection::SQLServer(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLServer(src), ActiveConnection::SQLite(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLite(src), ActiveConnection::Postgres(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLite(src), ActiveConnection::MySQL(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLite(src), ActiveConnection::SQLServer(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + (ActiveConnection::SQLite(src), ActiveConnection::SQLite(tgt)) => { + let src = src.lock().await; + let tgt = tgt.lock().await; + run_migration!(src, tgt) + } + } +} + +#[tauri::command] +pub async fn auto_map_migration_columns( + connection_id: String, + database: Option, + schema: Option, + table: String, + target_engine: String, + state: State<'_, AppState>, +) -> Result, String> { + use crate::database::DatabaseAdapter; + + let connections = state.connections.lock().await; + let connection = connections + .get(&connection_id) + .ok_or_else(|| "No active connection found".to_string())?; + + let target_db_type = match target_engine.to_lowercase().as_str() { + "postgresql" | "postgres" => DatabaseType::PostgreSQL, + "mysql" | "mariadb" => DatabaseType::MySQL, + "sqlite" => DatabaseType::SQLite, + "sqlserver" | "mssql" => DatabaseType::SqlServer, + _ => return Err("Unknown target engine".to_string()), + }; + + macro_rules! fetch_and_map { + ($adapter:expr) => { + { + let adapter = $adapter.lock().await; + let columns = adapter.list_columns( + database.as_deref(), + schema.as_deref(), + &table, + ).await.map_err(|e| e.to_string())?; + Ok(auto_map_columns(&columns, target_db_type)) + } + }; + } + + match connection { + ActiveConnection::Postgres(adapter) => fetch_and_map!(adapter), + ActiveConnection::MySQL(adapter) => fetch_and_map!(adapter), + ActiveConnection::SQLServer(adapter) => fetch_and_map!(adapter), + ActiveConnection::SQLite(adapter) => fetch_and_map!(adapter), + } +} \ No newline at end of file diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 556cd801..9ae40a67 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,7 +1,5 @@ -// Database adapter module pub mod database; - -// API response types +pub mod transfer; pub mod api_response; // Application state management @@ -128,6 +126,11 @@ pub fn run() { commands::list_saved_queries, commands::delete_query_file, commands::write_text_file, + commands::preview_export_data, + commands::execute_export_data, + commands::detect_file_format, + commands::preview_import_data, + commands::execute_import_data, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/transfer/ddl.rs b/src-tauri/src/transfer/ddl.rs new file mode 100644 index 00000000..303a042b --- /dev/null +++ b/src-tauri/src/transfer/ddl.rs @@ -0,0 +1,358 @@ +//! DDL generation for various database engines. + +use crate::database::types::ColumnInfo; +use crate::database::DatabaseType; + +use super::types::*; + +pub fn generate_ddl_for_engine( + engine: DatabaseType, + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + match engine { + DatabaseType::PostgreSQL => generate_postgres_ddl(schema, table, columns, options), + DatabaseType::MySQL => generate_mysql_ddl(schema, table, columns, options), + DatabaseType::SQLite => generate_sqlite_ddl(schema, table, columns, options), + DatabaseType::SqlServer => generate_sqlserver_ddl(schema, table, columns, options), + _ => generate_generic_ddl(schema, table, columns, options), + } +} + +fn generate_postgres_ddl( + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + let mut sql = String::new(); + let table_ref = format_table_ref_postgres(schema, table); + + if options.include_drop_if_exists { + sql.push_str(&format!("DROP TABLE IF EXISTS {} CASCADE;\n", table_ref)); + } + + if options.include_create_table { + let create_keyword = if options.include_if_not_exists { + "CREATE TABLE IF NOT EXISTS" + } else { + "CREATE TABLE" + }; + + sql.push_str(&format!("{} {} (\n", create_keyword, table_ref)); + + let col_defs: Vec = columns + .iter() + .map(|c| format_column_postgres(c, options)) + .collect(); + + let pk_cols: Vec = columns + .iter() + .filter(|c| c.is_primary_key) + .map(|c| quote_identifier_postgres(&c.name)) + .collect(); + + if !pk_cols.is_empty() && options.include_primary_keys { + let pk_def = format!(" PRIMARY KEY ({})", pk_cols.join(", ")); + sql.push_str(&col_defs.join(",\n")); + sql.push_str(",\n"); + sql.push_str(&pk_def); + } else { + sql.push_str(&col_defs.join(",\n")); + } + + sql.push_str("\n);\n"); + } + + if options.include_comments { + for col in columns { + if let Some(ref desc) = col.description { + if !desc.is_empty() { + sql.push_str(&format!( + "COMMENT ON COLUMN {}.{} IS '{}';\n", + table_ref, + quote_identifier_postgres(&col.name), + desc.replace('\'', "''") + )); + } + } + } + } + + sql +} + +fn format_column_postgres(col: &ColumnInfo, options: &DdlOptions) -> String { + let name = quote_identifier_postgres(&col.name); + let mut def = format!(" {} {}", name, col.data_type); + + if !col.nullable && !col.is_primary_key { + def.push_str(" NOT NULL"); + } + + if col.is_primary_key && options.include_primary_keys { + if col.is_auto_increment && col.data_type.to_lowercase().contains("int") { + def.push_str(" PRIMARY KEY"); + } + } + + if let Some(ref default) = col.default_value { + if !col.is_auto_increment { + def.push_str(&format!(" DEFAULT {}", default)); + } + } + + def +} + +fn quote_identifier_postgres(name: &str) -> String { + format!("\"{}\"", name) +} + +fn format_table_ref_postgres(schema: Option<&str>, table: &str) -> String { + match schema { + Some(s) => format!("\"{}\".\"{}\"", s, table), + None => format!("\"{}\"", table), + } +} + +fn generate_mysql_ddl( + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + let mut sql = String::new(); + let table_ref = format_table_ref_mysql(schema, table); + + if options.include_drop_if_exists { + sql.push_str(&format!("DROP TABLE IF EXISTS {};\n", table_ref)); + } + + if options.include_create_table { + let create_keyword = if options.include_if_not_exists { + "CREATE TABLE IF NOT EXISTS" + } else { + "CREATE TABLE" + }; + + sql.push_str(&format!("{} {} (\n", create_keyword, table_ref)); + + let col_defs: Vec = columns.iter().map(|c| format_column_mysql(c)).collect(); + + let pk_cols: Vec = columns + .iter() + .filter(|c| c.is_primary_key) + .map(|c| quote_identifier_mysql(&c.name)) + .collect(); + + if !pk_cols.is_empty() && options.include_primary_keys { + let pk_def = format!(" PRIMARY KEY ({})", pk_cols.join(", ")); + sql.push_str(&col_defs.join(",\n")); + sql.push_str(",\n"); + sql.push_str(&pk_def); + } else { + sql.push_str(&col_defs.join(",\n")); + } + + sql.push_str("\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n"); + } + + if options.include_comments { + for col in columns { + if let Some(ref desc) = col.description { + if !desc.is_empty() { + sql.push_str(&format!( + "ALTER TABLE {} MODIFY COLUMN {} {} COMMENT '{}';\n", + table_ref, + quote_identifier_mysql(&col.name), + col.data_type, + desc.replace('\'', "''") + )); + } + } + } + } + + sql +} + +fn format_column_mysql(col: &ColumnInfo) -> String { + let name = quote_identifier_mysql(&col.name); + let mut def = format!(" {} {}", name, col.data_type); + + if col.is_auto_increment { + def.push_str(" AUTO_INCREMENT"); + } + + if !col.nullable && !col.is_primary_key { + def.push_str(" NOT NULL"); + } + + if let Some(ref default) = col.default_value { + if !col.is_auto_increment { + def.push_str(&format!(" DEFAULT {}", default)); + } + } + + def +} + +fn quote_identifier_mysql(name: &str) -> String { + format!("`{}`", name) +} + +fn format_table_ref_mysql(schema: Option<&str>, table: &str) -> String { + match schema { + Some(s) => format!("`{}`.`{}`", s, table), + None => format!("`{}`", table), + } +} + +fn generate_sqlite_ddl( + _schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + let mut sql = String::new(); + + if options.include_drop_if_exists { + sql.push_str(&format!("DROP TABLE IF EXISTS \"{}\";\n", table)); + } + + if options.include_create_table { + let create_keyword = if options.include_if_not_exists { + "CREATE TABLE IF NOT EXISTS" + } else { + "CREATE TABLE" + }; + + sql.push_str(&format!("{} \"{}\" (\n", create_keyword, table)); + + let col_defs: Vec = columns.iter().map(|c| format_column_sqlite(c)).collect(); + + let pk_cols: Vec = columns + .iter() + .filter(|c| c.is_primary_key) + .map(|c| format!("\"{}\"", c.name)) + .collect(); + + if !pk_cols.is_empty() && options.include_primary_keys { + let pk_def = format!(" PRIMARY KEY ({})", pk_cols.join(", ")); + sql.push_str(&col_defs.join(",\n")); + sql.push_str(",\n"); + sql.push_str(&pk_def); + } else { + sql.push_str(&col_defs.join(",\n")); + } + + sql.push_str("\n);\n"); + } + + sql +} + +fn format_column_sqlite(col: &ColumnInfo) -> String { + let name = format!("\"{}\"", col.name); + let mut def = format!(" {} {}", name, col.data_type); + + if col.is_primary_key && col.is_auto_increment { + def.push_str(" PRIMARY KEY AUTOINCREMENT"); + } else { + if !col.nullable && !col.is_primary_key { + def.push_str(" NOT NULL"); + } + + if let Some(ref default) = col.default_value { + def.push_str(&format!(" DEFAULT {}", default)); + } + } + + def +} + +fn generate_sqlserver_ddl( + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + let mut sql = String::new(); + let table_ref = format_table_ref_sqlserver(schema, table); + + if options.include_drop_if_exists { + sql.push_str(&format!( + "IF OBJECT_ID('{}', 'U') IS NOT NULL DROP TABLE {};\n", + table_ref, table_ref + )); + } + + if options.include_create_table { + sql.push_str(&format!("CREATE TABLE {} (\n", table_ref)); + + let col_defs: Vec = columns.iter().map(|c| format_column_sqlserver(c)).collect(); + + let pk_cols: Vec = columns + .iter() + .filter(|c| c.is_primary_key) + .map(|c| quote_identifier_sqlserver(&c.name)) + .collect(); + + if !pk_cols.is_empty() && options.include_primary_keys { + let pk_def = format!(" PRIMARY KEY ({})", pk_cols.join(", ")); + sql.push_str(&col_defs.join(",\n")); + sql.push_str(",\n"); + sql.push_str(&pk_def); + } else { + sql.push_str(&col_defs.join(",\n")); + } + + sql.push_str("\n);\n"); + } + + sql +} + +fn format_column_sqlserver(col: &ColumnInfo) -> String { + let name = quote_identifier_sqlserver(&col.name); + let mut def = format!(" {} {}", name, col.data_type); + + if col.is_auto_increment { + def.push_str(" IDENTITY(1,1)"); + } + + if !col.nullable && !col.is_primary_key { + def.push_str(" NOT NULL"); + } + + if let Some(ref default) = col.default_value { + if !col.is_auto_increment { + def.push_str(&format!(" DEFAULT {}", default)); + } + } + + def +} + +fn quote_identifier_sqlserver(name: &str) -> String { + format!("[{}]", name) +} + +fn format_table_ref_sqlserver(schema: Option<&str>, table: &str) -> String { + match schema { + Some(s) => format!("[{}].[{}]", s, table), + None => format!("[{}]", table), + } +} + +fn generate_generic_ddl( + schema: Option<&str>, + table: &str, + columns: &[ColumnInfo], + options: &DdlOptions, +) -> String { + generate_postgres_ddl(schema, table, columns, options) +} diff --git a/src-tauri/src/transfer/defaults.rs b/src-tauri/src/transfer/defaults.rs new file mode 100644 index 00000000..7fe446bc --- /dev/null +++ b/src-tauri/src/transfer/defaults.rs @@ -0,0 +1,47 @@ +//! Best-practice default configurations for transfer formats. + +use super::types::*; + +pub fn csv_export_defaults() -> CsvExportOptions { + CsvExportOptions { + delimiter: ',', + quote_char: '"', + encoding: "UTF-8".to_string(), + include_header: true, + quote_all: false, + line_ending: "LF".to_string(), + } +} + +pub fn jsonl_export_defaults() -> JsonlExportOptions { + JsonlExportOptions { + date_format: "ISO8601".to_string(), + } +} + +pub fn sql_export_defaults(table_name: &str) -> SqlExportOptions { + SqlExportOptions { + target_table: table_name.to_string(), + batch_size: 1000, + include_create_table: true, + include_drop_table: false, + target_engine: None, + } +} + +pub fn excel_export_defaults() -> ExcelExportOptions { + ExcelExportOptions { + sheet_name: "Sheet1".to_string(), + include_header: true, + auto_fit_columns: true, + freeze_header: true, + } +} + +pub fn csv_import_defaults() -> CsvImportOptions { + CsvImportOptions { + delimiter: ',', + encoding: "UTF-8".to_string(), + has_header: true, + } +} diff --git a/src-tauri/src/transfer/export.rs b/src-tauri/src/transfer/export.rs new file mode 100644 index 00000000..73b831d3 --- /dev/null +++ b/src-tauri/src/transfer/export.rs @@ -0,0 +1,471 @@ +//! Export implementation for CSV, JSONL, SQL, and Excel formats. + +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::time::Instant; + +use rust_xlsxwriter::{Workbook, Worksheet}; +use serde_json::Value as JsonValue; + +use super::defaults::*; +use super::progress::*; +use super::types::*; +use crate::database::{DatabaseAdapter, QueryValue}; + +/// Executes a data export operation. +pub async fn execute_export( + adapter: &A, + request: ExportRequest, + app_handle: &tauri::AppHandle, +) -> Result { + let start_time = Instant::now(); + let _operation_id = uuid::Uuid::new_v4().to_string(); + + let columns = request.source.columns.clone(); + let table = request.source.table.clone(); + let schema = request.schema.clone(); + + let csv_opts = request.csv_options.clone().unwrap_or_else(csv_export_defaults); + let jsonl_opts = request.jsonl_options.clone().unwrap_or_else(jsonl_export_defaults); + let sql_opts = request.sql_options.clone().unwrap_or_else(|| sql_export_defaults(&table)); + let excel_opts = request.excel_options.clone().unwrap_or_else(excel_export_defaults); + + let base_query = build_export_query(&schema, &table, &columns, &request.source); + + let count_query = build_count_query(&schema, &table, &request.source.where_clause); + let count_result = adapter.execute_query(&count_query).await.map_err(|e| e.to_string())?; + let total_rows = count_result.rows.first() + .and_then(|row| row.get("count")) + .and_then(|v| match v { + QueryValue::Int(n) => Some(*n as u64), + _ => None, + }) + .unwrap_or(0); + + emit_progress(app_handle, &create_progress("export", "preparing", 0, Some(total_rows), 0)); + + let output_path = Path::new(&request.output_path); + let file = File::create(output_path).map_err(|e| format!("Failed to create file: {}", e))?; + let mut writer = BufWriter::new(file); + + let mut processed_rows: u64 = 0; + let mut errors: Vec = Vec::new(); + let batch_size = 1000u64; + + match request.format { + ExportFormat::Csv => { + if csv_opts.include_header { + write_csv_header(&mut writer, &columns, csv_opts.delimiter).map_err(|e| e.to_string())?; + } + + let mut offset = 0u64; + while offset < total_rows { + let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let result = adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + for row in &result.rows { + write_csv_row(&mut writer, &columns, row, &csv_opts).map_err(|e| { + errors.push(TransferError { + row_number: Some(processed_rows + 1), + statement_number: None, + message: e, + sql: None, + }); + String::new() + })?; + processed_rows += 1; + } + + offset += batch_size; + emit_progress(app_handle, &create_progress("export", "processing", processed_rows, Some(total_rows), start_time.elapsed().as_millis() as u64)); + } + } + + ExportFormat::Jsonl => { + let mut offset = 0u64; + while offset < total_rows { + let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let result = adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + for row in &result.rows { + let json_obj = row_to_json_object(row, &jsonl_opts.date_format); + let json_line = serde_json::to_string(&json_obj).map_err(|e| e.to_string())?; + writer.write_all(json_line.as_bytes()).map_err(|e| e.to_string())?; + writer.write_all(b"\n").map_err(|e| e.to_string())?; + processed_rows += 1; + } + + offset += batch_size; + emit_progress(app_handle, &create_progress("export", "processing", processed_rows, Some(total_rows), start_time.elapsed().as_millis() as u64)); + } + } + + ExportFormat::Sql => { + if sql_opts.include_create_table { + let table_info = adapter.get_table_info(schema.as_deref(), None, &table).await.map_err(|e| e.to_string())?; + let create_stmt = generate_create_table_sql(&table, &table_info, sql_opts.include_drop_table); + writer.write_all(create_stmt.as_bytes()).map_err(|e| e.to_string())?; + writer.write_all(b"\n\n").map_err(|e| e.to_string())?; + } + + let mut batch_rows: Vec> = Vec::new(); + let mut offset = 0u64; + + while offset < total_rows { + let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let result = adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + for row in &result.rows { + let values: Vec = columns.iter() + .map(|col| row.get(col).cloned().unwrap_or(QueryValue::Null)) + .collect(); + batch_rows.push(values); + processed_rows += 1; + + if batch_rows.len() >= sql_opts.batch_size as usize { + let insert_stmt = generate_insert_sql(&schema, &table, &columns, &batch_rows); + writer.write_all(insert_stmt.as_bytes()).map_err(|e| e.to_string())?; + writer.write_all(b"\n").map_err(|e| e.to_string())?; + batch_rows.clear(); + } + } + + offset += batch_size; + emit_progress(app_handle, &create_progress("export", "processing", processed_rows, Some(total_rows), start_time.elapsed().as_millis() as u64)); + } + + if !batch_rows.is_empty() { + let insert_stmt = generate_insert_sql(&schema, &table, &columns, &batch_rows); + writer.write_all(insert_stmt.as_bytes()).map_err(|e| e.to_string())?; + } + } + + ExportFormat::Excel => { + let mut workbook = Workbook::new(); + let worksheet = workbook.add_worksheet().set_name(&excel_opts.sheet_name).map_err(|e| e.to_string())?; + + if excel_opts.include_header { + for (col_idx, col_name) in columns.iter().enumerate() { + worksheet.write_string(0, col_idx as u16, col_name).map_err(|e| e.to_string())?; + } + } + + if excel_opts.freeze_header && excel_opts.include_header { + worksheet.set_freeze_panes(1, 0).map_err(|e| e.to_string())?; + } + + let header_row_offset = if excel_opts.include_header { 1 } else { 0 }; + + let mut offset = 0u64; + let mut row_idx = header_row_offset; + + while offset < total_rows { + let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let result = adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + for row in &result.rows { + for (col_idx, col_name) in columns.iter().enumerate() { + let value = row.get(col_name).cloned().unwrap_or(QueryValue::Null); + write_excel_cell(worksheet, row_idx, col_idx as u16, &value)?; + } + row_idx += 1; + processed_rows += 1; + } + + offset += batch_size; + emit_progress(app_handle, &create_progress("export", "processing", processed_rows, Some(total_rows), start_time.elapsed().as_millis() as u64)); + } + + if excel_opts.auto_fit_columns { + let max_col = columns.len() as u16; + for col_idx in 0..max_col { + worksheet.set_column_width(col_idx, 12.0).map_err(|e| e.to_string())?; + } + } + + workbook.save(output_path).map_err(|e| format!("Failed to save Excel file: {}", e))?; + } + } + + writer.flush().map_err(|e| format!("Failed to flush file: {}", e))?; + + let file_size = std::fs::metadata(output_path) + .map(|m| m.len()) + .unwrap_or(0); + + emit_progress(app_handle, &create_progress("export", "finalizing", processed_rows, Some(total_rows), start_time.elapsed().as_millis() as u64)); + + Ok(TransferResult { + success: errors.is_empty(), + total_rows, + processed_rows, + skipped_rows: 0, + error_count: errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: Some(request.output_path), + output_size_bytes: Some(file_size), + errors, + }) +} + +fn build_export_query(schema: &Option, table: &str, columns: &[String], source: &ExportSource) -> String { + let schema_prefix = schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(); + let cols = columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let mut query = format!("SELECT {} FROM {}\"{}\"", cols, schema_prefix, table); + + if let Some(ref where_clause) = source.where_clause { + query.push_str(&format!(" WHERE {}", where_clause)); + } + + if let Some(ref order_by) = source.order_by { + query.push_str(&format!(" ORDER BY {}", order_by)); + } + + query +} + +fn build_count_query(schema: &Option, table: &str, where_clause: &Option) -> String { + let schema_prefix = schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(); + let mut query = format!("SELECT COUNT(*) AS count FROM {}\"{}\"", schema_prefix, table); + + if let Some(ref where_clause) = where_clause { + query.push_str(&format!(" WHERE {}", where_clause)); + } + + query +} + +fn write_csv_header(writer: &mut BufWriter, columns: &[String], delimiter: char) -> Result<(), std::io::Error> { + let header = columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(&delimiter.to_string()); + writer.write_all(header.as_bytes())?; + writer.write_all(b"\n")?; + Ok(()) +} + +fn write_csv_row(writer: &mut BufWriter, columns: &[String], row: &crate::database::QueryRow, opts: &CsvExportOptions) -> Result<(), String> { + let values: Vec = columns.iter() + .map(|col| { + match row.get(col) { + Some(QueryValue::Null) => "".to_string(), + Some(QueryValue::Bool(b)) => b.to_string(), + Some(QueryValue::Int(n)) => n.to_string(), + Some(QueryValue::Float(f)) => f.to_string(), + Some(QueryValue::String(s)) => { + if opts.quote_all || s.contains(&opts.delimiter.to_string()) || s.contains('"') || s.contains('\n') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.clone() + } + }, + Some(QueryValue::Bytes(b)) => format!("0x{}", hex::encode(b)), + Some(QueryValue::DateTime(dt)) => dt.clone(), + None => "".to_string(), + } + }) + .collect(); + + let line = values.join(&opts.delimiter.to_string()); + writer.write_all(line.as_bytes()).map_err(|e| e.to_string())?; + writer.write_all(b"\n").map_err(|e| e.to_string())?; + Ok(()) +} + +fn row_to_json_object(row: &crate::database::QueryRow, _date_format: &str) -> JsonValue { + let mut obj = serde_json::Map::new(); + for (key, value) in row { + let json_val = match value { + QueryValue::Null => JsonValue::Null, + QueryValue::Bool(b) => JsonValue::Bool(*b), + QueryValue::Int(n) => JsonValue::Number((*n).into()), + QueryValue::Float(f) => JsonValue::Number(serde_json::Number::from_f64(*f).unwrap_or_else(|| serde_json::Number::from(0))), + QueryValue::String(s) => JsonValue::String(s.clone()), + QueryValue::Bytes(b) => JsonValue::String(hex::encode(b)), + QueryValue::DateTime(dt) => JsonValue::String(dt.clone()), + }; + obj.insert(key.clone(), json_val); + } + JsonValue::Object(obj) +} + +fn generate_create_table_sql(table: &str, _table_info: &crate::database::TableInfo, include_drop: bool) -> String { + let mut sql = String::new(); + + if include_drop { + sql.push_str(&format!("DROP TABLE IF EXISTS \"{}\";\n", table)); + } + + sql.push_str(&format!("CREATE TABLE \"{}\" (\n", table)); + + sql.push_str(");"); + sql +} + +fn generate_insert_sql(schema: &Option, table: &str, columns: &[String], rows: &[Vec]) -> String { + let schema_prefix = schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(); + let col_list = columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let values_list: Vec = rows.iter() + .map(|row| { + let vals: Vec = row.iter() + .map(query_value_to_sql_literal) + .collect(); + format!("({})", vals.join(", ")) + }) + .collect(); + + format!("INSERT INTO {}\"{}\" ({}) VALUES {};", schema_prefix, table, col_list, values_list.join(", ")) +} + +fn query_value_to_sql_literal(value: &QueryValue) -> String { + match value { + QueryValue::Null => "NULL".to_string(), + QueryValue::Bool(b) => b.to_string(), + QueryValue::Int(n) => n.to_string(), + QueryValue::Float(f) => f.to_string(), + QueryValue::String(s) => format!("'{}'", s.replace('\'', "''")), + QueryValue::Bytes(b) => format!("'{}'", hex::encode(b)), + QueryValue::DateTime(dt) => format!("'{}'", dt), + } +} + +/// Generates a preview of export data. +pub async fn preview_export( + adapter: &A, + request: ExportRequest, + preview_rows: u32, +) -> Result { + let columns = request.source.columns.clone(); + let table = request.source.table.clone(); + let schema = request.schema.clone(); + + let base_query = build_export_query(&schema, &table, &columns, &request.source); + let query = format!("{} LIMIT {}", base_query, preview_rows); + + let result = adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + let sample_rows: Vec> = result.rows.iter() + .map(|row| { + columns.iter() + .map(|col| { + match row.get(col) { + Some(QueryValue::Null) => "".to_string(), + Some(QueryValue::Bool(b)) => b.to_string(), + Some(QueryValue::Int(n)) => n.to_string(), + Some(QueryValue::Float(f)) => f.to_string(), + Some(QueryValue::String(s)) => s.clone(), + Some(QueryValue::Bytes(b)) => hex::encode(b), + Some(QueryValue::DateTime(dt)) => dt.clone(), + None => "".to_string(), + } + }) + .collect() + }) + .collect(); + + let count_query = build_count_query(&schema, &table, &request.source.where_clause); + let count_result = adapter.execute_query(&count_query).await.map_err(|e| e.to_string())?; + let total_rows_estimate = count_result.rows.first() + .and_then(|row| row.get("count")) + .and_then(|v| match v { + QueryValue::Int(n) => Some(*n as u64), + _ => None, + }); + + let formatted_preview = format_preview(&request.format, &columns, &sample_rows, preview_rows); + + Ok(ExportPreview { + columns, + sample_rows, + total_rows_estimate, + formatted_preview, + }) +} + +fn format_preview(format: &ExportFormat, columns: &[String], rows: &[Vec], _limit: u32) -> String { + match format { + ExportFormat::Csv => { + let header = columns.join(","); + let data_lines: Vec = rows.iter() + .map(|row| row.join(",")) + .collect(); + format!("{}\n{}", header, data_lines.join("\n")) + }, + ExportFormat::Jsonl => { + let json_lines: Vec = rows.iter() + .map(|row| { + let mut obj = serde_json::Map::new(); + for (col, val) in columns.iter().zip(row.iter()) { + obj.insert(col.clone(), JsonValue::String(val.clone())); + } + serde_json::to_string(&JsonValue::Object(obj)).unwrap_or_default() + }) + .collect(); + json_lines.join("\n") + }, + ExportFormat::Sql => { + let col_list = columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let values_list: Vec = rows.iter() + .map(|row| { + let vals: Vec = row.iter() + .map(|v| if v.is_empty() { "NULL".to_string() } else { format!("'{}'", v) }) + .collect(); + format!("({})", vals.join(", ")) + }) + .collect(); + + format!("INSERT INTO \"table\" ({}) VALUES {};", col_list, values_list.join(", ")) + }, + ExportFormat::Excel => { + let header = columns.join("\t"); + let data_lines: Vec = rows.iter() + .map(|row| row.join("\t")) + .collect(); + format!("{}\n{}", header, data_lines.join("\n")) + }, + } +} + +fn write_excel_cell(worksheet: &mut Worksheet, row: u32, col: u16, value: &QueryValue) -> Result<(), String> { + match value { + QueryValue::Null => Ok(()), + QueryValue::Bool(b) => { + worksheet.write_boolean(row, col, *b).map_err(|e| e.to_string())?; + Ok(()) + }, + QueryValue::Int(n) => { + worksheet.write_number(row, col, *n as f64).map_err(|e| e.to_string())?; + Ok(()) + }, + QueryValue::Float(f) => { + worksheet.write_number(row, col, *f).map_err(|e| e.to_string())?; + Ok(()) + }, + QueryValue::String(s) => { + worksheet.write_string(row, col, s).map_err(|e| e.to_string())?; + Ok(()) + }, + QueryValue::Bytes(b) => { + worksheet.write_string(row, col, &hex::encode(b)).map_err(|e| e.to_string())?; + Ok(()) + }, + QueryValue::DateTime(dt) => { + worksheet.write_string(row, col, dt).map_err(|e| e.to_string())?; + Ok(()) + }, + } +} \ No newline at end of file diff --git a/src-tauri/src/transfer/import.rs b/src-tauri/src/transfer/import.rs new file mode 100644 index 00000000..3244f44b --- /dev/null +++ b/src-tauri/src/transfer/import.rs @@ -0,0 +1,691 @@ +//! Import implementation for CSV, JSONL, SQL, and Excel formats. + +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::time::Instant; + +use calamine::{Reader, Xlsx, open_workbook}; + +use super::defaults::*; +use super::progress::*; +use super::types::*; + +/// Executes a data import operation. +pub async fn execute_import( + adapter: &A, + request: ImportRequest, + app_handle: &tauri::AppHandle, +) -> Result { + let start_time = Instant::now(); + + emit_progress(app_handle, &create_progress("import", "preparing", 0, None, 0)); + + let csv_opts = request.csv_options.clone().unwrap_or_else(csv_import_defaults); + + let file_path = Path::new(&request.file_path); + let file = File::open(file_path).map_err(|e| format!("Failed to open file: {}", e))?; + + let mut processed_rows: u64 = 0; + let mut skipped_rows: u64 = 0; + let mut errors: Vec = Vec::new(); + + match request.format { + ImportFormat::Csv => { + let reader = BufReader::new(file); + let delimiter = csv_opts.delimiter; + + let mut lines = reader.lines().peekable(); + + let header_line = if csv_opts.has_header { + lines.next() + .transpose() + .map_err(|e| format!("Failed to read header: {}", e))? + .unwrap_or_default() + } else { + request.column_mappings.iter() + .map(|m| m.source_column.clone()) + .collect::>() + .join(&delimiter.to_string()) + }; + + let header_columns: Vec = if csv_opts.has_header { + parse_csv_line(&header_line, delimiter) + } else { + request.column_mappings.iter() + .map(|m| m.source_column.clone()) + .collect() + }; + + let mut batch_values: Vec> = Vec::new(); + + for (row_num, line_result) in lines.enumerate() { + let line = line_result.map_err(|e| format!("Failed to read line {}: {}", row_num + 2, e))?; + + if line.trim().is_empty() { + continue; + } + + let values = parse_csv_line(&line, delimiter); + + let mapped_values: Vec = header_columns.iter() + .enumerate() + .filter_map(|(i, col)| { + let mapping = request.column_mappings.iter() + .find(|m| m.source_column == *col); + + if mapping.is_none() || mapping.and_then(|m| m.target_column.as_ref()).is_none() { + None + } else { + Some(values.get(i).cloned().unwrap_or_default()) + } + }) + .collect(); + + batch_values.push(mapped_values); + processed_rows += 1; + + if batch_values.len() >= request.batch_size as usize { + let insert_result = execute_batch_insert( + adapter, + &request, + &batch_values, + processed_rows - batch_values.len() as u64, + ).await; + + match insert_result { + Ok(count) => { + processed_rows = processed_rows - batch_values.len() as u64 + count; + }, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows - batch_values.len() as u64 + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += batch_values.len() as u64; + } + } + + batch_values.clear(); + emit_progress(app_handle, &create_progress("import", "processing", processed_rows, None, start_time.elapsed().as_millis() as u64)); + } + } + + if !batch_values.is_empty() { + let insert_result = execute_batch_insert( + adapter, + &request, + &batch_values, + processed_rows - batch_values.len() as u64, + ).await; + + match insert_result { + Ok(count) => { + processed_rows = processed_rows - batch_values.len() as u64 + count; + }, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows - batch_values.len() as u64 + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += batch_values.len() as u64; + } + } + } + }, + + ImportFormat::Jsonl => { + let reader = BufReader::new(file); + let mut batch_values: Vec> = Vec::new(); +let _target_columns: Vec = request.column_mappings.iter() + .filter_map(|m| m.target_column.clone()) + .collect(); + + for (row_num, line_result) in reader.lines().enumerate() { + let line = line_result.map_err(|e| format!("Failed to read line {}: {}", row_num + 1, e))?; + + if line.trim().is_empty() { + continue; + } + + let json_obj: serde_json::Value = serde_json::from_str(&line) + .map_err(|e| { + errors.push(TransferError { + row_number: Some(row_num as u64 + 1), + statement_number: None, + message: format!("JSON parse error: {}", e), + sql: None, + }); + String::new() + })?; + + if !json_obj.is_object() { + skipped_rows += 1; + continue; + } + + let obj = json_obj.as_object().unwrap(); + let values: Vec = request.column_mappings.iter() + .filter_map(|m| { + let _target_col = m.target_column.as_ref()?; + let source_val = obj.get(&m.source_column); + match source_val { + Some(serde_json::Value::Null) => Some(String::new()), + Some(serde_json::Value::Bool(b)) => Some(b.to_string()), + Some(serde_json::Value::Number(n)) => Some(n.to_string()), + Some(serde_json::Value::String(s)) => Some(s.clone()), + Some(serde_json::Value::Array(arr)) => Some(serde_json::to_string(arr).unwrap_or_default()), + Some(serde_json::Value::Object(obj)) => Some(serde_json::to_string(obj).unwrap_or_default()), + None => Some(String::new()), + } + }) + .collect(); + + batch_values.push(values); + processed_rows += 1; + + if batch_values.len() >= request.batch_size as usize { + let insert_result = execute_batch_insert( + adapter, + &request, + &batch_values, + processed_rows - batch_values.len() as u64, + ).await; + + match insert_result { + Ok(count) => { + processed_rows = processed_rows - batch_values.len() as u64 + count; + }, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows - batch_values.len() as u64 + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += batch_values.len() as u64; + } + } + + batch_values.clear(); + emit_progress(app_handle, &create_progress("import", "processing", processed_rows, None, start_time.elapsed().as_millis() as u64)); + } + } + + if !batch_values.is_empty() { + execute_batch_insert(adapter, &request, &batch_values, processed_rows - batch_values.len() as u64).await?; + } + }, + + ImportFormat::Sql => { + let reader = BufReader::new(file); + let mut current_statement = String::new(); + let mut statement_count: u64 = 0; + + for line_result in reader.lines() { + let line = line_result.map_err(|e| format!("Failed to read line: {}", e))?; + + if line.trim().is_empty() { + continue; + } + + current_statement.push_str(&line); + current_statement.push('\n'); + + if line.trim().ends_with(';') { + let sql = current_statement.trim(); + if !sql.is_empty() { + statement_count += 1; + match adapter.execute_query(sql).await { + Ok(_) => processed_rows += 1, + Err(e) => { + errors.push(TransferError { + row_number: None, + statement_number: Some(statement_count), + message: e.to_string(), + sql: Some(sql.to_string()), + }); + skipped_rows += 1; + } + } + } + current_statement.clear(); + + emit_progress(app_handle, &create_progress("import", "processing", processed_rows, None, start_time.elapsed().as_millis() as u64)); + } + } + + if !current_statement.trim().is_empty() { + statement_count += 1; + match adapter.execute_query(current_statement.trim()).await { + Ok(_) => processed_rows += 1, + Err(e) => { + errors.push(TransferError { + row_number: None, + statement_number: Some(statement_count), + message: e.to_string(), + sql: Some(current_statement.trim().to_string()), + }); + skipped_rows += 1; + } + } + } + }, + + ImportFormat::Excel => { + let mut workbook: Xlsx<_> = open_workbook(file_path).map_err(|e| format!("Failed to open Excel file: {}", e))?; + + let sheet_name = request.excel_options.as_ref() + .map(|o| o.sheet_name.clone()) + .unwrap_or_else(|| "Sheet1".to_string()); + + let range = workbook.worksheet_range(&sheet_name) + .ok_or_else(|| format!("Sheet '{}' not found", sheet_name))? + .map_err(|e| format!("Failed to read sheet '{}': {:?}", sheet_name, e))?; + + let has_header = request.excel_options.as_ref() + .map(|o| o.has_header) + .unwrap_or(true); + + let mut rows_iter = range.rows(); + let header_row: Vec = if has_header { + rows_iter.next() + .map(|row| row.iter().map(|c: &calamine::DataType| c.to_string()).collect()) + .unwrap_or_default() + } else { + request.column_mappings.iter() + .map(|m| m.source_column.clone()) + .collect() + }; + + let mut batch_values: Vec> = Vec::new(); + + for row in range.rows() { + let values: Vec = header_row.iter() + .enumerate() + .filter_map(|(col_idx, col)| { + let mapping = request.column_mappings.iter() + .find(|m| m.source_column == *col); + + if mapping.is_none() || mapping.and_then(|m| m.target_column.as_ref()).is_none() { + None + } else { + Some(row.get(col_idx).map(|c: &calamine::DataType| c.to_string()).unwrap_or_default()) + } + }) + .collect(); + + batch_values.push(values); + processed_rows += 1; + + if batch_values.len() >= request.batch_size as usize { + let insert_result = execute_batch_insert( + adapter, + &request, + &batch_values, + processed_rows - batch_values.len() as u64, + ).await; + + match insert_result { + Ok(count) => { + processed_rows = processed_rows - batch_values.len() as u64 + count; + }, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows - batch_values.len() as u64 + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += batch_values.len() as u64; + } + } + + batch_values.clear(); + emit_progress(app_handle, &create_progress("import", "processing", processed_rows, None, start_time.elapsed().as_millis() as u64)); + } + } + + if !batch_values.is_empty() { + let insert_result = execute_batch_insert( + adapter, + &request, + &batch_values, + processed_rows - batch_values.len() as u64, + ).await; + + match insert_result { + Ok(count) => { + processed_rows = processed_rows - batch_values.len() as u64 + count; + }, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows - batch_values.len() as u64 + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += batch_values.len() as u64; + } + } + } + } + } + + emit_progress(app_handle, &create_progress("import", "finalizing", processed_rows, None, start_time.elapsed().as_millis() as u64)); + + Ok(TransferResult { + success: errors.is_empty(), + total_rows: processed_rows + skipped_rows, + processed_rows, + skipped_rows, + error_count: errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors, + }) +} + +fn parse_csv_line(line: &str, delimiter: char) -> Vec { + let mut values: Vec = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + let chars = line.chars().peekable(); + + for ch in chars { + if ch == '"' { + in_quotes = !in_quotes; + } else if ch == delimiter && !in_quotes { + values.push(current.trim().to_string()); + current = String::new(); + } else { + current.push(ch); + } + } + values.push(current.trim().to_string()); + values +} + +async fn execute_batch_insert( + adapter: &A, + request: &ImportRequest, + batch: &[Vec], + _start_row: u64, +) -> Result { + if batch.is_empty() { + return Ok(0); + } + + let schema_prefix = request.schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(); + let target_columns: Vec = request.column_mappings.iter() + .filter_map(|m| m.target_column.clone()) + .collect(); + + let col_list = target_columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let values_list: Vec = batch.iter() + .map(|row| { + let vals: Vec = row.iter() + .map(|v| { + if v.is_empty() { + "NULL".to_string() + } else { + format!("'{}'", v.replace('\'', "''")) + } + }) + .collect(); + format!("({})", vals.join(", ")) + }) + .collect(); + + let sql = format!( + "INSERT INTO {}\"{}\" ({}) VALUES {}", + schema_prefix, + request.table, + col_list, + values_list.join(", ") + ); + + let result = adapter.execute_query(&sql).await.map_err(|e| e.to_string())?; + + Ok(result.rows_affected.unwrap_or(batch.len() as u64)) +} + +/// Detects file format and metadata. +pub fn detect_file(file_path: &str) -> Result { + let path = Path::new(file_path); + let file = File::open(path).map_err(|e| format!("Failed to open file: {}", e))?; + + let file_size_bytes = std::fs::metadata(path) + .map(|m| m.len()) + .unwrap_or(0); + + let extension = path.extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()); + + let format = match extension.as_deref() { + Some("csv") => ImportFormat::Csv, + Some("jsonl") | Some("json") => ImportFormat::Jsonl, + Some("sql") => ImportFormat::Sql, + Some("xlsx") | Some("xls") => ImportFormat::Excel, + _ => return Err("Unknown file format".to_string()), + }; + + let reader = BufReader::new(file); + let mut lines_iter = reader.lines(); + + let first_line = lines_iter.next() + .transpose() + .map_err(|e| format!("Failed to read file: {}", e))? + .unwrap_or_default(); + + let (columns, csv_delimiter, has_header) = match format { + ImportFormat::Csv => { + let delimiter = detect_csv_delimiter(&first_line); + let cols = parse_csv_line(&first_line, delimiter); + let has_header = cols.iter().all(|c| !c.is_empty() && !c.chars().all(char::is_numeric)); + (cols, Some(delimiter), Some(has_header)) + }, + ImportFormat::Jsonl => { + let json_obj: serde_json::Value = serde_json::from_str(&first_line) + .map_err(|_| "Invalid JSONL format".to_string())?; + let cols = json_obj.as_object() + .map(|obj| obj.keys().cloned().collect()) + .unwrap_or_default(); + (cols, None, None) + }, + ImportFormat::Sql => (Vec::new(), None, None), + ImportFormat::Excel => { + let mut workbook: Xlsx<_> = open_workbook(file_path) + .map_err(|e| format!("Failed to open Excel file for detection: {}", e))?; + let range = workbook.worksheet_range("Sheet1") + .ok_or("Sheet 'Sheet1' not found")? + .map_err(|e| format!("Failed to read sheet: {:?}", e))?; + let cols = range.rows() + .next() + .map(|row| row.iter().map(|c: &calamine::DataType| c.to_string()).collect()) + .unwrap_or_default(); + (cols, None, Some(true)) + }, + }; + + let estimated_rows = estimate_row_count(file_path, file_size_bytes, &format); + + Ok(FileDetectionResult { + format, + encoding: "UTF-8".to_string(), + estimated_rows, + file_size_bytes, + columns, + csv_delimiter, + has_header, + }) +} + +fn detect_csv_delimiter(line: &str) -> char { + let comma_count = line.chars().filter(|&c| c == ',').count(); + let tab_count = line.chars().filter(|&c| c == '\t').count(); + let semicolon_count = line.chars().filter(|&c| c == ';').count(); + + if tab_count > comma_count && tab_count > semicolon_count { + '\t' + } else if semicolon_count > comma_count { + ';' + } else { + ',' + } +} + +fn estimate_row_count(file_path: &str, file_size: u64, _format: &ImportFormat) -> Option { + let file = File::open(file_path).ok()?; + let reader = BufReader::new(file); + + let sample_lines: Vec = reader.lines() + .take(100) + .filter_map(|l| l.ok()) + .collect(); + + if sample_lines.is_empty() { + return None; + } + + let avg_line_size = sample_lines.iter() + .map(|l| l.len()) + .sum::() / sample_lines.len(); + + if avg_line_size == 0 { + return None; + } + + Some(file_size / avg_line_size as u64) +} + +/// Generates a preview of import data. +pub fn preview_import(file_path: &str, format: ImportFormat, preview_rows: u32) -> Result { + let file = File::open(file_path).map_err(|e| format!("Failed to open file: {}", e))?; + let reader = BufReader::new(file); + + let mut columns: Vec = Vec::new(); + let mut sample_rows: Vec> = Vec::new(); + + match format { + ImportFormat::Csv => { + let mut lines_iter = reader.lines(); + + let first_line = lines_iter.next() + .transpose() + .map_err(|e| format!("Failed to read file: {}", e))? + .unwrap_or_default(); + + let delimiter = detect_csv_delimiter(&first_line); + columns = parse_csv_line(&first_line, delimiter); + + for line_result in lines_iter.take(preview_rows as usize) { + let line = line_result.map_err(|e| format!("Failed to read line: {}", e))?; + sample_rows.push(parse_csv_line(&line, delimiter)); + } + }, + ImportFormat::Jsonl => { + for (i, line_result) in reader.lines().enumerate() { + if i > preview_rows as usize { + break; + } + + let line = line_result.map_err(|e| format!("Failed to read line: {}", e))?; + let json_obj: serde_json::Value = serde_json::from_str(&line) + .map_err(|e| format!("JSON parse error: {}", e))?; + + if i == 0 && json_obj.is_object() { + columns = json_obj.as_object().unwrap().keys().cloned().collect(); + } + + if json_obj.is_object() { + let obj = json_obj.as_object().unwrap(); + let row: Vec = columns.iter() + .map(|col| { + match obj.get(col) { + Some(serde_json::Value::Null) => String::new(), + Some(serde_json::Value::Bool(b)) => b.to_string(), + Some(serde_json::Value::Number(n)) => n.to_string(), + Some(serde_json::Value::String(s)) => s.clone(), + Some(v) => v.to_string(), + None => String::new(), + } + }) + .collect(); + sample_rows.push(row); + } + } + }, + ImportFormat::Sql => { + let mut statements: Vec = Vec::new(); + let mut current_statement = String::new(); + + for line_result in reader.lines().take(50) { + let line = line_result.map_err(|e| format!("Failed to read line: {}", e))?; + if line.trim().is_empty() { + continue; + } + current_statement.push_str(&line); + current_statement.push('\n'); + if line.trim().ends_with(';') { + statements.push(current_statement.trim().to_string()); + current_statement.clear(); + } + } + + columns = vec!["statement".to_string()]; + sample_rows = statements.iter() + .take(preview_rows as usize) + .map(|s| vec![s.clone()]) + .collect(); + }, + ImportFormat::Excel => { + let mut workbook: Xlsx<_> = open_workbook(file_path) + .map_err(|e| format!("Failed to open Excel file: {}", e))?; + + let sheet_name = "Sheet1"; + let range = workbook.worksheet_range(sheet_name) + .ok_or_else(|| format!("Sheet '{}' not found", sheet_name))? + .map_err(|e| format!("Failed to read sheet: {:?}", e))?; + + let has_header = true; + + if has_header { + columns = range.rows() + .next() + .map(|row| row.iter().map(|c: &calamine::DataType| c.to_string()).collect()) + .unwrap_or_default(); + } + + for (row_idx, row) in range.rows().enumerate() { + if row_idx == 0 && has_header { + continue; + } + if sample_rows.len() >= preview_rows as usize { + break; + } + let row_values: Vec = row.iter() + .map(|cell: &calamine::DataType| cell.to_string()) + .collect(); + sample_rows.push(row_values); + } + } + } + + Ok(ExportPreview { + columns, + sample_rows, + total_rows_estimate: None, + formatted_preview: String::new(), + }) +} \ No newline at end of file diff --git a/src-tauri/src/transfer/migration.rs b/src-tauri/src/transfer/migration.rs new file mode 100644 index 00000000..dd8c75f4 --- /dev/null +++ b/src-tauri/src/transfer/migration.rs @@ -0,0 +1,387 @@ +//! Cross-engine data migration implementation. + +use std::time::Instant; + +use crate::database::{DatabaseAdapter, QueryValue}; +use crate::database::types::ColumnInfo; +use crate::database::DatabaseType; + +use super::progress::*; +use super::types::*; + +pub async fn execute_migration( + source_adapter: &A1, + target_adapter: &A2, + request: MigrationRequest, + app_handle: &tauri::AppHandle, +) -> Result { + let start_time = Instant::now(); + let mut total_processed: u64 = 0; + let mut total_skipped: u64 = 0; + let mut total_errors: Vec = Vec::new(); + + emit_progress(app_handle, &create_progress("migration", "preparing", 0, None, 0)); + + for (table_idx, table_plan) in request.table_plans.iter().enumerate() { + emit_progress(app_handle, &TransferProgress { + operation: "migration".to_string(), + phase: "processing".to_string(), + current_table: Some(table_plan.source_table.clone()), + total_rows: None, + processed_rows: total_processed, + skipped_rows: total_skipped, + error_count: total_errors.len() as u64, + percent: 0.0, + elapsed_ms: start_time.elapsed().as_millis() as u64, + estimated_remaining_ms: None, + message: Some(format!("Migrating table {} of {}", table_idx + 1, request.table_plans.len())), + }); + + let table_result = migrate_table( + source_adapter, + target_adapter, + &request, + table_plan, + app_handle, + start_time, + ).await; + + match table_result { + Ok(result) => { + total_processed += result.processed_rows; + total_skipped += result.skipped_rows; + if !result.success { + total_errors.extend(result.errors); + } + }, + Err(e) => { + total_errors.push(TransferError { + row_number: None, + statement_number: None, + message: format!("Table {} failed: {}", table_plan.source_table, e), + sql: None, + }); + if request.on_error == MigrationErrorStrategy::Abort { + break; + } + } + } + } + + emit_progress(app_handle, &create_progress("migration", "finalizing", total_processed, None, start_time.elapsed().as_millis() as u64)); + + Ok(TransferResult { + success: total_errors.is_empty(), + total_rows: total_processed + total_skipped, + processed_rows: total_processed, + skipped_rows: total_skipped, + error_count: total_errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors: total_errors, + }) +} + +async fn migrate_table( + source_adapter: &A1, + target_adapter: &A2, + request: &MigrationRequest, + table_plan: &MigrationTablePlan, + app_handle: &tauri::AppHandle, + start_time: Instant, +) -> Result { + let mut processed_rows: u64 = 0; + let mut skipped_rows: u64 = 0; + let mut errors: Vec = Vec::new(); + + let source_columns: Vec = table_plan.column_mappings.iter() + .map(|m| m.source_column.clone()) + .collect(); + + let target_columns: Vec = table_plan.column_mappings.iter() + .map(|m| m.target_column.clone()) + .collect(); + + let _schema_prefix = request.target_schema.as_ref() + .map(|s| format!("\"{}\".", s)) + .unwrap_or_default(); + + let count_query = format!( + "SELECT COUNT(*) AS count FROM {}\"{}\"", + request.source_schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(), + table_plan.source_table + ); + + let count_result = source_adapter.execute_query(&count_query).await.map_err(|e| e.to_string())?; + let total_rows = count_result.rows.first() + .and_then(|row| row.get("count")) + .and_then(|v| match v { + QueryValue::Int(n) => Some(*n as u64), + _ => None, + }) + .unwrap_or(0); + + let col_list = source_columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let base_query = format!( + "SELECT {} FROM {}\"{}\"", + col_list, + request.source_schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(), + table_plan.source_table + ); + + let batch_size = request.batch_size as u64; + let mut offset = 0u64; + + while offset < total_rows { + let query = format!("{} LIMIT {} OFFSET {}", base_query, batch_size, offset); + let result = source_adapter.execute_query(&query).await.map_err(|e| e.to_string())?; + + let insert_result = insert_batch_to_target( + target_adapter, + request, + table_plan, + &result.rows, + &source_columns, + &target_columns, + ).await; + + match insert_result { + Ok(count) => processed_rows += count, + Err(e) => { + errors.push(TransferError { + row_number: Some(processed_rows + 1), + statement_number: None, + message: e, + sql: None, + }); + skipped_rows += result.rows.len() as u64; + } + } + + offset += batch_size; + + emit_progress(app_handle, &TransferProgress { + operation: "migration".to_string(), + phase: "processing".to_string(), + current_table: Some(table_plan.source_table.clone()), + total_rows: Some(total_rows), + processed_rows: processed_rows, + skipped_rows: skipped_rows, + error_count: errors.len() as u64, + percent: if total_rows > 0 { (processed_rows as f32 / total_rows as f32) * 100.0 } else { 0.0 }, + elapsed_ms: start_time.elapsed().as_millis() as u64, + estimated_remaining_ms: None, + message: None, + }); + } + + Ok(TransferResult { + success: errors.is_empty(), + total_rows, + processed_rows, + skipped_rows, + error_count: errors.len() as u64, + duration_ms: start_time.elapsed().as_millis() as u64, + output_path: None, + output_size_bytes: None, + errors, + }) +} + +async fn insert_batch_to_target( + target_adapter: &A, + request: &MigrationRequest, + table_plan: &MigrationTablePlan, + rows: &[crate::database::QueryRow], + source_columns: &[String], + target_columns: &[String], +) -> Result { + if rows.is_empty() { + return Ok(0); + } + + let schema_prefix = request.target_schema.as_ref() + .map(|s| format!("\"{}\".", s)) + .unwrap_or_default(); + + let col_list = target_columns.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", "); + + let values_list: Vec = rows.iter() + .map(|row| { + let vals: Vec = source_columns.iter() + .map(|col| { + match row.get(col) { + Some(QueryValue::Null) => "NULL".to_string(), + Some(QueryValue::Bool(b)) => b.to_string(), + Some(QueryValue::Int(n)) => n.to_string(), + Some(QueryValue::Float(f)) => f.to_string(), + Some(QueryValue::String(s)) => format!("'{}'", s.replace('\'', "''")), + Some(QueryValue::Bytes(b)) => format!("'{}'", hex::encode(b)), + Some(QueryValue::DateTime(dt)) => format!("'{}'", dt), + None => "NULL".to_string(), + } + }) + .collect(); + format!("({})", vals.join(", ")) + }) + .collect(); + + let sql = format!( + "INSERT INTO {}\"{}\" ({}) VALUES {}", + schema_prefix, + table_plan.target_table, + col_list, + values_list.join(", ") + ); + + let result = target_adapter.execute_query(&sql).await.map_err(|e| e.to_string())?; + Ok(result.rows_affected.unwrap_or(rows.len() as u64)) +} + +pub async fn preview_migration( + source_adapter: &A, + request: &MigrationRequest, +) -> Result { + let mut tables: Vec = Vec::new(); + let mut total_rows: u64 = 0; + let mut type_conversions: u64 = 0; + + for table_plan in &request.table_plans { + let count_query = format!( + "SELECT COUNT(*) AS count FROM {}\"{}\"", + request.source_schema.as_ref().map(|s| format!("\"{}\".", s)).unwrap_or_default(), + table_plan.source_table + ); + + let count_result = source_adapter.execute_query(&count_query).await.map_err(|e| e.to_string())?; + let row_count = count_result.rows.first() + .and_then(|row| row.get("count")) + .and_then(|v| match v { + QueryValue::Int(n) => Some(*n as u64), + _ => None, + }) + .unwrap_or(0); + + let conversions = table_plan.column_mappings.iter() + .filter(|m| m.conversion != MigrationConversion::Direct) + .count() as u64; + + tables.push(MigrationTablePreview { + source_table: table_plan.source_table.clone(), + target_table: table_plan.target_table.clone(), + row_count, + column_count: table_plan.column_mappings.len() as u64, + mappings: table_plan.column_mappings.clone(), + }); + + total_rows += row_count; + type_conversions += conversions; + } + + Ok(MigrationPreview { + tables, + total_rows, + type_conversions, + }) +} + +pub fn auto_map_columns( + source_columns: &[ColumnInfo], + target_engine: DatabaseType, +) -> Vec { + source_columns.iter() + .map(|col| { + let target_type = map_type_to_engine(&col.data_type, target_engine); + let conversion = if target_type == col.data_type { + MigrationConversion::Direct + } else { + MigrationConversion::Mapped + }; + + MigrationMapping { + source_column: col.name.clone(), + source_type: col.data_type.clone(), + target_column: col.name.clone(), + target_type, + conversion, + } + }) + .collect() +} + +fn map_type_to_engine(source_type: &str, target_engine: DatabaseType) -> String { + let source_lower = source_type.to_lowercase(); + + match target_engine { + DatabaseType::PostgreSQL => { + match source_lower.as_str() { + "int" | "integer" => "INTEGER".to_string(), + "bigint" => "BIGINT".to_string(), + "smallint" => "SMALLINT".to_string(), + "tinyint" => "SMALLINT".to_string(), + "varchar" | "char" | "text" => "VARCHAR(255)".to_string(), + "datetime" | "timestamp" => "TIMESTAMP".to_string(), + "date" => "DATE".to_string(), + "boolean" | "bool" | "tinyint(1)" => "BOOLEAN".to_string(), + "float" | "double" => "DOUBLE PRECISION".to_string(), + "decimal" | "numeric" => "NUMERIC".to_string(), + "json" => "JSONB".to_string(), + "blob" | "binary" => "BYTEA".to_string(), + _ => source_type.to_string(), + } + }, + DatabaseType::MySQL => { + match source_lower.as_str() { + "int" | "integer" | "serial" => "INT".to_string(), + "bigint" => "BIGINT".to_string(), + "smallint" => "SMALLINT".to_string(), + "boolean" | "bool" => "TINYINT(1)".to_string(), + "varchar" | "text" => "VARCHAR(255)".to_string(), + "datetime" | "timestamp" => "DATETIME".to_string(), + "date" => "DATE".to_string(), + "float" | "double precision" => "DOUBLE".to_string(), + "decimal" | "numeric" => "DECIMAL".to_string(), + "jsonb" => "JSON".to_string(), + "bytea" => "BLOB".to_string(), + _ => source_type.to_string(), + } + }, + DatabaseType::SQLite => { + match source_lower.as_str() { + "int" | "integer" | "bigint" | "smallint" | "tinyint" => "INTEGER".to_string(), + "varchar" | "char" | "text" => "TEXT".to_string(), + "datetime" | "timestamp" | "date" => "TEXT".to_string(), + "boolean" | "bool" | "tinyint(1)" => "INTEGER".to_string(), + "float" | "double" | "decimal" | "numeric" => "REAL".to_string(), + "blob" | "binary" | "bytea" => "BLOB".to_string(), + _ => source_type.to_string(), + } + }, + DatabaseType::SqlServer => { + match source_lower.as_str() { + "int" | "integer" | "serial" => "INT".to_string(), + "bigint" => "BIGINT".to_string(), + "smallint" => "SMALLINT".to_string(), + "tinyint" => "TINYINT".to_string(), + "varchar" | "text" => "NVARCHAR(255)".to_string(), + "datetime" | "timestamp" => "DATETIME2".to_string(), + "date" => "DATE".to_string(), + "boolean" | "bool" | "tinyint(1)" => "BIT".to_string(), + "float" | "double precision" => "FLOAT".to_string(), + "decimal" | "numeric" => "DECIMAL".to_string(), + "json" | "jsonb" => "NVARCHAR(MAX)".to_string(), + "blob" | "binary" | "bytea" => "VARBINARY(MAX)".to_string(), + _ => source_type.to_string(), + } + }, + _ => source_type.to_string(), + } +} \ No newline at end of file diff --git a/src-tauri/src/transfer/mod.rs b/src-tauri/src/transfer/mod.rs new file mode 100644 index 00000000..27478b6c --- /dev/null +++ b/src-tauri/src/transfer/mod.rs @@ -0,0 +1,17 @@ +//! Transfer module for data export, import, and migration. + +pub mod defaults; +pub mod ddl; +pub mod export; +pub mod import; +pub mod migration; +pub mod progress; +pub mod types; + +pub use defaults::*; +pub use ddl::*; +pub use export::*; +pub use import::*; +pub use migration::*; +pub use progress::*; +pub use types::*; \ No newline at end of file diff --git a/src-tauri/src/transfer/progress.rs b/src-tauri/src/transfer/progress.rs new file mode 100644 index 00000000..0daa875d --- /dev/null +++ b/src-tauri/src/transfer/progress.rs @@ -0,0 +1,50 @@ +//! Progress reporting for transfer operations via Tauri events. + +use super::types::TransferProgress; +use tauri::Emitter; + +pub const TRANSFER_PROGRESS_EVENT: &str = "transfer-progress"; + +pub fn emit_progress(app_handle: &tauri::AppHandle, progress: &TransferProgress) { + let _ = app_handle.emit(TRANSFER_PROGRESS_EVENT, progress); +} + +pub fn create_progress( + operation: &str, + phase: &str, + processed_rows: u64, + total_rows: Option, + elapsed_ms: u64, +) -> TransferProgress { + let percent = match total_rows { + Some(total) if total > 0 => (processed_rows as f32 / total as f32) * 100.0, + _ => 0.0, + }; + + let estimated_remaining_ms = match total_rows { + Some(total) if processed_rows > 0 && elapsed_ms > 0 && total > processed_rows => { + let remaining = total - processed_rows; + let rate = processed_rows as f64 / elapsed_ms as f64; + if rate > 0.0 { + Some((remaining as f64 / rate) as u64) + } else { + None + } + } + _ => None, + }; + + TransferProgress { + operation: operation.to_string(), + phase: phase.to_string(), + current_table: None, + total_rows, + processed_rows, + skipped_rows: 0, + error_count: 0, + percent, + elapsed_ms, + estimated_remaining_ms, + message: None, + } +} diff --git a/src-tauri/src/transfer/types.rs b/src-tauri/src/transfer/types.rs new file mode 100644 index 00000000..1774076a --- /dev/null +++ b/src-tauri/src/transfer/types.rs @@ -0,0 +1,434 @@ +//! Transfer feature type definitions. +//! +//! This module defines all types for data export, import, and migration operations. + +use serde::{Deserialize, Serialize}; + +// ── Export Types ──────────────────────────────────────────────── + +/// Supported export formats. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum ExportFormat { + Csv, + Jsonl, + Sql, + Excel, +} + +/// CSV export options with sensible defaults. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CsvExportOptions { + #[serde(default = "default_delimiter")] + pub delimiter: char, + #[serde(default = "default_quote_char")] + pub quote_char: char, + #[serde(default = "default_encoding")] + pub encoding: String, + #[serde(default = "default_true")] + pub include_header: bool, + #[serde(default)] + pub quote_all: bool, + #[serde(default = "default_lf")] + pub line_ending: String, +} + +/// JSONL (JSON Lines) export options. +/// Simpler than JSON: one object per line, compact format. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JsonlExportOptions { + #[serde(default = "default_iso8601")] + pub date_format: String, +} + +/// SQL export options. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlExportOptions { + pub target_table: String, + #[serde(default = "default_batch_size")] + pub batch_size: u32, + #[serde(default = "default_true")] + pub include_create_table: bool, + #[serde(default)] + pub include_drop_table: bool, + pub target_engine: Option, +} + +/// Excel export options. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExcelExportOptions { + #[serde(default = "default_sheet_name")] + pub sheet_name: String, + #[serde(default = "default_true")] + pub include_header: bool, + #[serde(default = "default_true")] + pub auto_fit_columns: bool, + #[serde(default = "default_true")] + pub freeze_header: bool, +} + +/// Export source is always a table (Custom Query removed for simplicity). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportSource { + pub table: String, + pub columns: Vec, + pub where_clause: Option, + pub order_by: Option, + pub limit: Option, +} + +/// Export request payload. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub source: ExportSource, + pub format: ExportFormat, + pub csv_options: Option, + pub jsonl_options: Option, + pub sql_options: Option, + pub excel_options: Option, + pub output_path: String, +} + +// ── Import Types ──────────────────────────────────────────────── + +/// Supported import formats. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum ImportFormat { + Csv, + Jsonl, + Sql, + Excel, +} + +/// Column mapping for import. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnMapping { + pub source_column: String, + pub target_column: Option, // None = skip + pub target_type: Option, +} + +/// Conflict resolution strategy for import. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum ConflictStrategy { + #[default] + Skip, + Replace, + Upsert, + Abort, +} + +/// CSV import options. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CsvImportOptions { + #[serde(default = "default_delimiter")] + pub delimiter: char, + #[serde(default = "default_encoding")] + pub encoding: String, + #[serde(default = "default_true")] + pub has_header: bool, +} + +/// Excel import options. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExcelImportOptions { + #[serde(default = "default_sheet_name")] + pub sheet_name: String, + #[serde(default = "default_true")] + pub has_header: bool, +} + +/// Import request payload. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub table: String, + pub file_path: String, + pub format: ImportFormat, + pub column_mappings: Vec, + #[serde(default)] + pub conflict_strategy: ConflictStrategy, + #[serde(default = "default_import_batch_size")] + pub batch_size: u32, + #[serde(default)] + pub create_table: bool, + #[serde(default)] + pub truncate_before: bool, + #[serde(default)] + pub dry_run: bool, + pub csv_options: Option, + pub excel_options: Option, +} + +// ── Progress & Results ────────────────────────────────────────── + +/// Transfer operation progress. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferProgress { + pub operation: String, // "export" | "import" | "ddl" | "sql_file" | "migration" + pub phase: String, // "preparing" | "processing" | "finalizing" + pub current_table: Option, + pub total_rows: Option, + pub processed_rows: u64, + pub skipped_rows: u64, + pub error_count: u64, + pub percent: f32, // 0.0–100.0 + pub elapsed_ms: u64, + pub estimated_remaining_ms: Option, + pub message: Option, +} + +/// Transfer error details. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferError { + pub row_number: Option, + pub statement_number: Option, + pub message: String, + pub sql: Option, +} + +/// Transfer operation result. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferResult { + pub success: bool, + pub total_rows: u64, + pub processed_rows: u64, + pub skipped_rows: u64, + pub error_count: u64, + pub duration_ms: u64, + pub output_path: Option, + pub output_size_bytes: Option, + pub errors: Vec, +} + +// ── Preview & Detection ───────────────────────────────────────── + +/// File format detection result. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileDetectionResult { + pub format: ImportFormat, + pub encoding: String, + pub estimated_rows: Option, + pub file_size_bytes: u64, + pub columns: Vec, + pub csv_delimiter: Option, + pub has_header: Option, +} + +/// Export preview result. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportPreview { + pub columns: Vec, + pub sample_rows: Vec>, + pub total_rows_estimate: Option, + pub formatted_preview: String, // First N rows in target format +} + +// ── Default Value Functions ───────────────────────────────────── + +fn default_delimiter() -> char { + ',' +} +fn default_quote_char() -> char { + '"' +} +fn default_encoding() -> String { + "UTF-8".to_string() +} +fn default_lf() -> String { + "LF".to_string() +} +fn default_true() -> bool { + true +} +fn default_iso8601() -> String { + "ISO8601".to_string() +} +fn default_batch_size() -> u32 { + 1000 +} +fn default_import_batch_size() -> u32 { + 5000 +} +fn default_sheet_name() -> String { + "Sheet1".to_string() +} + +// ── DDL Types ──────────────────────────────────────────────────── + +/// DDL generation options. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DdlOptions { + #[serde(default = "default_true")] + pub include_create_table: bool, + #[serde(default = "default_true")] + pub include_primary_keys: bool, + #[serde(default = "default_true")] + pub include_foreign_keys: bool, + #[serde(default = "default_true")] + pub include_indexes: bool, + #[serde(default = "default_true")] + pub include_constraints: bool, + #[serde(default)] + pub include_comments: bool, + #[serde(default)] + pub include_storage_options: bool, + #[serde(default = "default_true")] + pub include_drop_if_exists: bool, + #[serde(default)] + pub include_if_not_exists: bool, + #[serde(default)] + pub include_data: bool, + pub target_engine: Option, +} + +/// DDL request payload. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DdlRequest { + pub connection_id: String, + pub database: Option, + pub schema: Option, + pub objects: Vec, + pub options: DdlOptions, +} + +/// Object selection for DDL generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DdlObject { + pub name: String, + pub object_type: DdlObjectType, + pub schema: Option, +} + +/// Object type for DDL generation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum DdlObjectType { + Table, + View, + Index, +} + +/// Index information for DDL generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IndexInfo { + pub name: String, + pub columns: Vec, + pub is_unique: bool, + pub is_primary: bool, + pub table: String, + pub schema: Option, +} + +// ── Migration Types ────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationMapping { + pub source_column: String, + pub source_type: String, + pub target_column: String, + pub target_type: String, + pub conversion: MigrationConversion, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum MigrationConversion { + Direct, + Mapped, + Custom, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationTablePlan { + pub source_table: String, + pub target_table: String, + pub column_mappings: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationRequest { + pub source_connection_id: String, + pub source_database: Option, + pub source_schema: Option, + pub target_connection_id: String, + pub target_database: Option, + pub target_schema: Option, + pub table_plans: Vec, + #[serde(default = "default_migration_batch_size")] + pub batch_size: u32, + #[serde(default)] + pub on_error: MigrationErrorStrategy, + #[serde(default = "default_true")] + pub create_tables: bool, + #[serde(default)] + pub drop_tables: bool, + #[serde(default)] + pub migrate_indexes: bool, + #[serde(default)] + pub migrate_foreign_keys: bool, + #[serde(default = "default_true")] + pub migrate_constraints: bool, + #[serde(default)] + pub disable_fk_checks: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum MigrationErrorStrategy { + #[default] + SkipRow, + SkipTable, + Abort, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationPreview { + pub tables: Vec, + pub total_rows: u64, + pub type_conversions: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MigrationTablePreview { + pub source_table: String, + pub target_table: String, + pub row_count: u64, + pub column_count: u64, + pub mappings: Vec, +} + +fn default_migration_batch_size() -> u32 { + 5000 +} diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 667c2e5b..59687a06 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -6,7 +6,7 @@ "build": { "beforeDevCommand": "npm run dev", "beforeBuildCommand": "npm run build", - "devUrl": "http://localhost:1420", + "devUrl": "http://localhost:1521", "frontendDist": "../dist" }, "app": { diff --git a/src/components/database-browser/DataTableView.vue b/src/components/database-browser/DataTableView.vue index 80de1ab3..c4cf156a 100644 --- a/src/components/database-browser/DataTableView.vue +++ b/src/components/database-browser/DataTableView.vue @@ -80,6 +80,7 @@ const loading = ref(false) const error = ref(null) const executionTimeMs = ref(null) const columnInfoList = ref([]) +const invalidated = ref(false) // --- Connection state --- const connectionStore = useConnectionStore() @@ -170,6 +171,9 @@ const formattedTime = computed(() => { }) async function fetchData() { + if (invalidated.value) { + return + } loading.value = true error.value = null @@ -200,6 +204,9 @@ async function fetchData() { } async function fetchCount() { + if (invalidated.value) { + return + } try { const count = await invoke('get_table_count', { connectionId: props.connectionId, @@ -255,6 +262,10 @@ async function handleRetry() { } async function fetchColumnInfo() { + if (invalidated.value) { + columnInfoList.value = [] + return + } if (!props.database || !props.tableName) { columnInfoList.value = [] return @@ -577,15 +588,24 @@ onMounted(async () => { watch( () => [props.connectionId, props.tableName, props.schema] as const, - async () => { + async ([newConnId, _newTable, _newSchema], [oldConnId]) => { currentPage.value = 1 appliedFilter.value = '' filterInput.value = '' hiddenColumns.value = new Set() connectionError.value = null + if (newConnId !== oldConnId) { + data.value = null + totalCount.value = 0 + columnInfoList.value = [] + invalidated.value = true + return + } + + invalidated.value = false const connected = await ensureConnection() - if (connected) { + if (connected && !invalidated.value) { fetchColumnInfo() refresh() } diff --git a/src/components/database-browser/DatabaseBrowser.vue b/src/components/database-browser/DatabaseBrowser.vue index 85174ff7..a72ad29c 100644 --- a/src/components/database-browser/DatabaseBrowser.vue +++ b/src/components/database-browser/DatabaseBrowser.vue @@ -16,7 +16,7 @@ import { SelectValue, } from '@/components/ui/select' import { deleteQueryFile, listSavedQueryFiles } from '@/datasources' -import { ConnectionStatus, useConnectionStore, useDatabaseStore } from '@/store' +import { ConnectionStatus, DatabaseType, useConnectionStore, useDatabaseStore } from '@/store' export type TreeNodeMetadata = TableInfo & { database: string @@ -39,6 +39,7 @@ export type TreeNode = { const props = defineProps<{ connectionId?: string selectedDatabase?: string + selectedSchema?: string }>() const emit = defineEmits<{ @@ -48,6 +49,7 @@ const emit = defineEmits<{ (e: 'viewStructure', table: TableInfo, database: string, schema?: string): void (e: 'exportData', table: TableInfo, database: string, schema?: string): void (e: 'update:selectedDatabase', database: string): void + (e: 'update:selectedSchema', schema: string): void (e: 'openSavedQuery', filePath: string): void (e: 'createNewQuery'): void }>() @@ -82,6 +84,18 @@ const isActiveConnectionConnected = computed(() => : false, ) +const supportsSchemas = computed(() => { + const type = activeConnection.value?.type + return type === DatabaseType.POSTGRESQL || type === DatabaseType.SQLSERVER +}) + +const availableSchemas = computed(() => { + if (!props.selectedDatabase || !connectionId.value || !supportsSchemas.value) { + return [] + } + return databaseStore.metadata[connectionId.value]?.schemas[props.selectedDatabase] ?? [] +}) + function createTableNode(database: string, schema: string | undefined, table: TableInfo, parentId: string): TreeNode { return { id: `table-${database}-${schema || ''}-${table.name}`, @@ -158,11 +172,13 @@ const tablesAndViews = computed(() => { const schemas = metadata.schemas[currentDb] || [] const allItems: TreeNode[] = schemas.length > 0 - ? schemas.flatMap((schema) => { + ? (() => { + const schema = props.selectedSchema || schemas[0] const tablesKey = `${currentDb}.${schema}` - const tables = metadata.tables[tablesKey] || [] - return tables.map(table => createTableNode(currentDb, schema, table, `schema-${currentDb}-${schema}`)) - }) + return (metadata.tables[tablesKey] || []).map(table => + createTableNode(currentDb, schema, table, `schema-${currentDb}-${schema}`), + ) + })() : (metadata.tables[currentDb] || []).map(table => createTableNode(currentDb, undefined, table, `db-${currentDb}`), ) @@ -453,6 +469,18 @@ watch(() => props.selectedDatabase, async (newDb, oldDb) => { await loadDatabaseData(connectionId.value, newDb) }) +watch(() => props.selectedSchema, async (newSchema, oldSchema) => { + if (!newSchema || !props.selectedDatabase || !connectionId.value || newSchema === oldSchema) { + return + } + + const tablesKey = `${props.selectedDatabase}.${newSchema}` + const meta = databaseStore.metadata[connectionId.value] + if (!meta?.tables[tablesKey]) { + await databaseStore.fetchTables(connectionId.value, props.selectedDatabase, newSchema) + } +}) + watch(showSavedQueries, async (isExpanded) => { if (isExpanded && savedQueryFiles.value.length === 0) { await fetchSavedQueryFiles() @@ -565,6 +593,24 @@ defineExpose({ fetchSavedQueryFiles }) + +
+ +
+
diff --git a/src/components/database-browser/DbTypeIcon.vue b/src/components/database-browser/DbTypeIcon.vue new file mode 100644 index 00000000..8bb9d52e --- /dev/null +++ b/src/components/database-browser/DbTypeIcon.vue @@ -0,0 +1,23 @@ + + + diff --git a/src/components/database-browser/QueryTabs.vue b/src/components/database-browser/QueryTabs.vue index d6576e97..f0828a41 100644 --- a/src/components/database-browser/QueryTabs.vue +++ b/src/components/database-browser/QueryTabs.vue @@ -1,6 +1,6 @@ + + diff --git a/src/components/transfer/export/ExportFormatStep.vue b/src/components/transfer/export/ExportFormatStep.vue new file mode 100644 index 00000000..c33ba3a7 --- /dev/null +++ b/src/components/transfer/export/ExportFormatStep.vue @@ -0,0 +1,111 @@ + + + diff --git a/src/components/transfer/export/ExportPreviewStep.vue b/src/components/transfer/export/ExportPreviewStep.vue new file mode 100644 index 00000000..acb0854b --- /dev/null +++ b/src/components/transfer/export/ExportPreviewStep.vue @@ -0,0 +1,161 @@ + + + diff --git a/src/components/transfer/export/ExportSourceStep.vue b/src/components/transfer/export/ExportSourceStep.vue new file mode 100644 index 00000000..1924fc5b --- /dev/null +++ b/src/components/transfer/export/ExportSourceStep.vue @@ -0,0 +1,123 @@ + + + diff --git a/src/components/transfer/export/ExportWizard.vue b/src/components/transfer/export/ExportWizard.vue new file mode 100644 index 00000000..b69aad9f --- /dev/null +++ b/src/components/transfer/export/ExportWizard.vue @@ -0,0 +1,367 @@ + + + diff --git a/src/components/transfer/import/ImportExecuteStep.vue b/src/components/transfer/import/ImportExecuteStep.vue new file mode 100644 index 00000000..d1979bfd --- /dev/null +++ b/src/components/transfer/import/ImportExecuteStep.vue @@ -0,0 +1,115 @@ + + + diff --git a/src/components/transfer/import/ImportFileStep.vue b/src/components/transfer/import/ImportFileStep.vue new file mode 100644 index 00000000..6a801a8a --- /dev/null +++ b/src/components/transfer/import/ImportFileStep.vue @@ -0,0 +1,145 @@ + + + diff --git a/src/components/transfer/import/ImportMappingStep.vue b/src/components/transfer/import/ImportMappingStep.vue new file mode 100644 index 00000000..61c325ea --- /dev/null +++ b/src/components/transfer/import/ImportMappingStep.vue @@ -0,0 +1,227 @@ + + + diff --git a/src/components/transfer/import/ImportOptionsStep.vue b/src/components/transfer/import/ImportOptionsStep.vue new file mode 100644 index 00000000..e6032819 --- /dev/null +++ b/src/components/transfer/import/ImportOptionsStep.vue @@ -0,0 +1,166 @@ + + + diff --git a/src/components/transfer/import/ImportWizard.vue b/src/components/transfer/import/ImportWizard.vue new file mode 100644 index 00000000..1fa4fd21 --- /dev/null +++ b/src/components/transfer/import/ImportWizard.vue @@ -0,0 +1,671 @@ + + + diff --git a/src/components/transfer/index.ts b/src/components/transfer/index.ts new file mode 100644 index 00000000..e26bfa68 --- /dev/null +++ b/src/components/transfer/index.ts @@ -0,0 +1,28 @@ +export { default as ExportExecuteStep } from './export/ExportExecuteStep.vue' +export { default as ExportFormatStep } from './export/ExportFormatStep.vue' +export { default as ExportPreviewStep } from './export/ExportPreviewStep.vue' +export { default as ExportSourceStep } from './export/ExportSourceStep.vue' +export { default as ExportWizard } from './export/ExportWizard.vue' + +export { default as ImportExecuteStep } from './import/ImportExecuteStep.vue' +export { default as ImportFileStep } from './import/ImportFileStep.vue' +export { default as ImportMappingStep } from './import/ImportMappingStep.vue' +export { default as ImportOptionsStep } from './import/ImportOptionsStep.vue' +export { default as ImportWizard } from './import/ImportWizard.vue' + +export { default as MigrationWizard } from './migration/MigrationWizard.vue' +export { default as ColumnSelector } from './shared/ColumnSelector.vue' +export { default as ConnectionSelector } from './shared/ConnectionSelector.vue' + +export { default as FileDropZone } from './shared/FileDropZone.vue' +export { default as ProgressPanel } from './shared/ProgressPanel.vue' +export { default as ResultPanel } from './shared/ResultPanel.vue' + +export { default as TableSelector } from './shared/TableSelector.vue' +export { default as WizardStepper } from './shared/WizardStepper.vue' +export { default as GenerateDdl } from './structure/GenerateDdl.vue' +export { default as RunSqlFile } from './structure/RunSqlFile.vue' +export { default as StructureWizard } from './structure/StructureWizard.vue' +export { default as TaskCard } from './tasks/TaskCard.vue' +export { default as TaskManagerButton } from './tasks/TaskManagerButton.vue' +export { default as TaskManagerPanel } from './tasks/TaskManagerPanel.vue' diff --git a/src/components/transfer/migration/MigrationWizard.vue b/src/components/transfer/migration/MigrationWizard.vue new file mode 100644 index 00000000..5718295d --- /dev/null +++ b/src/components/transfer/migration/MigrationWizard.vue @@ -0,0 +1,474 @@ + + + diff --git a/src/components/transfer/shared/ColumnSelector.vue b/src/components/transfer/shared/ColumnSelector.vue new file mode 100644 index 00000000..09023635 --- /dev/null +++ b/src/components/transfer/shared/ColumnSelector.vue @@ -0,0 +1,161 @@ + + + + diff --git a/src/components/transfer/shared/ConnectionSelector.vue b/src/components/transfer/shared/ConnectionSelector.vue new file mode 100644 index 00000000..5fe384a6 --- /dev/null +++ b/src/components/transfer/shared/ConnectionSelector.vue @@ -0,0 +1,311 @@ + + + + diff --git a/src/components/transfer/shared/FileDropZone.vue b/src/components/transfer/shared/FileDropZone.vue new file mode 100644 index 00000000..11e5f660 --- /dev/null +++ b/src/components/transfer/shared/FileDropZone.vue @@ -0,0 +1,107 @@ + + + + diff --git a/src/components/transfer/shared/MultiTableSelector.vue b/src/components/transfer/shared/MultiTableSelector.vue new file mode 100644 index 00000000..cef61b10 --- /dev/null +++ b/src/components/transfer/shared/MultiTableSelector.vue @@ -0,0 +1,229 @@ + + + + diff --git a/src/components/transfer/shared/ProgressPanel.vue b/src/components/transfer/shared/ProgressPanel.vue new file mode 100644 index 00000000..5a570b4c --- /dev/null +++ b/src/components/transfer/shared/ProgressPanel.vue @@ -0,0 +1,115 @@ + + + + diff --git a/src/components/transfer/shared/ResultPanel.vue b/src/components/transfer/shared/ResultPanel.vue new file mode 100644 index 00000000..a5a134e2 --- /dev/null +++ b/src/components/transfer/shared/ResultPanel.vue @@ -0,0 +1,143 @@ + + + + diff --git a/src/components/transfer/shared/TabbedColumnSelector.vue b/src/components/transfer/shared/TabbedColumnSelector.vue new file mode 100644 index 00000000..07a0ab6c --- /dev/null +++ b/src/components/transfer/shared/TabbedColumnSelector.vue @@ -0,0 +1,407 @@ + + + + diff --git a/src/components/transfer/shared/TableSelector.vue b/src/components/transfer/shared/TableSelector.vue new file mode 100644 index 00000000..1e4263da --- /dev/null +++ b/src/components/transfer/shared/TableSelector.vue @@ -0,0 +1,125 @@ + + + + diff --git a/src/components/transfer/shared/TransferStepCard.vue b/src/components/transfer/shared/TransferStepCard.vue new file mode 100644 index 00000000..b82c4f96 --- /dev/null +++ b/src/components/transfer/shared/TransferStepCard.vue @@ -0,0 +1,53 @@ + + + + diff --git a/src/components/transfer/shared/WizardStepper.vue b/src/components/transfer/shared/WizardStepper.vue new file mode 100644 index 00000000..16b21196 --- /dev/null +++ b/src/components/transfer/shared/WizardStepper.vue @@ -0,0 +1,55 @@ + + + + diff --git a/src/components/transfer/structure/GenerateDdl.vue b/src/components/transfer/structure/GenerateDdl.vue new file mode 100644 index 00000000..6545dbc5 --- /dev/null +++ b/src/components/transfer/structure/GenerateDdl.vue @@ -0,0 +1,291 @@ + + + diff --git a/src/components/transfer/structure/RunSqlFile.vue b/src/components/transfer/structure/RunSqlFile.vue new file mode 100644 index 00000000..5e56db56 --- /dev/null +++ b/src/components/transfer/structure/RunSqlFile.vue @@ -0,0 +1,222 @@ + + + diff --git a/src/components/transfer/structure/StructureWizard.vue b/src/components/transfer/structure/StructureWizard.vue new file mode 100644 index 00000000..de1e3393 --- /dev/null +++ b/src/components/transfer/structure/StructureWizard.vue @@ -0,0 +1,45 @@ + + + diff --git a/src/components/transfer/tasks/TaskCard.vue b/src/components/transfer/tasks/TaskCard.vue new file mode 100644 index 00000000..8d2f8a64 --- /dev/null +++ b/src/components/transfer/tasks/TaskCard.vue @@ -0,0 +1,127 @@ + + + diff --git a/src/components/transfer/tasks/TaskManagerButton.vue b/src/components/transfer/tasks/TaskManagerButton.vue new file mode 100644 index 00000000..3393ef21 --- /dev/null +++ b/src/components/transfer/tasks/TaskManagerButton.vue @@ -0,0 +1,22 @@ + + + diff --git a/src/components/transfer/tasks/TaskManagerPanel.vue b/src/components/transfer/tasks/TaskManagerPanel.vue new file mode 100644 index 00000000..835abd52 --- /dev/null +++ b/src/components/transfer/tasks/TaskManagerPanel.vue @@ -0,0 +1,86 @@ + + + diff --git a/src/components/ui/checkbox/Checkbox.vue b/src/components/ui/checkbox/Checkbox.vue index 2e84b88e..81040e18 100644 --- a/src/components/ui/checkbox/Checkbox.vue +++ b/src/components/ui/checkbox/Checkbox.vue @@ -1,6 +1,6 @@ + + + + diff --git a/src/router/index.ts b/src/router/index.ts index 0318d98d..ecf6b6cc 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -11,9 +11,9 @@ const routes = [ component: () => import('@/pages/ConnectionsPage.vue'), }, { - path: '/import-export', - name: 'import-export', - component: () => import('@/pages/ImportExportPage.vue'), + path: '/transfer', + name: 'transfer', + component: () => import('@/pages/TransferPage.vue'), }, { path: '/data-studio', diff --git a/src/store/tabStore.ts b/src/store/tabStore.ts index bb6c8561..e8444b68 100644 --- a/src/store/tabStore.ts +++ b/src/store/tabStore.ts @@ -24,8 +24,10 @@ export type QueryTab = { id: string name: string content: string - connectionId: string + /** Connection this tab originated from. Used to detect stale tabs across connection switches. */ + connectionId?: string database?: string + schema?: string isExecuting: boolean hasUnsavedChanges: boolean filePath?: string @@ -33,6 +35,8 @@ export type QueryTab = { error?: ApiError | string executionTime?: number tableView?: TableViewMeta + /** If set, this tab is orphaned from the specified connection and cannot execute queries */ + orphanFromConnectionId?: string } type TabStoreState = { @@ -61,17 +65,24 @@ export const useTabStore = defineStore('tabs', { unsavedTabs: (state): QueryTab[] => state.tabs.filter(t => t.hasUnsavedChanges), + orphanTabs: (state): QueryTab[] => + state.tabs.filter(t => t.orphanFromConnectionId), + + isOrphanTab: state => (id: string): boolean => + state.tabs.find(t => t.id === id)?.orphanFromConnectionId !== undefined, + tabCount: (state): number => state.tabs.length, }, actions: { - createTab(connectionId: string, database?: string): QueryTab { + createTab(database?: string, schema?: string, connectionId?: string): QueryTab { const tab: QueryTab = { id: generateId(), name: `Query ${this.tabs.length + 1}`, content: '', connectionId, database, + schema, isExecuting: false, hasUnsavedChanges: false, } @@ -80,14 +91,14 @@ export const useTabStore = defineStore('tabs', { return tab }, - /** Open a table-view tab for the given table, or switch to one that is already open. */ - openTableViewTab(connectionId: string, database: string, tableName: string, schema?: string): QueryTab { + openTableViewTab(database: string, tableName: string, schema?: string, connectionId?: string): QueryTab { const existing = this.tabs.find( t => t.tableView && t.tableView.tableName === tableName && t.tableView.database === database && (t.tableView.schema ?? null) === (schema ?? null) - && t.connectionId === connectionId, + && t.connectionId === connectionId + && !t.orphanFromConnectionId, ) if (existing) { this.activeTabId = existing.id @@ -100,6 +111,7 @@ export const useTabStore = defineStore('tabs', { content: '', connectionId, database, + schema, isExecuting: false, hasUnsavedChanges: false, tableView: { tableName, database, schema }, @@ -126,10 +138,83 @@ export const useTabStore = defineStore('tabs', { this.activeTabId = null }, - closeTabsForConnection(connectionId: string) { - this.tabs = this.tabs.filter(t => t.connectionId !== connectionId) + closeNonOrphanTabs() { + this.tabs = this.tabs.filter(t => t.orphanFromConnectionId) + if (this.activeTabId && !this.tabs.find(t => t.id === this.activeTabId)) { + const orphanTab = this.tabs.find(t => t.orphanFromConnectionId) + this.activeTabId = orphanTab?.id ?? null + } + }, + + reconcileTabsForConnection(currentConnectionId: string) { + const isStale = (t: QueryTab) => + !t.orphanFromConnectionId + && t.connectionId !== currentConnectionId + + const staleTableViewIds = this.tabs + .filter(t => isStale(t) && t.tableView) + .map(t => t.id) + + const staleSavedQueryIds = this.tabs + .filter(t => isStale(t) && !t.tableView && !t.hasUnsavedChanges) + .map(t => t.id) + + const staleUnsavedQueryTabs = this.tabs.filter( + t => isStale(t) && !t.tableView && t.hasUnsavedChanges, + ) + + const toCloseIds = new Set([...staleTableViewIds, ...staleSavedQueryIds]) + + const orphanedIds = new Set(staleUnsavedQueryTabs.map(t => t.id)) + + this.tabs = this.tabs + .filter(t => !toCloseIds.has(t.id)) + .map(t => + orphanedIds.has(t.id) + ? { ...t, orphanFromConnectionId: t.connectionId ?? 'unknown' } + : t, + ) + + if (this.activeTabId && !this.tabs.find(t => t.id === this.activeTabId)) { + const nonOrphanTab = this.tabs.find(t => !t.orphanFromConnectionId) + const orphanTab = this.tabs.find(t => t.orphanFromConnectionId) + this.activeTabId = nonOrphanTab?.id ?? orphanTab?.id ?? null + } + }, + + transitionTabsForConnection(oldConnectionId: string) { + const tableViewTabIds = this.tabs + .filter(t => t.tableView && !t.orphanFromConnectionId) + .map(t => t.id) + + const savedQueryTabIds = this.tabs + .filter(t => !t.tableView && !t.hasUnsavedChanges && !t.orphanFromConnectionId) + .map(t => t.id) + + const unsavedQueryTabs = this.tabs.filter( + t => !t.tableView && t.hasUnsavedChanges && !t.orphanFromConnectionId, + ) + + const toCloseIds = new Set([...tableViewTabIds, ...savedQueryTabIds]) + + this.tabs = this.tabs.filter(t => !toCloseIds.has(t.id)) + + unsavedQueryTabs.forEach((tab) => { + const index = this.tabs.findIndex(t => t.id === tab.id) + if (index !== -1) { + const orphanedTab = { ...tab, orphanFromConnectionId: oldConnectionId } + this.tabs = [ + ...this.tabs.slice(0, index), + orphanedTab, + ...this.tabs.slice(index + 1), + ] + } + }) + if (this.activeTabId && !this.tabs.find(t => t.id === this.activeTabId)) { - this.activeTabId = this.tabs[0]?.id ?? null + const nonOrphanTab = this.tabs.find(t => !t.orphanFromConnectionId) + const orphanTab = this.tabs.find(t => t.orphanFromConnectionId) + this.activeTabId = nonOrphanTab?.id ?? orphanTab?.id ?? null } }, @@ -148,15 +233,14 @@ export const useTabStore = defineStore('tabs', { } }, - async executeQuery(tabId: string, sqlToExecute?: string) { + async executeQuery(tabId: string, activeConnectionId: string, sqlToExecute?: string) { const tab = this.tabs.find(t => t.id === tabId) - if (!tab) { + if (!tab || tab.orphanFromConnectionId) { return } const sql = sqlToExecute !== undefined ? sqlToExecute : tab.content - // Validate SQL is a non-empty string if (typeof sql !== 'string' || sql.trim() === '') { return } @@ -165,7 +249,7 @@ export const useTabStore = defineStore('tabs', { tab.error = undefined const connectionStore = useConnectionStore() - const connection = connectionStore.getConnectionById(tab.connectionId) + const connection = connectionStore.getConnectionById(activeConnectionId) const historyStore = useHistoryStore() const queryStartTime = Date.now() @@ -176,7 +260,7 @@ export const useTabStore = defineStore('tabs', { await withMinLoadingTime(async () => { response = await invoke>('execute_query', { - connectionId: tab.connectionId, + connectionId: activeConnectionId, sql, database: tab.database ?? null, }) @@ -188,8 +272,8 @@ export const useTabStore = defineStore('tabs', { tab.executionTime = actualExecutionTime historyStore.addEntry({ sql, - connectionId: tab.connectionId, - connectionName: connection?.name ?? tab.connectionId, + connectionId: activeConnectionId, + connectionName: connection?.name ?? activeConnectionId, database: tab.database, timestamp: Date.now(), executionTime: actualExecutionTime, @@ -213,8 +297,8 @@ export const useTabStore = defineStore('tabs', { tab.error = err historyStore.addEntry({ sql, - connectionId: tab.connectionId, - connectionName: connection?.name ?? tab.connectionId, + connectionId: activeConnectionId, + connectionName: connection?.name ?? activeConnectionId, database: tab.database, timestamp: Date.now(), executionTime: actualExecutionTime, @@ -228,8 +312,8 @@ export const useTabStore = defineStore('tabs', { tab.error = String(error) historyStore.addEntry({ sql, - connectionId: tab.connectionId, - connectionName: connection?.name ?? tab.connectionId, + connectionId: activeConnectionId, + connectionName: connection?.name ?? activeConnectionId, database: tab.database, timestamp: Date.now(), executionTime: actualExecutionTime, diff --git a/src/store/transferStore.ts b/src/store/transferStore.ts new file mode 100644 index 00000000..f71aa45d --- /dev/null +++ b/src/store/transferStore.ts @@ -0,0 +1,231 @@ +import type { + BackgroundTask, + ExportRequest, + ExportTaskConfig, + ImportRequest, + ImportTaskConfig, + TaskConfig, + TaskKind, + TaskRuntime, + TaskStatus, + TransferProgress, + TransferResult, +} from '@/types/transfer' +import { defineStore } from 'pinia' + +import { computed, ref } from 'vue' + +export const useTransferStore = defineStore('transfer', () => { + const activeTab = ref<'export' | 'import'>('export') + + const isRunning = ref(false) + const progress = ref(null) + const lastResult = ref(null) + + const exportStep = ref(0) + const exportRequest = ref>({}) + + const importStep = ref(0) + const importRequest = ref>({}) + + const runningTasks = ref([]) + const activeExportTaskId = ref(null) + const activeImportTaskId = ref(null) + + const progressPercent = computed(() => progress.value?.percent ?? 0) + + const taskCount = computed(() => runningTasks.value.length) + + const hasRunningTasks = computed(() => + runningTasks.value.some(t => t.status === 'running'), + ) + + const activeTaskId = computed(() => { + switch (activeTab.value) { + case 'export': return activeExportTaskId.value + case 'import': return activeImportTaskId.value + default: return null + } + }) + + const setActiveTab = (tab: typeof activeTab.value) => { + activeTab.value = tab + } + + const updateProgress = (p: TransferProgress) => { + progress.value = p + } + + const startOperation = () => { + isRunning.value = true + progress.value = null + lastResult.value = null + } + + const completeOperation = (result: TransferResult) => { + isRunning.value = false + lastResult.value = result + progress.value = null + } + + const resetExport = () => { + exportStep.value = 0 + exportRequest.value = {} + lastResult.value = null + } + + const resetImport = () => { + importStep.value = 0 + importRequest.value = {} + lastResult.value = null + } + + const addRunningTask = (task: BackgroundTask) => { + runningTasks.value = [...runningTasks.value, task] + } + + const updateTaskRuntime = (taskId: string, runtime: Partial) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { + ...t, + runtime: { ...t.runtime, ...runtime }, + progress: { + complete: runtime.complete ?? t.progress.complete, + total: runtime.total ?? t.progress.total, + }, + } + : t, + ) + } + + const updateTaskStatus = (taskId: string, status: TaskStatus, error?: string) => { + runningTasks.value = runningTasks.value.map(t => + t.id === taskId + ? { + ...t, + status, + endTime: status === 'completed' || status === 'failed' ? new Date() : undefined, + error, + } + : t, + ) + } + + const removeTask = (taskId: string) => { + runningTasks.value = runningTasks.value.filter(t => t.id !== taskId) + } + + const clearCompletedTasks = () => { + runningTasks.value = runningTasks.value.filter(t => + t.status === 'running' || t.status === 'pending', + ) + } + + const syncProgressToTask = (taskId: string, p: TransferProgress) => { + updateTaskRuntime(taskId, { + complete: p.processedRows, + total: p.totalRows ?? 0, + skipped: p.skippedRows, + errorCount: p.errorCount, + }) + } + + const detachActiveTask = (kind: TaskKind) => { + switch (kind) { + case 'export': + activeExportTaskId.value = null + break + case 'import': + activeImportTaskId.value = null + break + } + } + + const openTask = (taskId: string) => { + const task = runningTasks.value.find(t => t.id === taskId) + if (!task) + return + + const tabMap: Record = { + export: 'export', + import: 'import', + sqlFile: 'export', + migration: 'import', + } + activeTab.value = tabMap[task.kind] + + switch (task.kind) { + case 'export': + activeExportTaskId.value = taskId + break + case 'import': + activeImportTaskId.value = taskId + break + } + } + + const generateTaskLabel = (kind: TaskKind, config: TaskConfig): string => { + switch (kind) { + case 'export': { + const cfg = config as ExportTaskConfig + return `Export ${cfg.table} → ${cfg.format.toUpperCase()}` + } + case 'import': { + const cfg = config as ImportTaskConfig + const fileName = cfg.filePath.split('/').pop() || cfg.filePath + return `Import ${fileName}` + } + default: + return 'Transfer task' + } + } + + const createTask = (kind: TaskKind, config: TaskConfig, total: number): BackgroundTask => { + const id = crypto.randomUUID() + const label = generateTaskLabel(kind, config) + return { + id, + kind, + status: 'running', + progress: { complete: 0, total }, + config, + runtime: { complete: 0, total, skipped: 0, errorCount: 0 }, + label, + startTime: new Date(), + } + } + + return { + activeTab, + setActiveTab, + isRunning, + progress, + lastResult, + progressPercent, + updateProgress, + startOperation, + completeOperation, + exportStep, + exportRequest, + importStep, + importRequest, + resetExport, + resetImport, + runningTasks, + activeExportTaskId, + activeImportTaskId, + taskCount, + hasRunningTasks, + activeTaskId, + addRunningTask, + updateTaskRuntime, + updateTaskStatus, + removeTask, + clearCompletedTasks, + syncProgressToTask, + detachActiveTask, + openTask, + createTask, + } +}) diff --git a/src/types/connection.ts b/src/types/connection.ts index ef62d9ee..22f3ffa4 100644 --- a/src/types/connection.ts +++ b/src/types/connection.ts @@ -149,3 +149,20 @@ function isValidCertPath(path: string): boolean { export function hasSslValidationErrors(sslConfig: SslConfig, dbType: string): boolean { return validateSslConfig(sslConfig, dbType).length > 0 } + +/** + * Column metadata from database table + */ +export type ColumnInfo = { + name: string + data_type: string + nullable: boolean + default_value?: string + is_primary_key: boolean + is_auto_increment: boolean + max_length?: number + precision?: number + scale?: number + description?: string + metadata?: Record +} diff --git a/src/types/transfer.ts b/src/types/transfer.ts new file mode 100644 index 00000000..0a7e7643 --- /dev/null +++ b/src/types/transfer.ts @@ -0,0 +1,296 @@ +export type ExportFormat = 'csv' | 'jsonl' | 'sql' | 'excel' + +export type ColumnInfo = { + name: string + data_type?: string + nullable?: boolean + default_value?: string + is_primary_key?: boolean +} + +export type TableColumns = { + tableName: string + columns: ColumnInfo[] + selectedColumns: string[] +} + +export type ExportSource = { + table: string + columns: string[] + whereClause?: string + orderBy?: string + limit?: number +} + +export type CsvExportOptions = { + delimiter?: string + quoteChar?: string + encoding?: string + includeHeader?: boolean + quoteAll?: boolean + lineEnding?: 'LF' | 'CRLF' +} + +export type JsonlExportOptions = { + dateFormat?: string +} + +export type SqlExportOptions = { + targetTable: string + batchSize?: number + includeCreateTable?: boolean + includeDropTable?: boolean + targetEngine?: string +} + +export type ExcelExportOptions = { + sheetName?: string + includeHeader?: boolean + autoFitColumns?: boolean + freezeHeader?: boolean +} + +export type ExportRequest = { + connectionId: string + database?: string + schema?: string + source: ExportSource + format: ExportFormat + csvOptions?: CsvExportOptions + jsonlOptions?: JsonlExportOptions + sqlOptions?: SqlExportOptions + excelOptions?: ExcelExportOptions + outputPath: string +} + +export type ImportFormat = 'csv' | 'jsonl' | 'sql' | 'excel' + +export type ColumnMapping = { + sourceColumn: string + targetColumn?: string + targetType?: string +} + +export type ConflictStrategy = 'skip' | 'replace' | 'upsert' | 'abort' + +export type CsvImportOptions = { + delimiter?: string + encoding?: string + hasHeader?: boolean +} + +export type ImportRequest = { + connectionId: string + database?: string + schema?: string + table: string + filePath: string + format: ImportFormat + columnMappings: ColumnMapping[] + conflictStrategy?: ConflictStrategy + batchSize?: number + createTable?: boolean + truncateBefore?: boolean + dryRun?: boolean + csvOptions?: CsvImportOptions + excelOptions?: ExcelImportOptions +} + +export type TransferProgress = { + operation: string + phase: string + currentTable?: string + totalRows?: number + processedRows: number + skippedRows: number + errorCount: number + percent: number + elapsedMs: number + estimatedRemainingMs?: number + message?: string +} + +export type TransferError = { + rowNumber?: number + statementNumber?: number + message: string + sql?: string +} + +export type TransferResult = { + success: boolean + totalRows: number + processedRows: number + skippedRows: number + errorCount: number + durationMs: number + outputPath?: string + outputSizeBytes?: number + errors: TransferError[] +} + +export type FileDetectionResult = { + format: ImportFormat + encoding: string + estimatedRows?: number + fileSizeBytes: number + columns: string[] + csvDelimiter?: string + hasHeader?: boolean +} + +export type ExportPreview = { + columns: string[] + sampleRows: string[][] + totalRowsEstimate?: number + formattedPreview: string +} + +export type TaskKind = 'export' | 'import' | 'sqlFile' | 'migration' + +export type TaskStatus = 'pending' | 'running' | 'completed' | 'failed' + +export type TaskRuntime = { + complete: number + total: number + skipped: number + errorCount: number +} + +export type ExportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + columns: string[] + whereClause?: string + orderBy?: string + limit?: number + format: ExportFormat + outputPath: string +} + +export type ImportTaskConfig = { + connectionId: string + database?: string + schema?: string + table: string + filePath: string + format: ImportFormat + conflictStrategy?: ConflictStrategy +} + +export type SqlFileTaskConfig = { + connectionId: string + database?: string + filePath: string + onError: 'rollback' | 'skipAndContinue' | 'stop' +} + +export type MigrationTaskConfig = { + sourceConnectionId: string + sourceDatabase?: string + targetConnectionId: string + targetDatabase?: string + tables: string[] +} + +export type TaskConfig = ExportTaskConfig | ImportTaskConfig | SqlFileTaskConfig | MigrationTaskConfig + +export type BackgroundTask = { + id: string + kind: TaskKind + status: TaskStatus + progress: { complete: number, total: number } + config: TaskConfig + runtime: TaskRuntime + label: string + startTime: Date + endTime?: Date + error?: string +} + +export type DdlObjectType = 'table' | 'view' | 'index' + +export type DdlObject = { + name: string + objectType: DdlObjectType + schema?: string +} + +export type DdlOptions = { + includeCreateTable?: boolean + includePrimaryKeys?: boolean + includeForeignKeys?: boolean + includeIndexes?: boolean + includeConstraints?: boolean + includeComments?: boolean + includeStorageOptions?: boolean + includeDropIfExists?: boolean + includeIfNotExists?: boolean + includeData?: boolean + targetEngine?: string +} + +export type DdlRequest = { + connectionId: string + database?: string + schema?: string + objects: DdlObject[] + options: DdlOptions +} + +export type ExcelImportOptions = { + sheetName?: string + hasHeader?: boolean +} + +export type MigrationConversion = 'direct' | 'mapped' | 'custom' + +export type MigrationMapping = { + sourceColumn: string + sourceType: string + targetColumn: string + targetType: string + conversion: MigrationConversion +} + +export type MigrationTablePlan = { + sourceTable: string + targetTable: string + columnMappings: MigrationMapping[] +} + +export type MigrationErrorStrategy = 'skipRow' | 'skipTable' | 'abort' + +export type MigrationRequest = { + sourceConnectionId: string + sourceDatabase?: string + sourceSchema?: string + targetConnectionId: string + targetDatabase?: string + targetSchema?: string + tablePlans: MigrationTablePlan[] + batchSize?: number + onError?: MigrationErrorStrategy + createTables?: boolean + dropTables?: boolean + migrateIndexes?: boolean + migrateForeignKeys?: boolean + migrateConstraints?: boolean + disableFkChecks?: boolean +} + +export type MigrationTablePreview = { + sourceTable: string + targetTable: string + rowCount: number + columnCount: number + mappings: MigrationMapping[] +} + +export type MigrationPreview = { + tables: MigrationTablePreview[] + totalRows: number + typeConversions: number +} diff --git a/vite.config.ts b/vite.config.ts index b28a9a47..fbdce491 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -23,14 +23,14 @@ export default defineConfig(async () => ({ clearScreen: false, // 2. tauri expects a fixed port, fail if that port is not available server: { - port: 1420, + port: 1521, strictPort: true, host: host || false, hmr: host ? { protocol: 'ws', host, - port: 1421, + port: 1522, } : undefined, watch: {