From 5b60ef9ab1bf773cc3955912291524b57fec852e Mon Sep 17 00:00:00 2001 From: tianzhou Date: Tue, 23 Sep 2025 17:27:04 +0800 Subject: [PATCH 1/3] feat: separate env --- .env.example | 16 +- README.md | 82 ++++++-- src/config/__tests__/env.test.ts | 332 +++++++++++++++++++++++++++++++ src/config/env.ts | 93 ++++++++- 4 files changed, 505 insertions(+), 18 deletions(-) create mode 100644 src/config/__tests__/env.test.ts diff --git a/.env.example b/.env.example index 98821c09..0ff31fda 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ # DBHub Configuration -# Connection String (preferred method) +# Method 1: Connection String (DSN) # Use one of these DSN formats: # DSN=postgres://user:password@localhost:5432/dbname # DSN=sqlite:///path/to/database.db @@ -9,6 +9,20 @@ # DSN=mysql://user:password@localhost:3306/dbname DSN= +# Method 2: Individual Database Parameters +# Use this method if your password contains special characters like @, :, /, #, etc. +# that would break URL parsing in the DSN format above +# DB_TYPE=postgres +# DB_HOST=localhost +# DB_PORT=5432 +# DB_USER=postgres +# DB_PASSWORD=my@password:with/special#chars +# DB_NAME=mydatabase + +# Supported DB_TYPE values: postgres, mysql, mariadb, sqlserver, sqlite +# DB_PORT is optional - defaults to standard port for each database type +# For SQLite: only DB_TYPE and DB_NAME are required (DB_NAME is the file path) + # Transport configuration # --transport=stdio (default) for stdio transport # --transport=sse for SSE transport with HTTP server diff --git a/README.md b/README.md index ac23c78d..d60c89d1 100644 --- a/README.md +++ b/README.md @@ -308,9 +308,14 @@ npx @bytebase/dbhub --demo ``` > [!WARNING] -> If your user/password contains special characters, you need to escape them first. (e.g. `pass#word` should be escaped as `pass%23word`) +> If your user/password contains special characters, you have two options: +> +> 1. Escape them in the DSN (e.g. `pass#word` should be escaped as `pass%23word`) +> 2. Use the individual database parameters method below (recommended) -For real databases, a Database Source Name (DSN) is required. You can provide this in several ways: +For real databases, you can configure the database connection in two ways: + +#### Method 1: Database Source Name (DSN) - **Command line argument** (highest priority): @@ -332,6 +337,45 @@ For real databases, a Database Source Name (DSN) is required. You can provide th DSN=postgres://user:password@localhost:5432/dbname?sslmode=disable ``` +#### Method 2: Individual Database Parameters + +If your password contains special characters that would break URL parsing, use individual environment variables instead: + +- **Environment variables**: + + ```bash + export DB_TYPE=postgres + export DB_HOST=localhost + export DB_PORT=5432 + export DB_USER=myuser + export DB_PASSWORD='my@complex:password/with#special&chars' + export DB_NAME=mydatabase + npx @bytebase/dbhub + ``` + +- **Environment file**: + ``` + DB_TYPE=postgres + DB_HOST=localhost + DB_PORT=5432 + DB_USER=myuser + DB_PASSWORD=my@complex:password/with#special&chars + DB_NAME=mydatabase + ``` + +**Supported DB_TYPE values**: `postgres`, `mysql`, `mariadb`, `sqlserver`, `sqlite` + +**Default ports** (when DB_PORT is omitted): + +- PostgreSQL: `5432` +- MySQL/MariaDB: `3306` +- SQL Server: `1433` + +**For SQLite**: Only `DB_TYPE=sqlite` and `DB_NAME=/path/to/database.db` are required. + +> [!TIP] +> Use the individual parameter method when your password contains special characters like `@`, `:`, `/`, `#`, `&`, `=` that would break DSN parsing. + > [!WARNING] > When running in Docker, use `host.docker.internal` instead of `localhost` to connect to databases running on your host machine. For example: `mysql://user:password@host.docker.internal:3306/dbname` @@ -368,20 +412,26 @@ Extra query parameters: ### Command line options -| Option | Environment Variable | Description | Default | -| -------------- | -------------------- | ---------------------------------------------------------------- | ---------------------------- | -| dsn | `DSN` | Database connection string | Required if not in demo mode | -| transport | `TRANSPORT` | Transport mode: `stdio` or `http` | `stdio` | -| port | `PORT` | HTTP server port (only applicable when using `--transport=http`) | `8080` | -| readonly | `READONLY` | Restrict SQL execution to read-only operations | `false` | -| max-rows | N/A | Limit the number of rows returned from SELECT queries | No limit | -| demo | N/A | Run in demo mode with sample employee database | `false` | -| ssh-host | `SSH_HOST` | SSH server hostname for tunnel connection | N/A | -| ssh-port | `SSH_PORT` | SSH server port | `22` | -| ssh-user | `SSH_USER` | SSH username | N/A | -| ssh-password | `SSH_PASSWORD` | SSH password (for password authentication) | N/A | -| ssh-key | `SSH_KEY` | Path to SSH private key file | N/A | -| ssh-passphrase | `SSH_PASSPHRASE` | Passphrase for SSH private key | N/A | +| Option | Environment Variable | Description | Default | +| -------------- | -------------------- | --------------------------------------------------------------------- | ---------------------------- | +| dsn | `DSN` | Database connection string | Required if not in demo mode | +| N/A | `DB_TYPE` | Database type: `postgres`, `mysql`, `mariadb`, `sqlserver`, `sqlite` | N/A | +| N/A | `DB_HOST` | Database server hostname (not needed for SQLite) | N/A | +| N/A | `DB_PORT` | Database server port (uses default if omitted, not needed for SQLite) | N/A | +| N/A | `DB_USER` | Database username (not needed for SQLite) | N/A | +| N/A | `DB_PASSWORD` | Database password (not needed for SQLite) | N/A | +| N/A | `DB_NAME` | Database name or SQLite file path | N/A | +| transport | `TRANSPORT` | Transport mode: `stdio` or `http` | `stdio` | +| port | `PORT` | HTTP server port (only applicable when using `--transport=http`) | `8080` | +| readonly | `READONLY` | Restrict SQL execution to read-only operations | `false` | +| max-rows | N/A | Limit the number of rows returned from SELECT queries | No limit | +| demo | N/A | Run in demo mode with sample employee database | `false` | +| ssh-host | `SSH_HOST` | SSH server hostname for tunnel connection | N/A | +| ssh-port | `SSH_PORT` | SSH server port | `22` | +| ssh-user | `SSH_USER` | SSH username | N/A | +| ssh-password | `SSH_PASSWORD` | SSH password (for password authentication) | N/A | +| ssh-key | `SSH_KEY` | Path to SSH private key file | N/A | +| ssh-passphrase | `SSH_PASSPHRASE` | Passphrase for SSH private key | N/A | The demo mode uses an in-memory SQLite database loaded with the [sample employee database](https://github.com/bytebase/dbhub/tree/main/resources/employee-sqlite) that includes tables for employees, departments, titles, salaries, department employees, and department managers. The sample database includes SQL scripts for table creation, data loading, and testing. diff --git a/src/config/__tests__/env.test.ts b/src/config/__tests__/env.test.ts new file mode 100644 index 00000000..a2fea5e8 --- /dev/null +++ b/src/config/__tests__/env.test.ts @@ -0,0 +1,332 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { buildDSNFromEnvParams, resolveDSN } from '../env.js'; + +describe('Environment Configuration Tests', () => { + // Store original env values to restore after tests + const originalEnv = { ...process.env }; + + beforeEach(() => { + // Clear relevant environment variables before each test + delete process.env.DB_TYPE; + delete process.env.DB_HOST; + delete process.env.DB_PORT; + delete process.env.DB_USER; + delete process.env.DB_PASSWORD; + delete process.env.DB_NAME; + delete process.env.DSN; + }); + + afterEach(() => { + // Restore original environment + process.env = { ...originalEnv }; + }); + + describe('buildDSNFromEnvParams', () => { + it('should build PostgreSQL DSN with all parameters', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_PORT = '5432'; + process.env.DB_USER = 'testuser'; + process.env.DB_PASSWORD = 'testpass'; + process.env.DB_NAME = 'testdb'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'postgres://testuser:testpass@localhost:5432/testdb', + source: 'individual environment variables' + }); + }); + + it('should build MySQL DSN with default port when port not specified', () => { + process.env.DB_TYPE = 'mysql'; + process.env.DB_HOST = 'mysql.example.com'; + process.env.DB_USER = 'admin'; + process.env.DB_PASSWORD = 'secret'; + process.env.DB_NAME = 'myapp'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'mysql://admin:secret@mysql.example.com:3306/myapp', + source: 'individual environment variables' + }); + }); + + it('should build MariaDB DSN with default port', () => { + process.env.DB_TYPE = 'mariadb'; + process.env.DB_HOST = 'mariadb.example.com'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'database'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'mariadb://user:pass@mariadb.example.com:3306/database', + source: 'individual environment variables' + }); + }); + + it('should build SQL Server DSN with default port', () => { + process.env.DB_TYPE = 'sqlserver'; + process.env.DB_HOST = 'sqlserver.example.com'; + process.env.DB_USER = 'sa'; + process.env.DB_PASSWORD = 'strongpass'; + process.env.DB_NAME = 'master'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'sqlserver://sa:strongpass@sqlserver.example.com:1433/master', + source: 'individual environment variables' + }); + }); + + it('should build SQLite DSN with only DB_TYPE and DB_NAME', () => { + process.env.DB_TYPE = 'sqlite'; + process.env.DB_NAME = '/path/to/database.db'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'sqlite:////path/to/database.db', + source: 'individual environment variables' + }); + }); + + it('should handle postgresql type and normalize to postgres protocol', () => { + process.env.DB_TYPE = 'postgresql'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe('postgres://user:pass@localhost:5432/db'); + }); + + it('should properly encode special characters in password', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'test@pass:with/special#chars&more=special'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe( + 'postgres://user:test%40pass%3Awith%2Fspecial%23chars%26more%3Dspecial@localhost:5432/db' + ); + }); + + it('should properly encode special characters in username', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user@domain.com'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe( + 'postgres://user%40domain.com:pass@localhost:5432/db' + ); + }); + + it('should properly encode special characters in database name', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'my-db@test'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe( + 'postgres://user:pass@localhost:5432/my-db%40test' + ); + }); + + it('should handle SQLite with special characters in file path', () => { + process.env.DB_TYPE = 'sqlite'; + process.env.DB_NAME = '/tmp/test_db@#$.db'; + + const result = buildDSNFromEnvParams(); + + expect(result).toEqual({ + dsn: 'sqlite:////tmp/test_db@#$.db', + source: 'individual environment variables' + }); + }); + + it('should return null when required parameters are missing for non-SQLite databases', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + // Missing DB_USER, DB_PASSWORD, DB_NAME + + const result = buildDSNFromEnvParams(); + + expect(result).toBeNull(); + }); + + it('should return null when DB_TYPE is missing', () => { + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result).toBeNull(); + }); + + it('should return null when SQLite is missing DB_NAME', () => { + process.env.DB_TYPE = 'sqlite'; + // Missing DB_NAME + + const result = buildDSNFromEnvParams(); + + expect(result).toBeNull(); + }); + + it('should throw error for unsupported database type', () => { + process.env.DB_TYPE = 'oracle'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + expect(() => buildDSNFromEnvParams()).toThrow( + 'Unsupported DB_TYPE: oracle. Supported types: postgres, postgresql, mysql, mariadb, sqlserver, sqlite' + ); + }); + + it('should use custom port when provided', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_PORT = '9999'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe('postgres://user:pass@localhost:9999/db'); + }); + + it('should return null for empty password (required field)', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = ''; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result).toBeNull(); + }); + }); + + describe('resolveDSN integration with individual parameters', () => { + it('should use DSN when both DSN and individual parameters are provided', () => { + process.env.DSN = 'postgres://direct:dsn@localhost:5432/directdb'; + process.env.DB_TYPE = 'mysql'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = resolveDSN(); + + expect(result).toEqual({ + dsn: 'postgres://direct:dsn@localhost:5432/directdb', + source: 'environment variable' + }); + }); + + it('should fall back to individual parameters when DSN is not provided', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = resolveDSN(); + + expect(result).toEqual({ + dsn: 'postgres://user:pass@localhost:5432/db', + source: 'individual environment variables' + }); + }); + + it('should return null when neither DSN nor complete individual parameters are provided', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + // Missing required parameters + + const result = resolveDSN(); + + expect(result).toBeNull(); + }); + + it('should handle SQLite individual parameters correctly', () => { + process.env.DB_TYPE = 'sqlite'; + process.env.DB_NAME = ':memory:'; + + const result = resolveDSN(); + + expect(result).toEqual({ + dsn: 'sqlite:///:memory:', + source: 'individual environment variables' + }); + }); + }); + + describe('edge cases and complex scenarios', () => { + it('should handle password with all special URL characters', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = '!@#$%^&*()+={}[]|\\:";\'<>?,./~`'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + // Verify it builds without error and contains encoded characters + expect(result).toBeTruthy(); + // Note: encodeURIComponent doesn't encode ! so it remains as ! + expect(result?.dsn).toContain('!'); // ! is not encoded + expect(result?.dsn).toContain('%40'); // @ + expect(result?.dsn).toContain('%23'); // # + expect(result?.dsn).toContain('%24'); // $ + expect(result?.dsn).toContain('%25'); // % + }); + + it('should handle database names with Unicode characters', () => { + process.env.DB_TYPE = 'postgres'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'тест_база_данных'; // Cyrillic characters + + const result = buildDSNFromEnvParams(); + + expect(result).toBeTruthy(); + expect(result?.dsn).toContain('%D1%82%D0%B5%D1%81%D1%82'); // Encoded Cyrillic + }); + + it('should be case insensitive for database type', () => { + process.env.DB_TYPE = 'POSTGRES'; + process.env.DB_HOST = 'localhost'; + process.env.DB_USER = 'user'; + process.env.DB_PASSWORD = 'pass'; + process.env.DB_NAME = 'db'; + + const result = buildDSNFromEnvParams(); + + expect(result?.dsn).toBe('postgres://user:pass@localhost:5432/db'); + }); + }); +}); \ No newline at end of file diff --git a/src/config/env.ts b/src/config/env.ts index 7495c73d..814ff96c 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -104,6 +104,78 @@ export function isReadOnlyMode(): boolean { return false; } +/** + * Build DSN from individual environment variables + * Returns the constructed DSN or null if required variables are missing + */ +export function buildDSNFromEnvParams(): { dsn: string; source: string } | null { + // Check for required environment variables + const dbType = process.env.DB_TYPE; + const dbHost = process.env.DB_HOST; + const dbUser = process.env.DB_USER; + const dbPassword = process.env.DB_PASSWORD; + const dbName = process.env.DB_NAME; + const dbPort = process.env.DB_PORT; + + // For SQLite, only DB_TYPE and DB_NAME are required + if (dbType?.toLowerCase() === 'sqlite') { + if (!dbType || !dbName) { + return null; + } + } else { + // For other databases, require all essential parameters + if (!dbType || !dbHost || !dbUser || !dbPassword || !dbName) { + return null; + } + } + + // Validate supported database types + const supportedTypes = ['postgres', 'postgresql', 'mysql', 'mariadb', 'sqlserver', 'sqlite']; + if (!supportedTypes.includes(dbType.toLowerCase())) { + throw new Error(`Unsupported DB_TYPE: ${dbType}. Supported types: ${supportedTypes.join(', ')}`); + } + + // Determine default port based on database type + let port = dbPort; + if (!port) { + switch (dbType.toLowerCase()) { + case 'postgres': + case 'postgresql': + port = '5432'; + break; + case 'mysql': + case 'mariadb': + port = '3306'; + break; + case 'sqlserver': + port = '1433'; + break; + case 'sqlite': + // SQLite doesn't use host/port, handle differently + return { + dsn: `sqlite:///${dbName}`, + source: 'individual environment variables' + }; + default: + throw new Error(`Unknown database type for port determination: ${dbType}`); + } + } + + // URL encode components to handle special characters + const encodedUser = encodeURIComponent(dbUser!); + const encodedPassword = encodeURIComponent(dbPassword!); + const encodedDbName = encodeURIComponent(dbName!); + + // Construct DSN + const protocol = dbType.toLowerCase() === 'postgresql' ? 'postgres' : dbType.toLowerCase(); + const dsn = `${protocol}://${encodedUser}:${encodedPassword}@${dbHost}:${port}/${encodedDbName}`; + + return { + dsn, + source: 'individual environment variables' + }; +} + /** * Resolve DSN from command line args, environment variables, or .env files * Returns the DSN and its source, or null if not found @@ -132,12 +204,31 @@ export function resolveDSN(): { dsn: string; source: string; isDemo?: boolean } return { dsn: process.env.DSN, source: "environment variable" }; } - // 3. Try loading from .env files + // 3. Check for individual DB parameters from environment + const envParamsResult = buildDSNFromEnvParams(); + if (envParamsResult) { + return envParamsResult; + } + + // 4. Try loading from .env files const loadedEnvFile = loadEnvFiles(); + + // 5. Check for DSN in .env file if (loadedEnvFile && process.env.DSN) { return { dsn: process.env.DSN, source: `${loadedEnvFile} file` }; } + // 6. Check for individual DB parameters from .env file + if (loadedEnvFile) { + const envFileParamsResult = buildDSNFromEnvParams(); + if (envFileParamsResult) { + return { + dsn: envFileParamsResult.dsn, + source: `${loadedEnvFile} file (individual parameters)` + }; + } + } + return null; } From 17be6dd9205acd05871b976c8eb565cc30be3b17 Mon Sep 17 00:00:00 2001 From: Tianzhou Date: Tue, 23 Sep 2025 17:29:48 +0800 Subject: [PATCH 2/3] Update src/config/env.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/config/env.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/env.ts b/src/config/env.ts index 814ff96c..6d5bf819 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -119,7 +119,7 @@ export function buildDSNFromEnvParams(): { dsn: string; source: string } | null // For SQLite, only DB_TYPE and DB_NAME are required if (dbType?.toLowerCase() === 'sqlite') { - if (!dbType || !dbName) { + if (!dbName) { return null; } } else { From 4f456f3e4881a9b6064e8299030244e7d240bfb4 Mon Sep 17 00:00:00 2001 From: Tianzhou Date: Tue, 23 Sep 2025 17:29:58 +0800 Subject: [PATCH 3/3] Update src/config/env.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/config/env.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/config/env.ts b/src/config/env.ts index 6d5bf819..c986323b 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -161,10 +161,13 @@ export function buildDSNFromEnvParams(): { dsn: string; source: string } | null } } - // URL encode components to handle special characters - const encodedUser = encodeURIComponent(dbUser!); - const encodedPassword = encodeURIComponent(dbPassword!); - const encodedDbName = encodeURIComponent(dbName!); + // At this point, dbUser, dbPassword, and dbName are guaranteed to be non-null due to earlier checks. + const user: string = dbUser as string; + const password: string = dbPassword as string; + const dbNameStr: string = dbName as string; + const encodedUser = encodeURIComponent(user); + const encodedPassword = encodeURIComponent(password); + const encodedDbName = encodeURIComponent(dbNameStr); // Construct DSN const protocol = dbType.toLowerCase() === 'postgresql' ? 'postgres' : dbType.toLowerCase();