diff --git a/AGENTS.md b/AGENTS.md index 361331a1..dc34d8f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,9 +28,9 @@ - Follow the lightweight rules in the active GitHub Project item: one PR should cover only 1–2 small tasks, and every stage must pass the MCP Inspector gate before it is marked complete. - Capture TODOs in code comments or checklists but resolve them within the same iteration; avoid carrying speculative work between stages. - Log significant findings, regressions, or retest evidence back into the active GitHub Project item so the current working record stays authoritative. -- Pre-freeze stance: before Loocor declares API/data compatibility freeze, schema and configuration breaking changes may use clean rebuilds with companion updates in the same PR. Do not add migrations, compatibility layers, or fallbacks unless the active Project item explicitly requires them. +- Pre-freeze stance: before Loocor declares API/data compatibility freeze, schema and configuration breaking changes may use clean rebuilds with companion updates in the same PR. Persistent SQLite schema or one-time data changes use the embedded migration contract below; do not add unrelated compatibility layers or fallbacks. - Do not add fallback behavior unless the design or product requirements explicitly call for it. If fallback semantics are ambiguous, stop and ask rather than inventing one. -- Do not embed migration logic in the main program. When migration is needed, provide it as a separate tool or script so runtime code stays simple and focused. +- `backend/crates/mcpmate-migrations` is the sole owner of durable SQLite schema migration. Before designing a persistent schema or one-time data change, read its `README.md`; add the change through its versioned migration ledger, never through `ensure_schema`, `ensure_column`, or ad-hoc DDL in a business module. Only physical database initialization may execute pending migrations; business modules remain read-only verifiers of the migrated contract. ## GitHub Project Workflow - Use the GitHub Project **MCPMate** as the canonical task center for roadmap planning, development slices, release/distribution work, marketing follow-up, and cross-repository coordination. diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0b71e43d..4519b1b7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,6 +1517,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2649,6 +2659,7 @@ dependencies = [ "lru", "mcpmate-capability-store", "mcpmate-llm", + "mcpmate-migrations", "mcpmate-secrets", "mockall", "nanoid", @@ -2705,6 +2716,7 @@ dependencies = [ "chrono", "dashmap 6.1.0", "lru", + "mcpmate-migrations", "rmcp", "schemars 0.9.0", "serde", @@ -2731,6 +2743,20 @@ dependencies = [ "url", ] +[[package]] +name = "mcpmate-migrations" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "fs2", + "sha2", + "sqlx", + "tempfile", + "tokio", +] + [[package]] name = "mcpmate-secrets" version = "0.1.0" @@ -2738,6 +2764,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "keyring", + "mcpmate-migrations", "ring", "serde", "serde_json", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d2336069..d5a734d6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -39,6 +39,7 @@ windows-sys = { version = "0.59.0", features = ["Win32_Foundation", "Win32_Stora anyhow = "1.0" async-trait = "0.1" mcpmate-capability-store = { path = "crates/mcpmate-capability-store" } +mcpmate-migrations = { path = "crates/mcpmate-migrations" } mcpmate-llm = { path = "crates/mcpmate-llm" } mcpmate-secrets = { path = "crates/mcpmate-secrets" } is-terminal = "0.4" diff --git a/backend/crates/mcpmate-capability-store/Cargo.toml b/backend/crates/mcpmate-capability-store/Cargo.toml index 780ab688..f8966533 100644 --- a/backend/crates/mcpmate-capability-store/Cargo.toml +++ b/backend/crates/mcpmate-capability-store/Cargo.toml @@ -10,6 +10,7 @@ async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } dashmap = "6.1" lru = "0.12" +mcpmate-migrations = { path = "../mcpmate-migrations" } rmcp = { version = "=3.0.1" } schemars = { version = "0.9", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } diff --git a/backend/crates/mcpmate-capability-store/src/schema.rs b/backend/crates/mcpmate-capability-store/src/schema.rs index be0d38dc..5fa3a336 100644 --- a/backend/crates/mcpmate-capability-store/src/schema.rs +++ b/backend/crates/mcpmate-capability-store/src/schema.rs @@ -1,505 +1,11 @@ use sqlx::{Pool, Sqlite}; -use crate::Result; - -const CAPABILITY_SCHEMA_EPOCH: i64 = 4; -const LEGACY_CAPABILITY_TABLES: &[&str] = &[ - "capability_records", - "profile_tool", - "profile_prompt", - "profile_resource", - "profile_resource_template", - "profile_server", -]; +use crate::{CatalogError, Result}; pub(crate) async fn ensure_schema(pool: &Pool) -> Result<()> { - let mut transaction = pool.begin().await?; - ensure_compatible_schema(&mut transaction).await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_server_snapshots ( - server_id TEXT PRIMARY KEY, - server_name TEXT NOT NULL, - config_fingerprint TEXT NOT NULL, - record_format_version INTEGER NOT NULL, - catalog_revision INTEGER NOT NULL, - snapshot_state TEXT NOT NULL, - initialize_payload TEXT NOT NULL, - observed_at TEXT NOT NULL, - committed_at TEXT NOT NULL, - last_error TEXT - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_kind_states ( - server_id TEXT NOT NULL, - position INTEGER NOT NULL, - kind TEXT NOT NULL, - declaration_state TEXT NOT NULL, - inventory_state TEXT NOT NULL, - error TEXT, - failure_kind TEXT, - timeout_ms INTEGER, - catalog_revision INTEGER NOT NULL, - observed_at TEXT NOT NULL, - PRIMARY KEY (server_id, kind), - FOREIGN KEY (server_id) REFERENCES capability_server_snapshots(server_id) ON DELETE CASCADE - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_refs ( - ref_id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - kind TEXT NOT NULL, - origin_key TEXT NOT NULL, - state TEXT NOT NULL, - state_generation INTEGER NOT NULL, - first_observed_revision INTEGER NOT NULL, - last_observed_revision INTEGER NOT NULL, - FOREIGN KEY (server_id) REFERENCES capability_server_snapshots(server_id) ON DELETE CASCADE, - UNIQUE (server_id, kind, origin_key) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query("CREATE INDEX IF NOT EXISTS idx_capability_refs_server_kind ON capability_refs(server_id, kind)") - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_versions ( - capability_id TEXT PRIMARY KEY, - ref_id TEXT NOT NULL, - canonical_record BLOB NOT NULL, - source_payload BLOB NOT NULL, - effective_payload BLOB NOT NULL, - record_format TEXT NOT NULL, - first_observed_revision INTEGER NOT NULL, - FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id) ON DELETE CASCADE, - UNIQUE (ref_id, capability_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query("CREATE INDEX IF NOT EXISTS idx_capability_versions_ref ON capability_versions(ref_id)") - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_ref_current ( - ref_id TEXT PRIMARY KEY, - capability_id TEXT NOT NULL, - catalog_revision INTEGER NOT NULL, - FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id) ON DELETE CASCADE, - FOREIGN KEY (capability_id) REFERENCES capability_versions(capability_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - "CREATE INDEX IF NOT EXISTS idx_capability_ref_current_version ON capability_ref_current(capability_id)", - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_manifests ( - manifest_id TEXT PRIMARY KEY, - consumer_id TEXT NOT NULL, - canonical_content BLOB NOT NULL, - created_at TEXT NOT NULL - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_manifest_entries ( - manifest_id TEXT NOT NULL, - position INTEGER NOT NULL, - ref_id TEXT NOT NULL, - capability_id TEXT NOT NULL, - PRIMARY KEY (manifest_id, position), - UNIQUE (manifest_id, ref_id), - FOREIGN KEY (manifest_id) REFERENCES surface_manifests(manifest_id) ON DELETE CASCADE, - FOREIGN KEY (ref_id, capability_id) REFERENCES capability_versions(ref_id, capability_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_proposals ( - proposal_id TEXT PRIMARY KEY, - consumer_id TEXT NOT NULL, - base_publication_id TEXT, - proposed_manifest_id TEXT NOT NULL, - trigger_kind TEXT NOT NULL, - trigger_id TEXT NOT NULL, - source_revision_set TEXT NOT NULL, - diff_summary TEXT NOT NULL, - lifecycle TEXT NOT NULL CHECK (lifecycle IN ('pending', 'resolved', 'superseded')), - created_at TEXT NOT NULL, - resolved_at TEXT, - FOREIGN KEY (proposed_manifest_id) REFERENCES surface_manifests(manifest_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_review_items ( - review_item_id TEXT PRIMARY KEY, - created_by_proposal_id TEXT NOT NULL, - consumer_id TEXT NOT NULL, - ref_id TEXT NOT NULL, - before_capability_id TEXT, - target_capability_id TEXT, - target_key TEXT NOT NULL, - change_class TEXT NOT NULL, - policy_action TEXT NOT NULL, - lifecycle TEXT NOT NULL CHECK (lifecycle IN ('pending', 'resolved', 'obsolete')), - current_decision_id TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - UNIQUE (consumer_id, ref_id, target_key), - FOREIGN KEY (created_by_proposal_id) REFERENCES surface_proposals(proposal_id), - FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id), - FOREIGN KEY (before_capability_id) REFERENCES capability_versions(capability_id), - FOREIGN KEY (target_capability_id) REFERENCES capability_versions(capability_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_review_decisions ( - decision_id TEXT PRIMARY KEY, - review_item_id TEXT NOT NULL, - resolution_action TEXT NOT NULL CHECK ( - resolution_action IN ('approve_target', 'reject_target', 'keep_intent', 'remove_intent', 'rebind_ref') - ), - resolution_payload TEXT, - actor TEXT NOT NULL, - decided_at TEXT NOT NULL, - supersedes_decision_id TEXT, - FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id), - FOREIGN KEY (supersedes_decision_id) REFERENCES surface_review_decisions(decision_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_proposal_review_items ( - proposal_id TEXT NOT NULL, - review_item_id TEXT NOT NULL, - PRIMARY KEY (proposal_id, review_item_id), - FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id) ON DELETE CASCADE, - FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id) ON DELETE CASCADE - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_review_owners ( - review_item_id TEXT NOT NULL, - owner_type TEXT NOT NULL CHECK ( - owner_type IN ( - 'standard_profile', - 'custom_profile', - 'consumer_direct_exposure', - 'profile_server_exposure', - 'consumer_server_exposure', - 'mode_rule' - ) - ), - owner_id TEXT NOT NULL, - active INTEGER NOT NULL CHECK (active IN (0, 1)), - first_proposal_id TEXT NOT NULL, - last_proposal_id TEXT NOT NULL, - PRIMARY KEY (review_item_id, owner_type, owner_id), - FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id) ON DELETE CASCADE, - FOREIGN KEY (first_proposal_id) REFERENCES surface_proposals(proposal_id), - FOREIGN KEY (last_proposal_id) REFERENCES surface_proposals(proposal_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_publications ( - publication_id TEXT PRIMARY KEY, - consumer_id TEXT NOT NULL, - manifest_id TEXT NOT NULL, - proposal_id TEXT, - reason TEXT NOT NULL, - published_by TEXT NOT NULL, - published_at TEXT NOT NULL, - supersedes_publication_id TEXT, - FOREIGN KEY (manifest_id) REFERENCES surface_manifests(manifest_id), - FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id), - FOREIGN KEY (supersedes_publication_id) REFERENCES surface_publications(publication_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS consumer_surface_bindings ( - consumer_id TEXT PRIMARY KEY, - active_publication_id TEXT NOT NULL, - generation INTEGER NOT NULL CHECK (generation > 0), - FOREIGN KEY (active_publication_id) REFERENCES surface_publications(publication_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS consumer_surface_generations ( - consumer_id TEXT PRIMARY KEY, - last_generation INTEGER NOT NULL CHECK (last_generation >= 0) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - "CREATE INDEX IF NOT EXISTS idx_surface_proposals_consumer ON surface_proposals(consumer_id, created_at)", - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - "CREATE INDEX IF NOT EXISTS idx_surface_reviews_consumer_state ON surface_review_items(consumer_id, lifecycle)", - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - "CREATE INDEX IF NOT EXISTS idx_surface_publications_consumer ON surface_publications(consumer_id, published_at)", - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_reconciliation_jobs ( - idempotency_key TEXT PRIMARY KEY, - cause_kind TEXT NOT NULL, - cause_id TEXT NOT NULL, - consumer_id TEXT NOT NULL, - target_revision_set TEXT NOT NULL, - expected_binding_generation INTEGER NOT NULL, - status TEXT NOT NULL CHECK (status IN ('pending', 'leased', 'succeeded', 'failed')), - attempt_count INTEGER NOT NULL, - leased_by TEXT, - lease_expires_at TEXT, - next_attempt_at TEXT NOT NULL, - last_error TEXT, - success_receipt TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ) - "#, - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_surface_jobs_lease - ON surface_reconciliation_jobs(status, next_attempt_at, lease_expires_at) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS surface_outbox_events ( - event_id TEXT PRIMARY KEY, - event_kind TEXT NOT NULL, - aggregate_id TEXT NOT NULL, - payload TEXT NOT NULL, - created_at TEXT NOT NULL, - delivered_at TEXT - ) - "#, - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_surface_outbox_pending - ON surface_outbox_events(delivered_at, created_at) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS capability_change_events ( - event_id TEXT PRIMARY KEY, - consumer_id TEXT NOT NULL, - proposal_id TEXT NOT NULL, - ref_id TEXT NOT NULL, - before_capability_id TEXT, - target_capability_id TEXT, - change_class TEXT NOT NULL, - policy_action TEXT NOT NULL, - actor TEXT NOT NULL, - occurred_at TEXT NOT NULL, - FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id), - FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id), - FOREIGN KEY (before_capability_id) REFERENCES capability_versions(capability_id), - FOREIGN KEY (target_capability_id) REFERENCES capability_versions(capability_id) - ) - "#, - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_capability_change_events_consumer_time - ON capability_change_events(consumer_id, occurred_at) - "#, - ) - .execute(&mut *transaction) - .await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS configuration_mode_transitions ( - transition_id TEXT PRIMARY KEY, - previous_mode TEXT NOT NULL CHECK (previous_mode IN ('unify', 'hosted', 'transparent')), - target_mode TEXT NOT NULL CHECK (target_mode IN ('unify', 'hosted', 'transparent')), - status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), - created_at TEXT NOT NULL, - completed_at TEXT - ) - "#, - ) - .execute(&mut *transaction) - .await?; - sqlx::query( - r#" - CREATE UNIQUE INDEX IF NOT EXISTS idx_configuration_mode_transitions_single_pending - ON configuration_mode_transitions(status) - WHERE status = 'pending' - "#, - ) - .execute(&mut *transaction) - .await?; - - transaction.commit().await?; - Ok(()) -} - -async fn ensure_compatible_schema(transaction: &mut sqlx::Transaction<'_, Sqlite>) -> Result<()> { - let metadata_exists: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'capability_schema_metadata')", - ) - .fetch_one(&mut **transaction) - .await?; - if metadata_exists { - let epoch: i64 = sqlx::query_scalar("SELECT schema_epoch FROM capability_schema_metadata WHERE singleton = 1") - .fetch_optional(&mut **transaction) - .await? - .ok_or_else(|| crate::CatalogError::IncompatibleSchema { - details: "capability_schema_metadata is missing its singleton epoch row".to_string(), - })?; - if epoch != CAPABILITY_SCHEMA_EPOCH { - return Err(crate::CatalogError::IncompatibleSchema { - details: format!( - "schema epoch {epoch} is not supported; clean rebuild is required for epoch {CAPABILITY_SCHEMA_EPOCH}" - ), - }); - } - return Ok(()); - } - - let existing_tables: Vec = - sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") - .fetch_all(&mut **transaction) - .await?; - let incompatible_tables = existing_tables - .iter() - .filter(|table| { - LEGACY_CAPABILITY_TABLES.contains(&table.as_str()) - || matches!( - table.as_str(), - "capability_refs" - | "capability_versions" - | "capability_ref_current" - | "surface_manifests" - | "surface_publications" - | "consumer_surface_generations" - ) + mcpmate_migrations::verify_capability_catalog_database(pool) + .await + .map_err(|error| CatalogError::IncompatibleSchema { + details: error.to_string(), }) - .cloned() - .collect::>(); - if !incompatible_tables.is_empty() { - return Err(crate::CatalogError::IncompatibleSchema { - details: format!( - "database contains unversioned capability tables [{}]; clean rebuild is required", - incompatible_tables.join(", ") - ), - }); - } - - sqlx::query( - r#" - CREATE TABLE capability_schema_metadata ( - singleton INTEGER PRIMARY KEY CHECK (singleton = 1), - schema_epoch INTEGER NOT NULL - ) - "#, - ) - .execute(&mut **transaction) - .await?; - sqlx::query("INSERT INTO capability_schema_metadata (singleton, schema_epoch) VALUES (1, ?)") - .bind(CAPABILITY_SCHEMA_EPOCH) - .execute(&mut **transaction) - .await?; - Ok(()) } diff --git a/backend/crates/mcpmate-capability-store/tests/catalog_contract.rs b/backend/crates/mcpmate-capability-store/tests/catalog_contract.rs index 9376a188..c9f30117 100644 --- a/backend/crates/mcpmate-capability-store/tests/catalog_contract.rs +++ b/backend/crates/mcpmate-capability-store/tests/catalog_contract.rs @@ -1,9 +1,10 @@ use mcpmate_capability_store::{ CapabilityCatalog, CapabilityFailureObservation, CapabilityId, CapabilityKind, CapabilityObservation, - CapabilityPayload, CapabilityRefId, CapabilityRefState, CatalogRecord, DeclarationState, DerivedCapabilityCache, - EffectiveCapabilityRecordV1, InventoryState, KindFailureKind, KindObservation, ProjectionKey, ProjectionNameDomain, - ProjectionPayload, SnapshotState, SqliteCapabilityCatalog, + CapabilityPayload, CapabilityRefId, CapabilityRefState, CatalogError, CatalogRecord, DeclarationState, + DerivedCapabilityCache, EffectiveCapabilityRecordV1, InventoryState, KindFailureKind, KindObservation, + ProjectionKey, ProjectionNameDomain, ProjectionPayload, SnapshotState, SqliteCapabilityCatalog, }; +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; use rmcp::model::{InitializeResult, Prompt, Resource, ResourceTemplate, Tool}; use serde::de::DeserializeOwned; use serde_json::{Value, json}; @@ -130,6 +131,9 @@ async fn test_pool() -> Pool { .connect("sqlite::memory:") .await .unwrap(); + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config schema"); SqliteCapabilityCatalog::new(pool.clone()) .ensure_schema() .await @@ -137,6 +141,22 @@ async fn test_pool() -> Pool { pool } +#[tokio::test] +async fn ensure_schema_rejects_a_missing_capability_table() { + let pool = test_pool().await; + sqlx::query("DROP TABLE capability_refs") + .execute(&pool) + .await + .expect("remove capability table"); + + let error = SqliteCapabilityCatalog::new(pool) + .ensure_schema() + .await + .expect_err("damaged capability schema must fail initialization"); + + assert!(matches!(error, CatalogError::IncompatibleSchema { .. })); +} + fn test_tool(name: &str) -> Tool { decode(json!({ "name": name, @@ -1004,6 +1024,15 @@ async fn concurrent_writers_from_independent_pools_commit_consecutive_revisions( .await .unwrap(), ); + prepare_config_database( + first.pool(), + DatabaseSource::File { + path: &directory.path().join("catalog.db"), + existed_before_open: false, + }, + ) + .await + .unwrap(); first.ensure_schema().await.unwrap(); let second = SqliteCapabilityCatalog::new( SqlitePoolOptions::new() @@ -1062,6 +1091,15 @@ async fn concurrent_identical_writers_produce_one_change_and_one_noop() { .await .unwrap(), ); + prepare_config_database( + first.pool(), + DatabaseSource::File { + path: &directory.path().join("catalog-noop.db"), + existed_before_open: false, + }, + ) + .await + .unwrap(); first.ensure_schema().await.unwrap(); let second = SqliteCapabilityCatalog::new( SqlitePoolOptions::new() @@ -1143,6 +1181,9 @@ async fn remove_server_retires_refs_and_preserves_history_without_foreign_keys() ); let catalog = SqliteCapabilityCatalog::new(pool.clone()); + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config schema"); catalog.ensure_schema().await.unwrap(); catalog .commit_observation(CapabilityObservation::new( @@ -1480,6 +1521,15 @@ async fn concurrent_readers_observe_atomic_server_local_revisions() { .connect_with(options) .await .expect("open concurrent catalog"); + prepare_config_database( + &pool, + DatabaseSource::File { + path: &directory.path().join("concurrent-catalog.db"), + existed_before_open: false, + }, + ) + .await + .expect("prepare config schema"); let catalog = Arc::new(SqliteCapabilityCatalog::new(pool)); catalog.ensure_schema().await.expect("initialize catalog schema"); let cache = Arc::new(DerivedCapabilityCache::new(32, 32)); @@ -1642,60 +1692,3 @@ async fn concurrent_readers_observe_atomic_server_local_revisions() { .expect("load fresh projection"); assert_eq!(fresh_result.as_ref(), &fresh_projection); } - -#[tokio::test] -async fn schema_initialization_rejects_legacy_capability_tables() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("connect legacy schema fixture"); - sqlx::query("CREATE TABLE capability_records (id TEXT PRIMARY KEY)") - .execute(&pool) - .await - .expect("create legacy capability table"); - - let error = SqliteCapabilityCatalog::new(pool) - .ensure_schema() - .await - .expect_err("legacy schemas must require a clean rebuild"); - - assert!(matches!( - error, - mcpmate_capability_store::CatalogError::IncompatibleSchema { .. } - )); -} - -#[tokio::test] -async fn schema_initialization_rejects_an_unknown_capability_epoch() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("connect epoch fixture"); - sqlx::query( - r#" - CREATE TABLE capability_schema_metadata ( - singleton INTEGER PRIMARY KEY, - schema_epoch INTEGER NOT NULL - ) - "#, - ) - .execute(&pool) - .await - .expect("create schema metadata"); - sqlx::query("INSERT INTO capability_schema_metadata (singleton, schema_epoch) VALUES (1, 999)") - .execute(&pool) - .await - .expect("seed unsupported epoch"); - - let error = SqliteCapabilityCatalog::new(pool) - .ensure_schema() - .await - .expect_err("unknown schema epochs must require a clean rebuild"); - - assert!(matches!( - error, - mcpmate_capability_store::CatalogError::IncompatibleSchema { .. } - )); -} diff --git a/backend/crates/mcpmate-capability-store/tests/reconciliation_store_contract.rs b/backend/crates/mcpmate-capability-store/tests/reconciliation_store_contract.rs index 6b232d56..bea174c8 100644 --- a/backend/crates/mcpmate-capability-store/tests/reconciliation_store_contract.rs +++ b/backend/crates/mcpmate-capability-store/tests/reconciliation_store_contract.rs @@ -4,6 +4,7 @@ use chrono::{Duration as ChronoDuration, Utc}; use mcpmate_capability_store::{ ReconciliationJobStatus, SqliteCapabilityCatalog, SqliteSurfaceStore, SurfaceOutboxEvent, SurfaceReconciliationJob, }; +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; use serde_json::json; use sqlx::sqlite::SqlitePoolOptions; @@ -14,6 +15,9 @@ async fn jobs_are_idempotent_leased_recoverable_and_receipted() { .connect("sqlite::memory:?cache=shared") .await .unwrap(); + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config schema"); SqliteCapabilityCatalog::new(pool.clone()) .ensure_schema() .await @@ -107,6 +111,9 @@ async fn outbox_events_are_insert_or_verified_and_delivered_once() { .connect("sqlite::memory:") .await .unwrap(); + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config schema"); SqliteCapabilityCatalog::new(pool.clone()) .ensure_schema() .await diff --git a/backend/crates/mcpmate-capability-store/tests/scale_evidence.rs b/backend/crates/mcpmate-capability-store/tests/scale_evidence.rs index 56b589a0..c807a5f3 100644 --- a/backend/crates/mcpmate-capability-store/tests/scale_evidence.rs +++ b/backend/crates/mcpmate-capability-store/tests/scale_evidence.rs @@ -6,6 +6,7 @@ use mcpmate_capability_store::{ CatalogSnapshot, DeclarationState, DerivedCapabilityCache, InventoryState, KindObservation, SqliteCapabilityCatalog, }; +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; use rmcp::model::{Implementation, InitializeResult, ProtocolVersion, ServerCapabilities, Tool}; use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}; @@ -81,6 +82,15 @@ async fn measures_sqlite_lru_and_serialization_at_configured_scale() { .connect_with(options) .await .expect("open scale database"); + prepare_config_database( + &pool, + DatabaseSource::File { + path: &database_path, + existed_before_open: false, + }, + ) + .await + .expect("prepare config schema"); let catalog = SqliteCapabilityCatalog::new(pool); catalog.ensure_schema().await.expect("create catalog schema"); diff --git a/backend/crates/mcpmate-capability-store/tests/surface_store_contract.rs b/backend/crates/mcpmate-capability-store/tests/surface_store_contract.rs index 5eddf1fa..a392d19b 100644 --- a/backend/crates/mcpmate-capability-store/tests/surface_store_contract.rs +++ b/backend/crates/mcpmate-capability-store/tests/surface_store_contract.rs @@ -5,6 +5,7 @@ use mcpmate_capability_store::{ SurfaceManifestEntryInput, SurfaceProposal, SurfacePublication, SurfaceReviewDecisionDraft, SurfaceReviewFilter, SurfaceReviewItemDraft, SurfaceReviewOwner, }; +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; use rmcp::model::{InitializeResult, Tool}; use serde_json::json; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; @@ -58,6 +59,9 @@ async fn test_store() -> (Pool, SqliteCapabilityCatalog, SqliteSurfaceSt .connect("sqlite::memory:") .await .unwrap(); + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config schema"); let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let store = SqliteSurfaceStore::new(pool.clone()); diff --git a/backend/crates/mcpmate-migrations/Cargo.toml b/backend/crates/mcpmate-migrations/Cargo.toml new file mode 100644 index 00000000..07cdaa90 --- /dev/null +++ b/backend/crates/mcpmate-migrations/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mcpmate-migrations" +version = "0.1.0" +edition = "2024" +description = "Versioned SQLite schema migrations for MCPMate" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +chrono = "0.4" +fs2 = "0.4" +sha2 = "0.10" +sqlx = { version = "0.8.1", features = ["runtime-tokio", "sqlite"] } +tokio = { version = "1.44", features = ["rt"] } + +[dev-dependencies] +tempfile = "3.2" +tokio = { version = "1.44", features = ["macros", "rt-multi-thread"] } diff --git a/backend/crates/mcpmate-migrations/README.md b/backend/crates/mcpmate-migrations/README.md new file mode 100644 index 00000000..469b2c4d --- /dev/null +++ b/backend/crates/mcpmate-migrations/README.md @@ -0,0 +1,70 @@ +# MCPMate Database Migrations + +`mcpmate-migrations` is the sole owner of durable SQLite schema evolution in MCPMate. + +## Why this exists + +Users upgrade MCPMate by running a newer version, not by locating and executing a separate migration tool. When a database needs an upgrade, the product creates a recovery backup, applies the pending steps exactly once, and refuses to start if that work cannot complete safely. + +## Ownership boundary + +- Put durable `CREATE`, `ALTER`, `DROP`, indexes, constraints, and one-time historical data rewrites here. +- Keep ordinary business reads and writes in their domain modules. +- Only the physical database initialization path may invoke `prepare_config_database` or `prepare_audit_database`. +- Domain modules may call the read-only verifier for their database target and validate the tables they consume, but they must not create or upgrade durable schema themselves. +- Each physical database has one ordered migration stream. MCPMate currently has `config` and `audit` targets. + +This boundary includes tests. Database, transaction, schema, revision, and concurrency contracts belong in this crate's integration tests. Domain tests should prepare their fixture through the real migration entrypoint and then exercise the domain path; they must not reproduce schema with test-only DDL. + +## Artifact layout + +Each migration remains visible as a versioned source artifact under `src/migrations//`: + +- `vNNNN_.sql` contains declarative SQLite schema changes. +- `vNNNN_.rs` contains schema-aware or data-preserving transactional steps. +- `/mod.rs` is the ordered registry and the only place that appends a migration to a target stream. +- `runner.rs` owns ledger validation, locking, backup creation, transactions, and the public prepare/verify boundary. + +Do not delete old artifacts after release. A bounded upgrade window is a future product policy, not permission to remove the historical ledger source. If such a window is introduced, keep enough immutable artifacts to support every advertised source version and reject older versions explicitly. + +## Adding a migration + +1. Read this document before designing persistent data changes. +2. Add an immutable, strictly increasing version to the target stream. +3. Use `SqlMigration` for simple DDL. Use a `MigrationStep` implementation for a transactional data transformation that needs schema inspection or Rust control flow. +4. Register the actual SQL or Rust artifact as the checksum source. Do not summarize executable logic into a manually maintained checksum string. +5. Give the migration a stable name. Never edit an already released migration; add a corrective migration instead. +6. Test a fresh database, the relevant legacy structure, rerun idempotence, and failure rollback at the storage-contract layer. +7. If a legacy shape is ambiguous, fail closed. Only rebuild or transform data when the preservation rule is explicit and tested. + +## SQL placement policy + +Durable schema SQL belongs in versioned migration artifacts. Business modules may retain ordinary queries for reads, writes, and explicit business transactions. They may also perform narrow read-only shape checks after migration, but must not contain schema repair SQL such as `CREATE TABLE IF NOT EXISTS`, `ALTER TABLE`, `ensure_column`, or opportunistic data conversion. + +When reviewing a new SQL statement, classify it by purpose: + +- Schema or one-time historical rewrite: migration artifact. +- Repeatable business read/write: owning domain module. +- Migration ledger, backup, or schema inspection: migration runner. +- Test fixture data: test support, inserted only after the real migration chain prepares the database. + +## Runtime guarantees + +The runner records target, version, name, checksum, and application time in `mcpmate_schema_migrations`, with a separate state row proving the applied prefix. It rejects gaps, deleted records, unknown records, modified names or checksums, and mismatched state. All pending steps and ledger updates run in one transaction. + +For file-backed databases, a sidecar lock serializes the pending check, recovery backup, migration transaction, and ledger update across processes. An existing file with pending work receives a unique timestamped `.migration-*.bak` snapshot before changes begin. A fresh file does not create an empty backup. Failed attempts keep their backups; a prepared database does not create another one. In-memory tests use the same migration chain without filesystem backup. + +The current config stream owns server, client, profile authoring, secure-store, and capability-catalog schemas. The audit stream owns audit storage. Legacy structures are accepted only when their preservation rule is explicit in a migration. A migration must stop with a clear error rather than inventing a transformation for data whose meaning or cryptographic material cannot be recovered safely. + +## Validation + +From `backend/`, run: + +```bash +cargo test -p mcpmate-migrations +cargo test -p mcpmate-capability-store --test catalog_contract +cargo test -p mcpmate-secrets +cargo clippy --all-targets --all-features -- -D warnings +``` + +Also run the affected caller tests. A compile check or a successful fresh install does not prove an upgrade: migration work must include a representative legacy database, rollback evidence for unsafe input, rerun idempotence, and a readable recovery backup for an existing file. diff --git a/backend/crates/mcpmate-migrations/src/lib.rs b/backend/crates/mcpmate-migrations/src/lib.rs new file mode 100644 index 00000000..22b5dcaf --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/lib.rs @@ -0,0 +1,9 @@ +//! The sole owner of durable SQLite schema evolution in MCPMate. + +mod migrations; +mod runner; + +pub use runner::{ + DatabaseSource, prepare_audit_database, prepare_config_database, verify_audit_database, + verify_capability_catalog_database, verify_config_database, +}; diff --git a/backend/crates/mcpmate-migrations/src/migrations/audit/mod.rs b/backend/crates/mcpmate-migrations/src/migrations/audit/mod.rs new file mode 100644 index 00000000..60a38c41 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/audit/mod.rs @@ -0,0 +1,7 @@ +use super::Migration; + +const INITIAL_SCHEMA: &str = include_str!("v0001_create_audit_storage.sql"); + +pub(crate) fn all() -> Vec { + vec![Migration::sql(1, "create audit storage", INITIAL_SCHEMA)] +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/audit/v0001_create_audit_storage.sql b/backend/crates/mcpmate-migrations/src/migrations/audit/v0001_create_audit_storage.sql new file mode 100644 index 00000000..30e7964f --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/audit/v0001_create_audit_storage.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS audit_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, action TEXT NOT NULL, status TEXT NOT NULL, + occurred_at_ms INTEGER NOT NULL, actor TEXT, request_id TEXT, client_id TEXT, + profile_id TEXT, server_id TEXT, session_id TEXT, protocol_version TEXT, + http_method TEXT, route TEXT, mcp_method TEXT, target TEXT, direction TEXT, + error_code TEXT, error_message TEXT, detail TEXT, duration_ms INTEGER, + data_json TEXT, task_id TEXT, related_task_id TEXT, progress_token TEXT +); +CREATE INDEX IF NOT EXISTS idx_audit_events_occurred_at ON audit_events (occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_category_action ON audit_events (category, action, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_status ON audit_events (status, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_server_id ON audit_events (server_id, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_profile_id ON audit_events (profile_id, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_client_id ON audit_events (client_id, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_session_id ON audit_events (session_id, occurred_at_ms DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_events_task_id ON audit_events (task_id, occurred_at_ms DESC, id DESC); +CREATE TABLE IF NOT EXISTS audit_policy ( + id INTEGER PRIMARY KEY CHECK (id = 1), policy TEXT NOT NULL, + sweep_interval_secs INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL +); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/mod.rs b/backend/crates/mcpmate-migrations/src/migrations/config/mod.rs new file mode 100644 index 00000000..2a7c0095 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/mod.rs @@ -0,0 +1,34 @@ +mod v0002_add_llm_provider_default; +mod v0004_upgrade_server_columns; +mod v0006_normalize_client_configuration; +mod v0008_validate_secure_store; +mod v0010_create_capability_catalog; + +use super::Migration; +use anyhow::Result; +use sqlx::{Pool, Sqlite}; + +pub(super) const LLM_PROVIDER_SCHEMA: &str = include_str!("v0001_create_llm_provider.sql"); +pub(super) const SERVER_SCHEMA: &str = include_str!("v0003_create_server_configuration.sql"); +pub(super) const CLIENT_SCHEMA: &str = include_str!("v0005_create_client_configuration.sql"); +pub(super) const SECURE_STORE_SCHEMA: &str = include_str!("v0007_create_secure_store.sql"); +pub(super) const PROFILE_SCHEMA: &str = include_str!("v0009_create_profile_authoring.sql"); + +pub(crate) fn all() -> Vec { + vec![ + Migration::sql(1, "create llm provider", LLM_PROVIDER_SCHEMA), + v0002_add_llm_provider_default::migration(), + Migration::sql(3, "create server configuration", SERVER_SCHEMA), + v0004_upgrade_server_columns::migration(), + Migration::sql(5, "create client configuration", CLIENT_SCHEMA), + v0006_normalize_client_configuration::migration(), + Migration::sql(7, "create secure store storage", SECURE_STORE_SCHEMA), + v0008_validate_secure_store::migration(), + Migration::sql(9, "create profile authoring storage", PROFILE_SCHEMA), + v0010_create_capability_catalog::migration(), + ] +} + +pub(crate) async fn verify_capability_catalog(pool: &Pool) -> Result<()> { + v0010_create_capability_catalog::verify(pool).await +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0001_create_llm_provider.sql b/backend/crates/mcpmate-migrations/src/migrations/config/v0001_create_llm_provider.sql new file mode 100644 index 00000000..5383c0e5 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0001_create_llm_provider.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS llm_provider ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, provider_type TEXT NOT NULL, + base_url TEXT NOT NULL, model_id TEXT NOT NULL, secret_alias TEXT, + default_params_json TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0002_add_llm_provider_default.rs b/backend/crates/mcpmate-migrations/src/migrations/config/v0002_add_llm_provider_default.rs new file mode 100644 index 00000000..3e6dde91 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0002_add_llm_provider_default.rs @@ -0,0 +1,36 @@ +use anyhow::Result; +use async_trait::async_trait; +use sqlx::{Sqlite, Transaction}; + +use super::super::{Migration, MigrationStep}; + +pub(super) fn migration() -> Migration { + Migration::rust( + 2, + "add llm provider default flag", + &[include_str!("v0002_add_llm_provider_default.rs")], + AddLlmProviderDefault, + ) +} + +struct AddLlmProviderDefault; + +#[async_trait] +impl MigrationStep for AddLlmProviderDefault { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pragma_table_info('llm_provider') WHERE name = 'is_default')", + ) + .fetch_one(&mut **transaction) + .await?; + if !exists { + sqlx::query("ALTER TABLE llm_provider ADD COLUMN is_default BOOLEAN NOT NULL DEFAULT 0") + .execute(&mut **transaction) + .await?; + } + Ok(()) + } +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0003_create_server_configuration.sql b/backend/crates/mcpmate-migrations/src/migrations/config/v0003_create_server_configuration.sql new file mode 100644 index 00000000..977f1ca5 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0003_create_server_configuration.sql @@ -0,0 +1,56 @@ +CREATE TABLE IF NOT EXISTS server_config ( + id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, + server_type TEXT NOT NULL CHECK (server_type IN ('stdio', 'sse', 'streamable_http')), + command TEXT, url TEXT, source TEXT, enabled BOOLEAN NOT NULL DEFAULT 1, + unify_direct_exposure_eligible BOOLEAN NOT NULL DEFAULT 0, + pending_import BOOLEAN NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS server_args ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, + arg_index INTEGER NOT NULL, arg_value TEXT NOT NULL, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, + UNIQUE(server_id, arg_index) +); +CREATE TABLE IF NOT EXISTS server_env ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, + env_key TEXT NOT NULL, env_value TEXT NOT NULL, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, + UNIQUE(server_id, env_key) +); +CREATE TABLE IF NOT EXISTS server_headers ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL, header_key TEXT NOT NULL, + header_value TEXT NOT NULL, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, + UNIQUE(server_id, header_key) +); +CREATE TABLE IF NOT EXISTS server_meta ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, + author TEXT, category TEXT, description TEXT, extras_json TEXT, icons_json TEXT, + protocol_version TEXT, rating INTEGER, recommended_scenario TEXT, registry_meta_json TEXT, + registry_version TEXT, repository TEXT, upstream_name TEXT, upstream_title TEXT, + server_version TEXT, website TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id) +); +CREATE TABLE IF NOT EXISTS server_namespace_issue ( + server_id TEXT PRIMARY KEY, issue_kind TEXT NOT NULL, capability_kind TEXT, + external_identifier TEXT, upstream_value TEXT, conflicting_server_id TEXT, + conflicting_upstream_value TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, + FOREIGN KEY (conflicting_server_id) REFERENCES server_config (id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS server_oauth_config ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL UNIQUE, authorization_endpoint TEXT NOT NULL, + token_endpoint TEXT NOT NULL, client_id TEXT NOT NULL, client_secret TEXT, scopes TEXT, + redirect_uri TEXT NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS server_oauth_tokens ( + id TEXT PRIMARY KEY, server_id TEXT NOT NULL UNIQUE, access_token TEXT NOT NULL, + refresh_token TEXT, token_type TEXT NOT NULL DEFAULT 'bearer', expires_at TEXT, scope TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE +); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0004_upgrade_server_columns.rs b/backend/crates/mcpmate-migrations/src/migrations/config/v0004_upgrade_server_columns.rs new file mode 100644 index 00000000..6afa045c --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0004_upgrade_server_columns.rs @@ -0,0 +1,70 @@ +use anyhow::{Context, Result}; +use async_trait::async_trait; +use sqlx::{Sqlite, Transaction}; + +use super::super::{Migration, MigrationStep}; + +pub(super) fn migration() -> Migration { + Migration::rust( + 4, + "upgrade server configuration columns", + &[include_str!("v0004_upgrade_server_columns.rs")], + UpgradeServerColumns, + ) +} + +struct UpgradeServerColumns; + +#[async_trait] +impl MigrationStep for UpgradeServerColumns { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + ensure_columns( + transaction, + "server_config", + &[ + ("pending_import", "BOOLEAN NOT NULL DEFAULT 0"), + ("unify_direct_exposure_eligible", "BOOLEAN NOT NULL DEFAULT 0"), + ("source", "TEXT"), + ], + ) + .await?; + ensure_columns( + transaction, + "server_meta", + &[ + ("extras_json", "TEXT"), + ("icons_json", "TEXT"), + ("protocol_version", "TEXT"), + ("registry_meta_json", "TEXT"), + ("registry_version", "TEXT"), + ("upstream_name", "TEXT"), + ("upstream_title", "TEXT"), + ("server_version", "TEXT"), + ], + ) + .await + } +} + +async fn ensure_columns( + transaction: &mut Transaction<'_, Sqlite>, + table: &str, + columns: &[(&str, &str)], +) -> Result<()> { + let existing: Vec = sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")) + .fetch_all(&mut **transaction) + .await + .with_context(|| format!("inspect {table} columns"))?; + for (column, definition) in columns { + if !existing.iter().any(|existing| existing == column) { + sqlx::query(&format!("ALTER TABLE {table} ADD COLUMN {column} {definition}")) + .execute(&mut **transaction) + .await + .with_context(|| format!("add {table}.{column}"))?; + } + } + Ok(()) +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0005_create_client_configuration.sql b/backend/crates/mcpmate-migrations/src/migrations/config/v0005_create_client_configuration.sql new file mode 100644 index 00000000..7f16351e --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0005_create_client_configuration.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS client (id TEXT PRIMARY KEY, name TEXT NOT NULL, display_name TEXT, identifier TEXT NOT NULL UNIQUE, config_path TEXT, config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), transport TEXT NOT NULL DEFAULT 'auto' CHECK (transport IN ('auto', 'sse', 'stdio', 'streamable_http')), client_version TEXT, backup_policy TEXT NOT NULL DEFAULT 'keep_n' CHECK (backup_policy IN ('keep_last', 'keep_n', 'off')), backup_limit INTEGER DEFAULT 5, capability_source TEXT NOT NULL DEFAULT 'activated' CHECK (capability_source IN ('activated', 'profiles', 'custom')), unify_route_mode TEXT NOT NULL DEFAULT 'broker_only' CHECK (unify_route_mode IN ('broker_only', 'server_level', 'capability_level')), governance_kind TEXT NOT NULL DEFAULT 'passive' CHECK (governance_kind IN ('passive', 'active')), connection_mode TEXT NOT NULL DEFAULT 'local_config_detected' CHECK (connection_mode IN ('local_config_detected', 'manual')), registration_origin TEXT NOT NULL DEFAULT 'manual' CHECK (registration_origin IN ('manual', 'config_detection', 'runtime_initialize')), runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), template_identifier TEXT, selected_profile_ids TEXT, custom_profile_id TEXT, approval_status TEXT NOT NULL DEFAULT 'approved' CHECK (approval_status IN ('pending', 'approved', 'suspended')), template_id TEXT, template_version TEXT, approval_metadata TEXT, config_format TEXT, protocol_revision TEXT, container_type TEXT, container_keys TEXT, storage_kind TEXT, storage_adapter TEXT, storage_path_strategy TEXT, merge_strategy TEXT, keep_original_config INTEGER, managed_source TEXT, transports TEXT, config_file_parse TEXT, attachment_state TEXT NOT NULL DEFAULT 'not_applicable' CHECK (attachment_state IN ('attached', 'detached', 'not_applicable')), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE IF NOT EXISTS client_template_runtime (identifier TEXT PRIMARY KEY, payload_json TEXT NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE IF NOT EXISTS client_writeback_policy (client_identifier TEXT PRIMARY KEY, merge_strategy TEXT NOT NULL CHECK (merge_strategy IN ('replace', 'deep_merge')), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (client_identifier) REFERENCES client(identifier) ON DELETE CASCADE); +CREATE TABLE IF NOT EXISTS client_runtime_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL CHECK (key != 'default_config_mode' OR value IN ('unify', 'hosted', 'transparent'))); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0006_normalize_client_configuration.rs b/backend/crates/mcpmate-migrations/src/migrations/config/v0006_normalize_client_configuration.rs new file mode 100644 index 00000000..b7ad21c3 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0006_normalize_client_configuration.rs @@ -0,0 +1,312 @@ +use anyhow::{Context, Result}; +use async_trait::async_trait; +use sqlx::{Sqlite, Transaction}; + +use super::{ + super::{Migration, MigrationStep}, + CLIENT_SCHEMA, +}; + +const CURRENT_CLIENT_COLUMNS: &[&str] = &[ + "id", + "name", + "display_name", + "identifier", + "config_path", + "config_mode", + "transport", + "client_version", + "backup_policy", + "backup_limit", + "capability_source", + "unify_route_mode", + "governance_kind", + "connection_mode", + "registration_origin", + "runtime_observed", + "template_identifier", + "selected_profile_ids", + "custom_profile_id", + "approval_status", + "template_id", + "template_version", + "approval_metadata", + "config_format", + "protocol_revision", + "container_type", + "container_keys", + "storage_kind", + "storage_adapter", + "storage_path_strategy", + "merge_strategy", + "keep_original_config", + "managed_source", + "transports", + "config_file_parse", + "attachment_state", + "created_at", + "updated_at", +]; + +pub(super) fn migration() -> Migration { + Migration::rust( + 6, + "normalize legacy client configuration", + &[include_str!("v0006_normalize_client_configuration.rs"), CLIENT_SCHEMA], + NormalizeClientSchema, + ) +} + +struct NormalizeClientSchema; + +#[async_trait] +impl MigrationStep for NormalizeClientSchema { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + let columns: Vec = sqlx::query_scalar("SELECT name FROM pragma_table_info('client')") + .fetch_all(&mut **transaction) + .await + .context("inspect client schema")?; + let create_sql: Option = + sqlx::query_scalar("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'client'") + .fetch_optional(&mut **transaction) + .await + .context("read client schema SQL")?; + let uses_current_connection_contract = create_sql + .as_deref() + .is_some_and(|sql| sql.contains("connection_mode IN ('local_config_detected', 'manual')")); + let current = create_sql.as_deref().is_some_and(|sql| { + sql.contains("streamable_http") + && uses_current_connection_contract + && CURRENT_CLIENT_COLUMNS + .iter() + .all(|required| columns.iter().any(|column| column == required)) + }); + if !current { + let direct_exposure_refs = snapshot_table_if_exists( + transaction, + "direct_exposure_refs", + "direct_exposure_refs_migration_snapshot", + ) + .await?; + let direct_exposure_servers = snapshot_table_if_exists( + transaction, + "direct_exposure_servers", + "direct_exposure_servers_migration_snapshot", + ) + .await?; + let client_table_sql = CLIENT_SCHEMA + .split(";\n") + .next() + .expect("client schema starts with the client table") + .replace( + "CREATE TABLE IF NOT EXISTS client", + "CREATE TABLE client_migration_current", + ); + sqlx::query(&client_table_sql) + .execute(&mut **transaction) + .await + .context("create current client replacement table")?; + let expr = |column: &str, fallback: &str| { + if columns.iter().any(|existing| existing == column) { + column.to_string() + } else { + fallback.to_string() + } + }; + let display_name = if columns.iter().any(|column| column == "display_name") { + "COALESCE(NULLIF(display_name, ''), name)".to_string() + } else { + "name".to_string() + }; + let config_mode = if columns.iter().any(|column| column == "config_mode") { + "NULLIF(config_mode, '')".to_string() + } else { + "NULL".to_string() + }; + let transport = if columns.iter().any(|column| column == "transport") { + "COALESCE(NULLIF(transport, ''), 'auto')".to_string() + } else { + "'auto'".to_string() + }; + let source = if columns.iter().any(|column| column == "capability_source") { + "COALESCE(NULLIF(capability_source, ''), 'activated')".to_string() + } else { + "'activated'".to_string() + }; + let connection_mode = if uses_current_connection_contract { + "connection_mode".to_string() + } else if columns.iter().any(|column| column == "config_path") { + "CASE WHEN config_path IS NOT NULL AND TRIM(config_path) <> '' THEN 'local_config_detected' ELSE 'manual' END".to_string() + } else { + "'manual'".to_string() + }; + let registration = if uses_current_connection_contract + && columns.iter().any(|column| column == "registration_origin") + { + "registration_origin".to_string() + } else if columns.iter().any(|column| column == "connection_mode") { + if columns.iter().any(|column| column == "config_path") { + "CASE WHEN connection_mode = 'remote_http' THEN 'runtime_initialize' WHEN config_path IS NOT NULL AND TRIM(config_path) <> '' THEN 'config_detection' ELSE 'manual' END".to_string() + } else { + "CASE WHEN connection_mode = 'remote_http' THEN 'runtime_initialize' ELSE 'manual' END".to_string() + } + } else { + "'manual'".to_string() + }; + let observed = + if uses_current_connection_contract && columns.iter().any(|column| column == "runtime_observed") { + "runtime_observed".to_string() + } else if columns.iter().any(|column| column == "connection_mode") { + "CASE WHEN connection_mode = 'remote_http' THEN 1 ELSE 0 END".to_string() + } else { + "0".to_string() + }; + let transports = if columns.iter().any(|column| column == "transports") { + "transports".to_string() + } else if columns.iter().any(|column| column == "format_rules") { + "format_rules".to_string() + } else { + "NULL".to_string() + }; + let select = vec![ + expr("id", "NULL"), + expr("name", "''"), + display_name, + expr("identifier", "''"), + expr("config_path", "NULL"), + config_mode, + transport, + expr("client_version", "NULL"), + format!("COALESCE({}, 'keep_n')", expr("backup_policy", "NULL")), + expr("backup_limit", "5"), + source, + expr("unify_route_mode", "'broker_only'"), + expr("governance_kind", "'passive'"), + connection_mode, + registration, + observed, + expr("template_identifier", "identifier"), + expr("selected_profile_ids", "NULL"), + expr("custom_profile_id", "NULL"), + expr("approval_status", "'approved'"), + expr("template_id", "NULL"), + expr("template_version", "NULL"), + expr("approval_metadata", "NULL"), + expr("config_format", "NULL"), + expr("protocol_revision", "NULL"), + expr("container_type", "NULL"), + expr("container_keys", "NULL"), + expr("storage_kind", "NULL"), + expr("storage_adapter", "NULL"), + expr("storage_path_strategy", "NULL"), + expr("merge_strategy", "NULL"), + expr("keep_original_config", "NULL"), + expr("managed_source", "NULL"), + transports, + expr("config_file_parse", "NULL"), + expr("attachment_state", "'not_applicable'"), + expr("created_at", "CURRENT_TIMESTAMP"), + expr("updated_at", "CURRENT_TIMESTAMP"), + ] + .join(", "); + sqlx::query(&format!( + "INSERT INTO client_migration_current SELECT {select} FROM client" + )) + .execute(&mut **transaction) + .await + .context("copy legacy client rows")?; + sqlx::query( + "CREATE TABLE client_writeback_policy_migration_current (client_identifier TEXT PRIMARY KEY, merge_strategy TEXT NOT NULL CHECK (merge_strategy IN ('replace', 'deep_merge')), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (client_identifier) REFERENCES client_migration_current(identifier) ON DELETE CASCADE)", + ) + .execute(&mut **transaction) + .await + .context("create current client writeback policy replacement table")?; + sqlx::query( + "INSERT INTO client_writeback_policy_migration_current SELECT client_identifier, merge_strategy, created_at, updated_at FROM client_writeback_policy", + ) + .execute(&mut **transaction) + .await + .context("copy client writeback policies")?; + sqlx::query("DROP TABLE client_writeback_policy") + .execute(&mut **transaction) + .await + .context("drop legacy client writeback policy table")?; + sqlx::query("DROP TABLE client") + .execute(&mut **transaction) + .await + .context("drop legacy client table")?; + sqlx::query("ALTER TABLE client_migration_current RENAME TO client") + .execute(&mut **transaction) + .await + .context("rename rebuilt client table")?; + sqlx::query("ALTER TABLE client_writeback_policy_migration_current RENAME TO client_writeback_policy") + .execute(&mut **transaction) + .await + .context("rename rebuilt client writeback policy table")?; + restore_table_snapshot( + transaction, + "direct_exposure_refs", + "direct_exposure_refs_migration_snapshot", + direct_exposure_refs, + ) + .await?; + restore_table_snapshot( + transaction, + "direct_exposure_servers", + "direct_exposure_servers_migration_snapshot", + direct_exposure_servers, + ) + .await?; + } + sqlx::query( + "INSERT OR IGNORE INTO client_runtime_settings (key, value) VALUES ('default_config_mode', 'unify')", + ) + .execute(&mut **transaction) + .await + .context("seed default client config mode")?; + Ok(()) + } +} + +async fn snapshot_table_if_exists( + transaction: &mut Transaction<'_, Sqlite>, + table: &str, + snapshot: &str, +) -> Result { + let exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(table) + .fetch_one(&mut **transaction) + .await + .with_context(|| format!("inspect {table} before client migration"))?; + if exists { + sqlx::query(&format!("CREATE TEMP TABLE {snapshot} AS SELECT * FROM {table}")) + .execute(&mut **transaction) + .await + .with_context(|| format!("snapshot {table} before client migration"))?; + } + Ok(exists) +} + +async fn restore_table_snapshot( + transaction: &mut Transaction<'_, Sqlite>, + table: &str, + snapshot: &str, + snapshot_exists: bool, +) -> Result<()> { + if !snapshot_exists { + return Ok(()); + } + sqlx::query(&format!("INSERT INTO {table} SELECT * FROM {snapshot}")) + .execute(&mut **transaction) + .await + .with_context(|| format!("restore {table} after client migration"))?; + sqlx::query(&format!("DROP TABLE {snapshot}")) + .execute(&mut **transaction) + .await + .with_context(|| format!("drop {table} migration snapshot"))?; + Ok(()) +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0007_create_secure_store.sql b/backend/crates/mcpmate-migrations/src/migrations/config/v0007_create_secure_store.sql new file mode 100644 index 00000000..b0d3e9ad --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0007_create_secure_store.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS secure_store_secrets (alias TEXT PRIMARY KEY, kind TEXT NOT NULL, label TEXT, origin_server_id TEXT, origin_server_name TEXT, origin_server_kind TEXT, origin_source TEXT, origin_field_group TEXT, origin_field_key TEXT, origin_field_index INTEGER, origin_field_path TEXT, provider_id TEXT NOT NULL, provider_kind TEXT NOT NULL, version INTEGER NOT NULL, key_nonce TEXT NOT NULL, encrypted_key TEXT NOT NULL, nonce TEXT NOT NULL, encrypted_value TEXT NOT NULL, key_wrap_alg TEXT NOT NULL DEFAULT 'AES-256-GCM', encryption_alg TEXT NOT NULL DEFAULT 'AES-256-GCM', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE IF NOT EXISTS secure_store_usages (id TEXT PRIMARY KEY, alias TEXT NOT NULL, server_id TEXT NOT NULL, location_kind TEXT NOT NULL, location_name TEXT, location_index INTEGER, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (alias) REFERENCES secure_store_secrets (alias) ON DELETE CASCADE, UNIQUE(alias, server_id, location_kind, location_name, location_index)); +CREATE TABLE IF NOT EXISTS secure_store_password_config (id INTEGER PRIMARY KEY CHECK (id = 1), password_hash TEXT NOT NULL, hash_salt TEXT NOT NULL, hash_iterations INTEGER NOT NULL DEFAULT 600000, protection_scope TEXT NOT NULL DEFAULT '["startup"]', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE IF NOT EXISTS secure_store_provider_config (id INTEGER PRIMARY KEY CHECK (id = 1), provider_mode TEXT NOT NULL DEFAULT 'operating_system', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0008_validate_secure_store.rs b/backend/crates/mcpmate-migrations/src/migrations/config/v0008_validate_secure_store.rs new file mode 100644 index 00000000..b5314341 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0008_validate_secure_store.rs @@ -0,0 +1,184 @@ +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use sqlx::{Sqlite, Transaction}; + +use super::{ + super::{Migration, MigrationStep}, + SECURE_STORE_SCHEMA, +}; + +pub(super) fn migration() -> Migration { + Migration::rust( + 8, + "validate legacy secure store storage", + &[include_str!("v0008_validate_secure_store.rs"), SECURE_STORE_SCHEMA], + ValidateSecureStoreSchema, + ) +} + +struct ValidateSecureStoreSchema; + +#[async_trait] +impl MigrationStep for ValidateSecureStoreSchema { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + if secure_store_schema_is_current(transaction).await? { + return Ok(()); + } + let count = secure_store_record_count(transaction).await?; + if count != 0 { + bail!( + "outdated secure store schema contains {count} legacy record(s); it cannot be safely upgraded without preserving its security constraints" + ); + } + rebuild_empty_secure_store(transaction).await?; + Ok(()) + } +} + +const SECURE_STORE_TABLES: &[(&str, &[&str])] = &[ + ( + "secure_store_secrets", + &[ + "alias|TEXT|0||1", + "kind|TEXT|1||0", + "label|TEXT|0||0", + "origin_server_id|TEXT|0||0", + "origin_server_name|TEXT|0||0", + "origin_server_kind|TEXT|0||0", + "origin_source|TEXT|0||0", + "origin_field_group|TEXT|0||0", + "origin_field_key|TEXT|0||0", + "origin_field_index|INTEGER|0||0", + "origin_field_path|TEXT|0||0", + "provider_id|TEXT|1||0", + "provider_kind|TEXT|1||0", + "version|INTEGER|1||0", + "key_nonce|TEXT|1||0", + "encrypted_key|TEXT|1||0", + "nonce|TEXT|1||0", + "encrypted_value|TEXT|1||0", + "key_wrap_alg|TEXT|1|'AES-256-GCM'|0", + "encryption_alg|TEXT|1|'AES-256-GCM'|0", + "created_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + "updated_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + ], + ), + ( + "secure_store_usages", + &[ + "id|TEXT|0||1", + "alias|TEXT|1||0", + "server_id|TEXT|1||0", + "location_kind|TEXT|1||0", + "location_name|TEXT|0||0", + "location_index|INTEGER|0||0", + "created_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + "updated_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + ], + ), + ( + "secure_store_password_config", + &[ + "id|INTEGER|0||1", + "password_hash|TEXT|1||0", + "hash_salt|TEXT|1||0", + "hash_iterations|INTEGER|1|600000|0", + "protection_scope|TEXT|1|'[\"startup\"]'|0", + "created_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + "updated_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + ], + ), + ( + "secure_store_provider_config", + &[ + "id|INTEGER|0||1", + "provider_mode|TEXT|1|'operating_system'|0", + "created_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + "updated_at|TIMESTAMP|1|CURRENT_TIMESTAMP|0", + ], + ), +]; + +async fn secure_store_schema_is_current(transaction: &mut Transaction<'_, Sqlite>) -> Result { + for (table, expected) in SECURE_STORE_TABLES { + let actual: Vec = sqlx::query_scalar(&format!( + "SELECT name || '|' || upper(type) || '|' || \"notnull\" || '|' || COALESCE(dflt_value, '') || '|' || pk FROM pragma_table_info('{table}') ORDER BY cid" + )) + .fetch_all(&mut **transaction) + .await + .with_context(|| format!("inspect {table} schema"))?; + if !actual.iter().map(String::as_str).eq(expected.iter().copied()) { + return Ok(false); + } + } + + let usage_foreign_key: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM pragma_foreign_key_list('secure_store_usages') + WHERE \"table\" = 'secure_store_secrets' + AND \"from\" = 'alias' + AND \"to\" = 'alias' + AND on_delete = 'CASCADE' + )", + ) + .fetch_one(&mut **transaction) + .await + .context("inspect secure store usage foreign key")?; + let usage_identity_unique: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM pragma_index_list('secure_store_usages') AS indexes + WHERE indexes.\"unique\" = 1 + AND (SELECT group_concat(name, ',') FROM pragma_index_info(indexes.name)) + = 'alias,server_id,location_kind,location_name,location_index' + )", + ) + .fetch_one(&mut **transaction) + .await + .context("inspect secure store usage identity constraint")?; + for table in ["secure_store_password_config", "secure_store_provider_config"] { + let create_sql: String = sqlx::query_scalar("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?") + .bind(table) + .fetch_one(&mut **transaction) + .await + .with_context(|| format!("inspect {table} constraints"))?; + if !create_sql.to_ascii_lowercase().contains("check (id = 1)") { + return Ok(false); + } + } + Ok(usage_foreign_key && usage_identity_unique) +} + +async fn secure_store_record_count(transaction: &mut Transaction<'_, Sqlite>) -> Result { + let mut count = 0; + for (table, _) in SECURE_STORE_TABLES { + count += sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*) FROM {table}")) + .fetch_one(&mut **transaction) + .await + .with_context(|| format!("count legacy {table} records"))?; + } + Ok(count) +} + +async fn rebuild_empty_secure_store(transaction: &mut Transaction<'_, Sqlite>) -> Result<()> { + for table in [ + "secure_store_usages", + "secure_store_password_config", + "secure_store_provider_config", + "secure_store_secrets", + ] { + sqlx::query(&format!("DROP TABLE {table}")) + .execute(&mut **transaction) + .await + .with_context(|| format!("drop empty legacy {table}"))?; + } + for statement in SECURE_STORE_SCHEMA.split(";\n").filter(|sql| !sql.trim().is_empty()) { + sqlx::query(statement) + .execute(&mut **transaction) + .await + .context("recreate current secure store schema")?; + } + Ok(()) +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0009_create_profile_authoring.sql b/backend/crates/mcpmate-migrations/src/migrations/config/v0009_create_profile_authoring.sql new file mode 100644 index 00000000..ca11f37d --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0009_create_profile_authoring.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS profile (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, description TEXT, type TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'user', multi_select BOOLEAN NOT NULL DEFAULT 0, priority INTEGER NOT NULL DEFAULT 0, is_active BOOLEAN NOT NULL DEFAULT 0, is_default BOOLEAN NOT NULL DEFAULT 0, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE IF NOT EXISTS profile_server_relationships (profile_id TEXT NOT NULL, server_id TEXT NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, new_ref_policy TEXT NOT NULL CHECK (new_ref_policy IN ('follow', 'review')), FOREIGN KEY (profile_id) REFERENCES profile (id) ON DELETE CASCADE, PRIMARY KEY(profile_id, server_id)); +CREATE TABLE IF NOT EXISTS server_tools (id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, tool_name TEXT NOT NULL, unique_name TEXT NOT NULL, description TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id, tool_name), UNIQUE(unique_name)); +CREATE INDEX IF NOT EXISTS idx_server_tools_lookup ON server_tools(server_id, tool_name); +CREATE INDEX IF NOT EXISTS idx_server_tools_unique_name ON server_tools(unique_name); +CREATE INDEX IF NOT EXISTS idx_server_tools_server_name ON server_tools(server_name); +CREATE TABLE IF NOT EXISTS server_prompts (id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, prompt_name TEXT NOT NULL, unique_name TEXT NOT NULL, description TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id, prompt_name), UNIQUE(unique_name)); +CREATE INDEX IF NOT EXISTS idx_server_prompts_lookup ON server_prompts(server_id, prompt_name); +CREATE INDEX IF NOT EXISTS idx_server_prompts_unique_name ON server_prompts(unique_name); +CREATE INDEX IF NOT EXISTS idx_server_prompts_server_name ON server_prompts(server_name); +CREATE TABLE IF NOT EXISTS server_resources (id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, resource_uri TEXT NOT NULL, unique_uri TEXT NOT NULL, name TEXT, description TEXT, mime_type TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id, resource_uri), UNIQUE(unique_uri)); +CREATE INDEX IF NOT EXISTS idx_server_resources_lookup ON server_resources(server_id, resource_uri); +CREATE INDEX IF NOT EXISTS idx_server_resources_unique_uri ON server_resources(unique_uri); +CREATE INDEX IF NOT EXISTS idx_server_resources_server_name ON server_resources(server_name); +CREATE TABLE IF NOT EXISTS server_resource_templates (id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, uri_template TEXT NOT NULL, unique_name TEXT NOT NULL, route_uri TEXT, name TEXT NOT NULL, description TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id, uri_template), UNIQUE(unique_name), UNIQUE(route_uri)); +CREATE INDEX IF NOT EXISTS idx_server_resource_templates_lookup ON server_resource_templates(server_id, uri_template); +CREATE INDEX IF NOT EXISTS idx_server_resource_templates_unique_name ON server_resource_templates(unique_name); +CREATE INDEX IF NOT EXISTS idx_server_resource_templates_route_uri ON server_resource_templates(route_uri); +CREATE INDEX IF NOT EXISTS idx_server_resource_templates_server_name ON server_resource_templates(server_name); +CREATE TABLE IF NOT EXISTS server_issued_resources (id TEXT PRIMARY KEY, server_id TEXT NOT NULL, server_name TEXT NOT NULL, resource_uri TEXT NOT NULL, unique_uri TEXT NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, last_seen_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, UNIQUE(server_id, resource_uri), UNIQUE(unique_uri)); +CREATE INDEX IF NOT EXISTS idx_server_issued_resources_lookup ON server_issued_resources(server_id, resource_uri); +CREATE INDEX IF NOT EXISTS idx_server_issued_resources_unique_uri ON server_issued_resources(unique_uri); +CREATE TABLE IF NOT EXISTS profile_capability_refs (profile_id TEXT NOT NULL, ref_id TEXT NOT NULL, enabled BOOLEAN NOT NULL, FOREIGN KEY (profile_id) REFERENCES profile (id) ON DELETE CASCADE, FOREIGN KEY (ref_id) REFERENCES capability_refs (ref_id) ON DELETE CASCADE, PRIMARY KEY(profile_id, ref_id)); +CREATE INDEX IF NOT EXISTS idx_profile_capability_refs_ref ON profile_capability_refs(ref_id); +CREATE TABLE IF NOT EXISTS direct_exposure_refs (consumer_id TEXT NOT NULL, ref_id TEXT NOT NULL, enabled BOOLEAN NOT NULL, FOREIGN KEY (consumer_id) REFERENCES client (identifier) ON DELETE CASCADE, FOREIGN KEY (ref_id) REFERENCES capability_refs (ref_id) ON DELETE CASCADE, PRIMARY KEY(consumer_id, ref_id)); +CREATE INDEX IF NOT EXISTS idx_direct_exposure_refs_ref ON direct_exposure_refs(ref_id); +CREATE TABLE IF NOT EXISTS direct_exposure_servers (consumer_id TEXT NOT NULL, server_id TEXT NOT NULL, new_ref_policy TEXT NOT NULL CHECK (new_ref_policy IN ('follow', 'review')), FOREIGN KEY (consumer_id) REFERENCES client (identifier) ON DELETE CASCADE, PRIMARY KEY(consumer_id, server_id)); diff --git a/backend/crates/mcpmate-migrations/src/migrations/config/v0010_create_capability_catalog.rs b/backend/crates/mcpmate-migrations/src/migrations/config/v0010_create_capability_catalog.rs new file mode 100644 index 00000000..5c5eeea4 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/config/v0010_create_capability_catalog.rs @@ -0,0 +1,562 @@ +use anyhow::{Result, bail}; +use async_trait::async_trait; +use sqlx::{Connection, Pool, Sqlite, SqliteConnection, Transaction}; + +use super::super::{Migration, MigrationStep}; + +const CAPABILITY_SCHEMA_EPOCH: i64 = 4; +const LEGACY_CAPABILITY_TABLES: &[&str] = &[ + "capability_records", + "profile_tool", + "profile_prompt", + "profile_resource", + "profile_resource_template", + "profile_server", +]; +const CURRENT_CAPABILITY_TABLES: &[&str] = &[ + "capability_server_snapshots", + "capability_kind_states", + "capability_refs", + "capability_versions", + "capability_ref_current", + "surface_manifests", + "surface_manifest_entries", + "surface_proposals", + "surface_review_items", + "surface_review_decisions", + "surface_proposal_review_items", + "surface_review_owners", + "surface_publications", + "consumer_surface_bindings", + "consumer_surface_generations", + "surface_reconciliation_jobs", + "surface_outbox_events", + "capability_change_events", + "configuration_mode_transitions", +]; + +pub(super) fn migration() -> Migration { + Migration::rust( + 10, + "create capability catalog", + &[include_str!("v0010_create_capability_catalog.rs")], + CreateCapabilityCatalog, + ) +} + +struct CreateCapabilityCatalog; + +pub(super) async fn verify(pool: &Pool) -> Result<()> { + let mut transaction = pool.begin().await?; + validate_current_epoch_schema(&mut transaction).await +} + +#[async_trait] +impl MigrationStep for CreateCapabilityCatalog { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + validate_legacy_catalog_schema(transaction).await?; + apply_catalog_schema(transaction).await + } +} + +async fn validate_legacy_catalog_schema(transaction: &mut Transaction<'_, Sqlite>) -> Result<()> { + let metadata_exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'capability_schema_metadata')", + ) + .fetch_one(&mut **transaction) + .await?; + if metadata_exists { + let epoch: Option = + sqlx::query_scalar("SELECT schema_epoch FROM capability_schema_metadata WHERE singleton = 1") + .fetch_optional(&mut **transaction) + .await?; + match epoch { + Some(CAPABILITY_SCHEMA_EPOCH) => { + validate_current_epoch_schema(transaction).await?; + return Ok(()); + } + Some(epoch) => bail!( + "capability schema epoch {epoch} is not supported; clean rebuild is required for epoch {CAPABILITY_SCHEMA_EPOCH}" + ), + None => bail!("capability_schema_metadata is missing its singleton epoch row"), + } + } + + let existing_tables: Vec = + sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") + .fetch_all(&mut **transaction) + .await?; + let incompatible_tables = existing_tables + .iter() + .filter(|table| { + LEGACY_CAPABILITY_TABLES.contains(&table.as_str()) || CURRENT_CAPABILITY_TABLES.contains(&table.as_str()) + }) + .cloned() + .collect::>(); + if !incompatible_tables.is_empty() { + bail!( + "database contains unversioned capability tables [{}]; clean rebuild is required", + incompatible_tables.join(", ") + ); + } + Ok(()) +} + +async fn validate_current_epoch_schema(transaction: &mut Transaction<'_, Sqlite>) -> Result<()> { + let actual = schema_contract(transaction).await?; + let mut reference_connection = SqliteConnection::connect("sqlite::memory:").await?; + let mut reference_transaction = reference_connection.begin().await?; + apply_catalog_schema(&mut reference_transaction).await?; + let expected = schema_contract(&mut reference_transaction).await?; + if actual != expected { + bail!( + "incomplete capability schema epoch {CAPABILITY_SCHEMA_EPOCH}; current catalog structure does not match the versioned contract" + ); + } + Ok(()) +} + +async fn schema_contract(transaction: &mut Transaction<'_, Sqlite>) -> Result> { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT type, name, tbl_name, sql + FROM sqlite_master + WHERE sql IS NOT NULL + ORDER BY type, name", + ) + .fetch_all(&mut **transaction) + .await?; + Ok(rows + .into_iter() + .filter(|(_, _, table, _)| CURRENT_CAPABILITY_TABLES.contains(&table.as_str())) + .map(|(kind, name, table, sql)| (kind, name, table, normalize_schema_sql(&sql))) + .collect()) +} + +fn normalize_schema_sql(sql: &str) -> String { + sql.split_whitespace() + .map(str::to_ascii_lowercase) + .collect::>() + .join(" ") +} + +async fn apply_catalog_schema(transaction: &mut Transaction<'_, Sqlite>) -> Result<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_server_snapshots ( + server_id TEXT PRIMARY KEY, + server_name TEXT NOT NULL, + config_fingerprint TEXT NOT NULL, + record_format_version INTEGER NOT NULL, + catalog_revision INTEGER NOT NULL, + snapshot_state TEXT NOT NULL, + initialize_payload TEXT NOT NULL, + observed_at TEXT NOT NULL, + committed_at TEXT NOT NULL, + last_error TEXT + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_kind_states ( + server_id TEXT NOT NULL, + position INTEGER NOT NULL, + kind TEXT NOT NULL, + declaration_state TEXT NOT NULL, + inventory_state TEXT NOT NULL, + error TEXT, + failure_kind TEXT, + timeout_ms INTEGER, + catalog_revision INTEGER NOT NULL, + observed_at TEXT NOT NULL, + PRIMARY KEY (server_id, kind), + FOREIGN KEY (server_id) REFERENCES capability_server_snapshots(server_id) ON DELETE CASCADE + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_refs ( + ref_id TEXT PRIMARY KEY, + server_id TEXT NOT NULL, + kind TEXT NOT NULL, + origin_key TEXT NOT NULL, + state TEXT NOT NULL, + state_generation INTEGER NOT NULL, + first_observed_revision INTEGER NOT NULL, + last_observed_revision INTEGER NOT NULL, + FOREIGN KEY (server_id) REFERENCES capability_server_snapshots(server_id) ON DELETE CASCADE, + UNIQUE (server_id, kind, origin_key) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query("CREATE INDEX IF NOT EXISTS idx_capability_refs_server_kind ON capability_refs(server_id, kind)") + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_versions ( + capability_id TEXT PRIMARY KEY, + ref_id TEXT NOT NULL, + canonical_record BLOB NOT NULL, + source_payload BLOB NOT NULL, + effective_payload BLOB NOT NULL, + record_format TEXT NOT NULL, + first_observed_revision INTEGER NOT NULL, + FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id) ON DELETE CASCADE, + UNIQUE (ref_id, capability_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query("CREATE INDEX IF NOT EXISTS idx_capability_versions_ref ON capability_versions(ref_id)") + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_ref_current ( + ref_id TEXT PRIMARY KEY, + capability_id TEXT NOT NULL, + catalog_revision INTEGER NOT NULL, + FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id) ON DELETE CASCADE, + FOREIGN KEY (capability_id) REFERENCES capability_versions(capability_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_capability_ref_current_version ON capability_ref_current(capability_id)", + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_manifests ( + manifest_id TEXT PRIMARY KEY, + consumer_id TEXT NOT NULL, + canonical_content BLOB NOT NULL, + created_at TEXT NOT NULL + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_manifest_entries ( + manifest_id TEXT NOT NULL, + position INTEGER NOT NULL, + ref_id TEXT NOT NULL, + capability_id TEXT NOT NULL, + PRIMARY KEY (manifest_id, position), + UNIQUE (manifest_id, ref_id), + FOREIGN KEY (manifest_id) REFERENCES surface_manifests(manifest_id) ON DELETE CASCADE, + FOREIGN KEY (ref_id, capability_id) REFERENCES capability_versions(ref_id, capability_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_proposals ( + proposal_id TEXT PRIMARY KEY, + consumer_id TEXT NOT NULL, + base_publication_id TEXT, + proposed_manifest_id TEXT NOT NULL, + trigger_kind TEXT NOT NULL, + trigger_id TEXT NOT NULL, + source_revision_set TEXT NOT NULL, + diff_summary TEXT NOT NULL, + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('pending', 'resolved', 'superseded')), + created_at TEXT NOT NULL, + resolved_at TEXT, + FOREIGN KEY (proposed_manifest_id) REFERENCES surface_manifests(manifest_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_review_items ( + review_item_id TEXT PRIMARY KEY, + created_by_proposal_id TEXT NOT NULL, + consumer_id TEXT NOT NULL, + ref_id TEXT NOT NULL, + before_capability_id TEXT, + target_capability_id TEXT, + target_key TEXT NOT NULL, + change_class TEXT NOT NULL, + policy_action TEXT NOT NULL, + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('pending', 'resolved', 'obsolete')), + current_decision_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (consumer_id, ref_id, target_key), + FOREIGN KEY (created_by_proposal_id) REFERENCES surface_proposals(proposal_id), + FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id), + FOREIGN KEY (before_capability_id) REFERENCES capability_versions(capability_id), + FOREIGN KEY (target_capability_id) REFERENCES capability_versions(capability_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_review_decisions ( + decision_id TEXT PRIMARY KEY, + review_item_id TEXT NOT NULL, + resolution_action TEXT NOT NULL CHECK ( + resolution_action IN ('approve_target', 'reject_target', 'keep_intent', 'remove_intent', 'rebind_ref') + ), + resolution_payload TEXT, + actor TEXT NOT NULL, + decided_at TEXT NOT NULL, + supersedes_decision_id TEXT, + FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id), + FOREIGN KEY (supersedes_decision_id) REFERENCES surface_review_decisions(decision_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_proposal_review_items ( + proposal_id TEXT NOT NULL, + review_item_id TEXT NOT NULL, + PRIMARY KEY (proposal_id, review_item_id), + FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id) ON DELETE CASCADE, + FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id) ON DELETE CASCADE + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_review_owners ( + review_item_id TEXT NOT NULL, + owner_type TEXT NOT NULL CHECK ( + owner_type IN ( + 'standard_profile', + 'custom_profile', + 'consumer_direct_exposure', + 'profile_server_exposure', + 'consumer_server_exposure', + 'mode_rule' + ) + ), + owner_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + first_proposal_id TEXT NOT NULL, + last_proposal_id TEXT NOT NULL, + PRIMARY KEY (review_item_id, owner_type, owner_id), + FOREIGN KEY (review_item_id) REFERENCES surface_review_items(review_item_id) ON DELETE CASCADE, + FOREIGN KEY (first_proposal_id) REFERENCES surface_proposals(proposal_id), + FOREIGN KEY (last_proposal_id) REFERENCES surface_proposals(proposal_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_publications ( + publication_id TEXT PRIMARY KEY, + consumer_id TEXT NOT NULL, + manifest_id TEXT NOT NULL, + proposal_id TEXT, + reason TEXT NOT NULL, + published_by TEXT NOT NULL, + published_at TEXT NOT NULL, + supersedes_publication_id TEXT, + FOREIGN KEY (manifest_id) REFERENCES surface_manifests(manifest_id), + FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id), + FOREIGN KEY (supersedes_publication_id) REFERENCES surface_publications(publication_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS consumer_surface_bindings ( + consumer_id TEXT PRIMARY KEY, + active_publication_id TEXT NOT NULL, + generation INTEGER NOT NULL CHECK (generation > 0), + FOREIGN KEY (active_publication_id) REFERENCES surface_publications(publication_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS consumer_surface_generations ( + consumer_id TEXT PRIMARY KEY, + last_generation INTEGER NOT NULL CHECK (last_generation >= 0) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_surface_proposals_consumer ON surface_proposals(consumer_id, created_at)", + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_surface_reviews_consumer_state ON surface_review_items(consumer_id, lifecycle)", + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_surface_publications_consumer ON surface_publications(consumer_id, published_at)", + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_reconciliation_jobs ( + idempotency_key TEXT PRIMARY KEY, + cause_kind TEXT NOT NULL, + cause_id TEXT NOT NULL, + consumer_id TEXT NOT NULL, + target_revision_set TEXT NOT NULL, + expected_binding_generation INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'leased', 'succeeded', 'failed')), + attempt_count INTEGER NOT NULL, + leased_by TEXT, + lease_expires_at TEXT, + next_attempt_at TEXT NOT NULL, + last_error TEXT, + success_receipt TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + "#, + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_surface_jobs_lease + ON surface_reconciliation_jobs(status, next_attempt_at, lease_expires_at) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS surface_outbox_events ( + event_id TEXT PRIMARY KEY, + event_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL, + delivered_at TEXT + ) + "#, + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_surface_outbox_pending + ON surface_outbox_events(delivered_at, created_at) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS capability_change_events ( + event_id TEXT PRIMARY KEY, + consumer_id TEXT NOT NULL, + proposal_id TEXT NOT NULL, + ref_id TEXT NOT NULL, + before_capability_id TEXT, + target_capability_id TEXT, + change_class TEXT NOT NULL, + policy_action TEXT NOT NULL, + actor TEXT NOT NULL, + occurred_at TEXT NOT NULL, + FOREIGN KEY (proposal_id) REFERENCES surface_proposals(proposal_id), + FOREIGN KEY (ref_id) REFERENCES capability_refs(ref_id), + FOREIGN KEY (before_capability_id) REFERENCES capability_versions(capability_id), + FOREIGN KEY (target_capability_id) REFERENCES capability_versions(capability_id) + ) + "#, + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_capability_change_events_consumer_time + ON capability_change_events(consumer_id, occurred_at) + "#, + ) + .execute(&mut **transaction) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS configuration_mode_transitions ( + transition_id TEXT PRIMARY KEY, + previous_mode TEXT NOT NULL CHECK (previous_mode IN ('unify', 'hosted', 'transparent')), + target_mode TEXT NOT NULL CHECK (target_mode IN ('unify', 'hosted', 'transparent')), + status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), + created_at TEXT NOT NULL, + completed_at TEXT + ) + "#, + ) + .execute(&mut **transaction) + .await?; + sqlx::query( + r#" + CREATE UNIQUE INDEX IF NOT EXISTS idx_configuration_mode_transitions_single_pending + ON configuration_mode_transitions(status) + WHERE status = 'pending' + "#, + ) + .execute(&mut **transaction) + .await?; + + Ok(()) +} diff --git a/backend/crates/mcpmate-migrations/src/migrations/mod.rs b/backend/crates/mcpmate-migrations/src/migrations/mod.rs new file mode 100644 index 00000000..3f870a34 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/migrations/mod.rs @@ -0,0 +1,110 @@ +pub(crate) mod audit; +pub(crate) mod config; + +use std::borrow::Cow; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use sha2::{Digest, Sha256}; +use sqlx::{Sqlite, Transaction}; + +#[async_trait] +pub(crate) trait MigrationStep: Send + Sync { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()>; +} + +pub(crate) struct SqlMigration { + sql: &'static str, +} + +impl SqlMigration { + const fn new(sql: &'static str) -> Self { + Self { sql } + } +} + +#[async_trait] +impl MigrationStep for SqlMigration { + async fn apply( + &self, + transaction: &mut Transaction<'_, Sqlite>, + ) -> Result<()> { + for statement in self.sql.split(";\n").filter(|statement| !statement.trim().is_empty()) { + sqlx::query(statement) + .execute(&mut **transaction) + .await + .context("execute SQL migration statement")?; + } + Ok(()) + } +} + +pub(crate) struct Migration { + pub(crate) version: i64, + pub(crate) name: &'static str, + checksum_sources: Vec<&'static str>, + pub(crate) step: Box, +} + +impl Migration { + pub(crate) fn sql( + version: i64, + name: &'static str, + sql: &'static str, + ) -> Self { + Self { + version, + name, + checksum_sources: vec![sql], + step: Box::new(SqlMigration::new(sql)), + } + } + + pub(crate) fn rust( + version: i64, + name: &'static str, + checksum_sources: &'static [&'static str], + step: impl MigrationStep + 'static, + ) -> Self { + Self { + version, + name, + checksum_sources: checksum_sources.to_vec(), + step: Box::new(step), + } + } + + pub(crate) fn checksum(&self) -> String { + let mut digest = Sha256::new(); + for source in &self.checksum_sources { + let source = normalize_line_endings(source); + digest.update((source.len() as u64).to_be_bytes()); + digest.update(source.as_bytes()); + } + format!("{:x}", digest.finalize()) + } +} + +fn normalize_line_endings(source: &str) -> Cow<'_, str> { + if source.contains('\r') { + Cow::Owned(source.replace("\r\n", "\n").replace('\r', "\n")) + } else { + Cow::Borrowed(source) + } +} + +#[cfg(test)] +mod tests { + use super::Migration; + + #[test] + fn checksum_is_stable_across_line_endings() { + let lf = Migration::sql(1, "line endings", "CREATE TABLE example (id TEXT);\n"); + let crlf = Migration::sql(1, "line endings", "CREATE TABLE example (id TEXT);\r\n"); + + assert_eq!(lf.checksum(), crlf.checksum()); + } +} diff --git a/backend/crates/mcpmate-migrations/src/runner.rs b/backend/crates/mcpmate-migrations/src/runner.rs new file mode 100644 index 00000000..2b0bf433 --- /dev/null +++ b/backend/crates/mcpmate-migrations/src/runner.rs @@ -0,0 +1,392 @@ +use std::{ + fs::{File, OpenOptions}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, bail}; +use fs2::FileExt; +use sqlx::{Pool, Sqlite}; + +use crate::migrations::{self, Migration}; + +const LEDGER_TABLE: &str = "mcpmate_schema_migrations"; +const LEDGER_STATE_TABLE: &str = "mcpmate_schema_migration_state"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DatabaseTarget { + Config, + Audit, +} + +impl DatabaseTarget { + const fn name(self) -> &'static str { + match self { + Self::Config => "config", + Self::Audit => "audit", + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum DatabaseSource<'a> { + InMemory, + File { path: &'a Path, existed_before_open: bool }, +} + +pub async fn prepare_config_database( + pool: &Pool, + source: DatabaseSource<'_>, +) -> Result> { + prepare_database(pool, DatabaseTarget::Config, source).await +} + +pub async fn prepare_audit_database( + pool: &Pool, + source: DatabaseSource<'_>, +) -> Result> { + prepare_database(pool, DatabaseTarget::Audit, source).await +} + +pub async fn verify_config_database(pool: &Pool) -> Result<()> { + verify_database(pool, DatabaseTarget::Config).await +} + +pub async fn verify_capability_catalog_database(pool: &Pool) -> Result<()> { + verify_config_database(pool).await?; + migrations::config::verify_capability_catalog(pool).await +} + +pub async fn verify_audit_database(pool: &Pool) -> Result<()> { + verify_database(pool, DatabaseTarget::Audit).await +} + +async fn prepare_database( + pool: &Pool, + target: DatabaseTarget, + source: DatabaseSource<'_>, +) -> Result> { + let migrations = migrations_for(target); + match source { + DatabaseSource::InMemory => { + run(pool, target, &migrations).await?; + Ok(None) + } + DatabaseSource::File { + path, + existed_before_open, + } => prepare_file_backed(pool, target, &migrations, path, existed_before_open).await, + } +} + +async fn verify_database( + pool: &Pool, + target: DatabaseTarget, +) -> Result<()> { + let migrations = migrations_for(target); + if has_pending(pool, target, &migrations).await? { + bail!("migration ledger for {} has pending migrations", target.name()); + } + Ok(()) +} + +fn migrations_for(target: DatabaseTarget) -> Vec { + match target { + DatabaseTarget::Config => migrations::config::all(), + DatabaseTarget::Audit => migrations::audit::all(), + } +} + +async fn run( + pool: &Pool, + target: DatabaseTarget, + migrations: &[Migration], +) -> Result<()> { + validate_migration_versions(target, migrations)?; + let mut transaction = pool.begin().await.context("begin migration transaction")?; + let ledger_table_exists = table_exists(&mut transaction, LEDGER_TABLE).await?; + let state_table_exists = table_exists(&mut transaction, LEDGER_STATE_TABLE).await?; + sqlx::query(&format!( + "CREATE TABLE IF NOT EXISTS {LEDGER_TABLE} (target TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL, checksum TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (target, version))" + )) + .execute(&mut *transaction) + .await + .context("create migration ledger")?; + sqlx::query(&format!( + "CREATE TABLE IF NOT EXISTS {LEDGER_STATE_TABLE} (target TEXT PRIMARY KEY, version INTEGER NOT NULL, checksum TEXT NOT NULL)" + )) + .execute(&mut *transaction) + .await + .context("create migration ledger state")?; + + let applied = read_applied(&mut transaction, target).await?; + let state = if state_table_exists { + read_state(&mut transaction, target).await? + } else { + None + }; + validate_ledger_history( + target, + migrations, + &applied, + state.as_ref(), + ledger_table_exists, + state_table_exists, + )?; + + for migration in migrations.iter().skip(applied.len()) { + let checksum = migration.checksum(); + migration.step.apply(&mut transaction).await.map_err(|error| { + anyhow::anyhow!( + "apply migration {} ({}) for {}: {error}", + migration.version, + migration.name, + target.name() + ) + })?; + sqlx::query(&format!( + "INSERT INTO {LEDGER_TABLE} (target, version, name, checksum) VALUES (?, ?, ?, ?)" + )) + .bind(target.name()) + .bind(migration.version) + .bind(migration.name) + .bind(checksum) + .execute(&mut *transaction) + .await + .context("record applied migration")?; + } + if let Some(last) = migrations.last() { + sqlx::query(&format!( + "INSERT INTO {LEDGER_STATE_TABLE} (target, version, checksum) VALUES (?, ?, ?) ON CONFLICT(target) DO UPDATE SET version = excluded.version, checksum = excluded.checksum" + )) + .bind(target.name()) + .bind(last.version) + .bind(last.checksum()) + .execute(&mut *transaction) + .await + .context("record migration ledger state")?; + } + transaction.commit().await.context("commit migrations") +} + +async fn has_pending( + pool: &Pool, + target: DatabaseTarget, + migrations: &[Migration], +) -> Result { + validate_migration_versions(target, migrations)?; + let ledger_table_exists = pool_table_exists(pool, LEDGER_TABLE).await?; + if !ledger_table_exists { + return Ok(!migrations.is_empty()); + } + let state_table_exists = pool_table_exists(pool, LEDGER_STATE_TABLE).await?; + let applied: Vec<(i64, String, String)> = sqlx::query_as(&format!( + "SELECT version, name, checksum FROM {LEDGER_TABLE} WHERE target = ? ORDER BY version" + )) + .bind(target.name()) + .fetch_all(pool) + .await + .context("read migration ledger")?; + let state: Option<(i64, String)> = if state_table_exists { + sqlx::query_as(&format!( + "SELECT version, checksum FROM {LEDGER_STATE_TABLE} WHERE target = ?" + )) + .bind(target.name()) + .fetch_optional(pool) + .await + .context("read migration ledger state")? + } else { + None + }; + validate_ledger_history( + target, + migrations, + &applied, + state.as_ref(), + ledger_table_exists, + state_table_exists, + )?; + Ok(applied.len() < migrations.len()) +} + +async fn prepare_file_backed( + pool: &Pool, + target: DatabaseTarget, + migrations: &[Migration], + path: &Path, + existed_before_open: bool, +) -> Result> { + let _lock = UpgradeLock::acquire(path).await?; + let backup = if existed_before_open { + backup_if_pending(pool, target, migrations, path).await? + } else { + None + }; + run(pool, target, migrations).await?; + Ok(backup) +} + +async fn backup_if_pending( + pool: &Pool, + target: DatabaseTarget, + migrations: &[Migration], + path: &Path, +) -> Result> { + if !has_pending(pool, target, migrations).await? { + return Ok(None); + } + let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + let backup_path = available_backup_path(path, ×tamp); + sqlx::query("VACUUM INTO ?") + .bind(backup_path.to_string_lossy().as_ref()) + .execute(pool) + .await + .with_context(|| format!("create migration backup at {}", backup_path.display()))?; + Ok(Some(backup_path)) +} + +fn available_backup_path( + database_path: &Path, + timestamp: &str, +) -> PathBuf { + for attempt in 0_u64.. { + let suffix = if attempt == 0 { + String::new() + } else { + format!("-{attempt}") + }; + let candidate = PathBuf::from(format!("{}.migration-{timestamp}{suffix}.bak", database_path.display())); + if !candidate.exists() { + return candidate; + } + } + unreachable!("the backup suffix space is unbounded") +} + +fn validate_migration_versions( + target: DatabaseTarget, + migrations: &[Migration], +) -> Result<()> { + for (index, migration) in migrations.iter().enumerate() { + let expected = index as i64 + 1; + if migration.version != expected { + bail!( + "migration versions for {} must be contiguous from 1; expected {expected}, found {}", + target.name(), + migration.version + ); + } + } + Ok(()) +} + +fn validate_ledger_history( + target: DatabaseTarget, + migrations: &[Migration], + applied: &[(i64, String, String)], + state: Option<&(i64, String)>, + ledger_table_exists: bool, + state_table_exists: bool, +) -> Result<()> { + for (index, (version, name, checksum)) in applied.iter().enumerate() { + let expected = migrations + .get(index) + .ok_or_else(|| anyhow::anyhow!("migration ledger for {} contains an unknown migration", target.name()))?; + if *version != expected.version || name != expected.name || checksum != &expected.checksum() { + bail!("migration ledger for {} is not a valid migration prefix", target.name()); + } + } + if !ledger_table_exists && !state_table_exists { + return Ok(()); + } + if !state_table_exists { + bail!("migration ledger state for {} is missing", target.name()); + } + match (applied.last(), state) { + (None, None) => Ok(()), + (Some((version, _, checksum)), Some((state_version, state_checksum))) + if version == state_version && checksum == state_checksum => + { + Ok(()) + } + _ => bail!( + "migration ledger state for {} does not match its history", + target.name() + ), + } +} + +async fn table_exists( + transaction: &mut sqlx::Transaction<'_, Sqlite>, + table: &str, +) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(table) + .fetch_one(&mut **transaction) + .await + .with_context(|| format!("inspect {table}")) +} + +async fn pool_table_exists( + pool: &Pool, + table: &str, +) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(table) + .fetch_one(pool) + .await + .with_context(|| format!("inspect {table}")) +} + +async fn read_applied( + transaction: &mut sqlx::Transaction<'_, Sqlite>, + target: DatabaseTarget, +) -> Result> { + sqlx::query_as(&format!( + "SELECT version, name, checksum FROM {LEDGER_TABLE} WHERE target = ? ORDER BY version" + )) + .bind(target.name()) + .fetch_all(&mut **transaction) + .await + .context("read migration ledger") +} + +async fn read_state( + transaction: &mut sqlx::Transaction<'_, Sqlite>, + target: DatabaseTarget, +) -> Result> { + sqlx::query_as(&format!( + "SELECT version, checksum FROM {LEDGER_STATE_TABLE} WHERE target = ?" + )) + .bind(target.name()) + .fetch_optional(&mut **transaction) + .await + .context("read migration ledger state") +} + +struct UpgradeLock(File); + +impl UpgradeLock { + async fn acquire(database_path: &Path) -> Result { + let lock_path = PathBuf::from(format!("{}.migration.lock", database_path.display())); + tokio::task::spawn_blocking(move || { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("open migration lock {}", lock_path.display()))?; + file.lock_exclusive() + .with_context(|| format!("lock migration path {}", lock_path.display()))?; + Ok(Self(file)) + }) + .await + .context("join migration lock task")? + } +} + +impl Drop for UpgradeLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} diff --git a/backend/crates/mcpmate-migrations/tests/config_upgrade_contract.rs b/backend/crates/mcpmate-migrations/tests/config_upgrade_contract.rs new file mode 100644 index 00000000..7f7a6ff2 --- /dev/null +++ b/backend/crates/mcpmate-migrations/tests/config_upgrade_contract.rs @@ -0,0 +1,333 @@ +#[path = "support/file.rs"] +mod file_support; +#[path = "support/memory.rs"] +mod memory_support; + +use std::fs; + +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; +use tempfile::tempdir; + +#[tokio::test] +async fn preserves_client_relationships_during_legacy_normalization() { + let pool = memory_support::pool().await; + sqlx::query( + "CREATE TABLE client ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + identifier TEXT NOT NULL UNIQUE, + config_mode TEXT NOT NULL DEFAULT 'hosted', + transport TEXT NOT NULL DEFAULT 'auto', + client_version TEXT, + backup_policy TEXT NOT NULL DEFAULT 'keep_n', + backup_limit INTEGER DEFAULT 5, + capability_source TEXT NOT NULL DEFAULT 'activated', + selected_profile_ids TEXT, + custom_profile_id TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + )", + ) + .execute(&pool) + .await + .expect("create legacy client table"); + sqlx::query( + "CREATE TABLE client_writeback_policy ( + client_identifier TEXT PRIMARY KEY, + merge_strategy TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (client_identifier) REFERENCES client(identifier) ON DELETE CASCADE + )", + ) + .execute(&pool) + .await + .expect("create legacy writeback policy table"); + sqlx::query( + "CREATE TABLE direct_exposure_refs ( + consumer_id TEXT NOT NULL, + ref_id TEXT NOT NULL, + enabled BOOLEAN NOT NULL, + FOREIGN KEY (consumer_id) REFERENCES client(identifier) ON DELETE CASCADE, + PRIMARY KEY (consumer_id, ref_id) + )", + ) + .execute(&pool) + .await + .expect("create legacy direct exposure reference table"); + sqlx::query( + "CREATE TABLE direct_exposure_servers ( + consumer_id TEXT NOT NULL, + server_id TEXT NOT NULL, + new_ref_policy TEXT NOT NULL, + FOREIGN KEY (consumer_id) REFERENCES client(identifier) ON DELETE CASCADE, + PRIMARY KEY (consumer_id, server_id) + )", + ) + .execute(&pool) + .await + .expect("create legacy direct exposure server table"); + + sqlx::query("INSERT INTO client (id, name, identifier) VALUES ('client-1', 'Cursor', 'cursor')") + .execute(&pool) + .await + .expect("insert legacy client"); + sqlx::query( + "INSERT INTO client_writeback_policy (client_identifier, merge_strategy) + VALUES ('cursor', 'deep_merge')", + ) + .execute(&pool) + .await + .expect("insert legacy writeback policy"); + sqlx::query( + "INSERT INTO direct_exposure_refs (consumer_id, ref_id, enabled) + VALUES ('cursor', 'tool:server:lookup', 1)", + ) + .execute(&pool) + .await + .expect("insert legacy direct exposure reference"); + sqlx::query( + "INSERT INTO direct_exposure_servers (consumer_id, server_id, new_ref_policy) + VALUES ('cursor', 'server-1', 'follow')", + ) + .execute(&pool) + .await + .expect("insert legacy direct exposure server"); + + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("migrate legacy config database"); + + let writeback_policy: String = + sqlx::query_scalar("SELECT merge_strategy FROM client_writeback_policy WHERE client_identifier = 'cursor'") + .fetch_one(&pool) + .await + .expect("load migrated writeback policy"); + let exposed_ref: (String, bool) = + sqlx::query_as("SELECT ref_id, enabled FROM direct_exposure_refs WHERE consumer_id = 'cursor'") + .fetch_one(&pool) + .await + .expect("load migrated direct exposure reference"); + let exposed_server: (String, String) = + sqlx::query_as("SELECT server_id, new_ref_policy FROM direct_exposure_servers WHERE consumer_id = 'cursor'") + .fetch_one(&pool) + .await + .expect("load migrated direct exposure server"); + + assert_eq!(writeback_policy, "deep_merge"); + assert_eq!(exposed_ref, ("tool:server:lookup".into(), true)); + assert_eq!(exposed_server, ("server-1".into(), "follow".into())); + let identity: (String, String, String) = + sqlx::query_as("SELECT name, display_name, connection_mode FROM client WHERE identifier = 'cursor'") + .fetch_one(&pool) + .await + .expect("load normalized client identity"); + assert_eq!(identity, ("Cursor".into(), "Cursor".into(), "manual".into())); + let foreign_key_errors: Vec = sqlx::query_scalar("PRAGMA foreign_key_check") + .fetch_all(&pool) + .await + .expect("check migrated foreign keys"); + assert!(foreign_key_errors.is_empty()); +} + +#[tokio::test] +async fn normalizes_a_client_schema_missing_unify_route_mode() { + let pool = memory_support::pool().await; + sqlx::query( + "CREATE TABLE client ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + identifier TEXT NOT NULL UNIQUE, + config_path TEXT, + transport TEXT NOT NULL DEFAULT 'auto' CHECK ( + transport IN ('auto', 'sse', 'stdio', 'streamable_http') + ), + connection_mode TEXT NOT NULL DEFAULT 'local_config_detected' CHECK ( + connection_mode IN ('local_config_detected', 'manual') + ), + registration_origin TEXT NOT NULL DEFAULT 'manual' CHECK ( + registration_origin IN ('manual', 'config_detection', 'runtime_initialize') + ), + runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), + attachment_state TEXT NOT NULL DEFAULT 'not_applicable' CHECK ( + attachment_state IN ('attached', 'detached', 'not_applicable') + ) + )", + ) + .execute(&pool) + .await + .expect("create pre-unify client table"); + sqlx::query( + "INSERT INTO client ( + id, name, identifier, config_path, connection_mode, registration_origin, runtime_observed + ) VALUES ( + 'client-1', 'Cursor', 'cursor', '/tmp/cursor.json', 'manual', 'runtime_initialize', 1 + )", + ) + .execute(&pool) + .await + .expect("insert pre-unify client"); + + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("migrate pre-unify config database"); + + let columns: Vec = sqlx::query_scalar("SELECT name FROM pragma_table_info('client')") + .fetch_all(&pool) + .await + .expect("inspect migrated client columns"); + assert!(columns.iter().any(|column| column == "unify_route_mode")); + let client: (String, String, bool, String) = sqlx::query_as( + "SELECT connection_mode, registration_origin, runtime_observed, unify_route_mode + FROM client WHERE identifier = 'cursor'", + ) + .fetch_one(&pool) + .await + .expect("load normalized pre-unify client"); + assert_eq!( + client, + ("manual".into(), "runtime_initialize".into(), true, "broker_only".into(),) + ); +} + +#[tokio::test] +async fn creates_a_distinct_backup_for_each_failed_upgrade_attempt() { + let directory = tempdir().expect("create temporary directory"); + let database_path = directory.path().join("config.db"); + let pool = file_support::pool(&database_path).await; + sqlx::query( + "CREATE TABLE secure_store_secrets ( + alias TEXT PRIMARY KEY, + kind TEXT NOT NULL, + encrypted_value TEXT NOT NULL + )", + ) + .execute(&pool) + .await + .expect("create incompatible secure store table"); + sqlx::query( + "INSERT INTO secure_store_secrets (alias, kind, encrypted_value) + VALUES ('legacy', 'api_key', 'ciphertext')", + ) + .execute(&pool) + .await + .expect("insert incompatible secure store record"); + + for _ in 0..2 { + let error = prepare_config_database( + &pool, + DatabaseSource::File { + path: &database_path, + existed_before_open: true, + }, + ) + .await + .expect_err("unsafe secure store migration must fail"); + assert!( + error.to_string().contains("cannot be safely upgraded"), + "unexpected migration error: {error:#}" + ); + } + + let backup_count = fs::read_dir(directory.path()) + .expect("read temporary directory") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().starts_with("config.db.migration-")) + .count(); + assert_eq!(backup_count, 2, "each failed attempt needs its own recovery backup"); +} + +#[tokio::test] +async fn creates_one_readable_backup_for_a_successful_existing_file_upgrade() { + let directory = tempdir().expect("create temporary directory"); + let database_path = directory.path().join("config.db"); + let pool = file_support::pool(&database_path).await; + sqlx::query("CREATE TABLE backup_probe (value TEXT NOT NULL)") + .execute(&pool) + .await + .expect("create backup probe table"); + sqlx::query("INSERT INTO backup_probe (value) VALUES ('before-migration')") + .execute(&pool) + .await + .expect("insert backup probe"); + + let backup = prepare_config_database( + &pool, + DatabaseSource::File { + path: &database_path, + existed_before_open: true, + }, + ) + .await + .expect("upgrade existing config database") + .expect("pending upgrade creates a recovery backup"); + let backup_pool = file_support::pool(&backup).await; + let value: String = sqlx::query_scalar("SELECT value FROM backup_probe") + .fetch_one(&backup_pool) + .await + .expect("read recovery backup"); + assert_eq!(value, "before-migration"); + + let second = prepare_config_database( + &pool, + DatabaseSource::File { + path: &database_path, + existed_before_open: true, + }, + ) + .await + .expect("recheck prepared config database"); + assert!(second.is_none()); +} + +#[tokio::test] +async fn fresh_file_upgrade_does_not_create_a_backup() { + let directory = tempdir().expect("create temporary directory"); + let database_path = directory.path().join("config.db"); + let pool = file_support::pool(&database_path).await; + + let backup = prepare_config_database( + &pool, + DatabaseSource::File { + path: &database_path, + existed_before_open: false, + }, + ) + .await + .expect("prepare fresh config database"); + + assert!(backup.is_none()); + let backup_count = fs::read_dir(directory.path()) + .expect("read temporary directory") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains(".migration-")) + .count(); + assert_eq!(backup_count, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn serializes_concurrent_file_upgrades() { + let directory = tempdir().expect("create temporary directory"); + let database_path = directory.path().join("config.db"); + let first_pool = file_support::pool(&database_path).await; + let second_pool = file_support::pool(&database_path).await; + let first_source = DatabaseSource::File { + path: &database_path, + existed_before_open: true, + }; + let second_source = DatabaseSource::File { + path: &database_path, + existed_before_open: true, + }; + + let (first, second) = tokio::join!( + prepare_config_database(&first_pool, first_source), + prepare_config_database(&second_pool, second_source), + ); + let backups = [first.expect("first upgrade"), second.expect("second upgrade")] + .into_iter() + .flatten() + .collect::>(); + + assert_eq!(backups.len(), 1); +} diff --git a/backend/crates/mcpmate-migrations/tests/ledger_contract.rs b/backend/crates/mcpmate-migrations/tests/ledger_contract.rs new file mode 100644 index 00000000..39ecba88 --- /dev/null +++ b/backend/crates/mcpmate-migrations/tests/ledger_contract.rs @@ -0,0 +1,118 @@ +#[path = "support/memory.rs"] +mod memory_support; + +use mcpmate_migrations::{DatabaseSource, prepare_config_database, verify_config_database}; + +async fn table_exists( + pool: &sqlx::SqlitePool, + table: &str, +) -> bool { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(table) + .fetch_one(pool) + .await + .expect("inspect SQLite table") +} + +#[tokio::test] +async fn verification_does_not_initialize_an_empty_database() { + let pool = memory_support::pool().await; + + let error = verify_config_database(&pool) + .await + .expect_err("an empty database is not prepared"); + + assert!( + error.to_string().contains("migration ledger"), + "unexpected verification error: {error:#}" + ); + assert!(!table_exists(&pool, "mcpmate_schema_migrations").await); +} + +#[tokio::test] +async fn preparation_applies_the_complete_config_stream_once() { + let pool = memory_support::pool().await; + + let backup = prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare in-memory config database"); + assert!(backup.is_none()); + verify_config_database(&pool) + .await + .expect("verify prepared config database"); + + let applied: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM mcpmate_schema_migrations WHERE target = 'config'") + .fetch_one(&pool) + .await + .expect("count applied config migrations"); + assert_eq!(applied, 10); + + let second_backup = prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config database again"); + assert!(second_backup.is_none()); +} + +#[tokio::test] +async fn verification_rejects_a_deleted_migration_record() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare in-memory config database"); + sqlx::query("DELETE FROM mcpmate_schema_migrations WHERE target = 'config' AND version = 5") + .execute(&pool) + .await + .expect("delete migration record"); + + let error = verify_config_database(&pool) + .await + .expect_err("deleted migration history must be rejected"); + assert!( + error.to_string().contains("valid migration prefix") + || error.to_string().contains("does not match its history"), + "unexpected verification error: {error:#}" + ); +} + +#[tokio::test] +async fn verification_rejects_an_unknown_migration_record() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare in-memory config database"); + sqlx::query( + "INSERT INTO mcpmate_schema_migrations (target, version, name, checksum) + VALUES ('config', 999, 'unknown migration', 'unknown')", + ) + .execute(&pool) + .await + .expect("insert unknown migration record"); + + let error = verify_config_database(&pool) + .await + .expect_err("unknown migration history must be rejected"); + assert!( + error.to_string().contains("unknown migration") || error.to_string().contains("valid migration prefix"), + "unexpected verification error: {error:#}" + ); +} + +#[tokio::test] +async fn verification_rejects_a_missing_ledger_state() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare in-memory config database"); + sqlx::query("DROP TABLE mcpmate_schema_migration_state") + .execute(&pool) + .await + .expect("remove ledger state"); + + let error = verify_config_database(&pool) + .await + .expect_err("missing ledger state must be rejected"); + assert!( + error.to_string().contains("ledger state") && error.to_string().contains("missing"), + "unexpected verification error: {error:#}" + ); +} diff --git a/backend/crates/mcpmate-migrations/tests/schema_contract.rs b/backend/crates/mcpmate-migrations/tests/schema_contract.rs new file mode 100644 index 00000000..4e28705f --- /dev/null +++ b/backend/crates/mcpmate-migrations/tests/schema_contract.rs @@ -0,0 +1,518 @@ +#[path = "support/memory.rs"] +mod memory_support; + +use mcpmate_migrations::{DatabaseSource, prepare_audit_database, prepare_config_database}; +use sha2::{Digest, Sha256}; + +async fn table_exists( + pool: &sqlx::SqlitePool, + table: &str, +) -> bool { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)") + .bind(table) + .fetch_one(pool) + .await + .expect("inspect SQLite table") +} + +async fn table_columns( + pool: &sqlx::SqlitePool, + table: &str, +) -> Vec { + sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")) + .fetch_all(pool) + .await + .expect("inspect SQLite columns") +} + +#[tokio::test] +async fn creates_config_and_audit_schema_through_independent_ledgers() { + let config = memory_support::pool().await; + prepare_config_database(&config, DatabaseSource::InMemory) + .await + .expect("prepare config database"); + for table in [ + "llm_provider", + "server_config", + "client", + "secure_store_secrets", + "profile", + "profile_server_relationships", + "profile_capability_refs", + "direct_exposure_refs", + "direct_exposure_servers", + "capability_server_snapshots", + "capability_refs", + "surface_manifests", + ] { + assert!(table_exists(&config, table).await, "missing config table {table}"); + } + for legacy in [ + "profile_tool", + "profile_prompt", + "profile_resource", + "profile_resource_template", + ] { + assert!(!table_exists(&config, legacy).await, "unexpected legacy table {legacy}"); + } + + let audit = memory_support::pool().await; + prepare_audit_database(&audit, DatabaseSource::InMemory) + .await + .expect("prepare audit database"); + assert!(table_exists(&audit, "audit_events").await); + assert!(table_exists(&audit, "audit_policy").await); + let audit_versions: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM mcpmate_schema_migrations WHERE target = 'audit'") + .fetch_one(&audit) + .await + .expect("count audit migrations"); + assert_eq!(audit_versions, 1); +} + +#[tokio::test] +async fn creates_resource_registry_routes_and_indexes() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config database"); + + let template_columns = table_columns(&pool, "server_resource_templates").await; + assert!(template_columns.iter().any(|column| column == "route_uri")); + let issued_columns = table_columns(&pool, "server_issued_resources").await; + for column in [ + "id", + "server_id", + "server_name", + "resource_uri", + "unique_uri", + "created_at", + "last_seen_at", + ] { + assert!(issued_columns.iter().any(|existing| existing == column)); + } + let indexes: Vec = sqlx::query_scalar("SELECT name FROM pragma_index_list('server_issued_resources')") + .fetch_all(&pool) + .await + .expect("inspect issued-resource indexes"); + for index in [ + "idx_server_issued_resources_lookup", + "idx_server_issued_resources_unique_uri", + ] { + assert!(indexes.iter().any(|existing| existing == index)); + } +} + +#[tokio::test] +async fn upgrades_legacy_llm_server_and_client_fields() { + let pool = memory_support::pool().await; + sqlx::query("CREATE TABLE llm_provider (id TEXT PRIMARY KEY, name TEXT NOT NULL, provider_type TEXT NOT NULL, base_url TEXT NOT NULL, model_id TEXT NOT NULL, secret_alias TEXT, default_params_json TEXT, created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL)") + .execute(&pool) + .await + .expect("create legacy LLM provider table"); + sqlx::query( + "CREATE TABLE server_meta ( + id TEXT PRIMARY KEY, + server_id TEXT NOT NULL UNIQUE, + server_name TEXT NOT NULL, + registry_version TEXT, + registry_meta_json TEXT, + extras_json TEXT + )", + ) + .execute(&pool) + .await + .expect("create legacy server metadata table"); + sqlx::query( + "CREATE TABLE client ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + identifier TEXT NOT NULL UNIQUE, + config_path TEXT, + config_mode TEXT, + transport TEXT NOT NULL DEFAULT 'auto', + backup_policy TEXT NOT NULL DEFAULT 'keep_n', + backup_limit INTEGER DEFAULT 5, + connection_mode TEXT NOT NULL DEFAULT 'manual', + registration_origin TEXT NOT NULL DEFAULT 'manual', + runtime_observed INTEGER NOT NULL DEFAULT 0, + format_rules TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + )", + ) + .execute(&pool) + .await + .expect("create legacy client table"); + sqlx::query( + "INSERT INTO client (id, name, identifier, connection_mode) + VALUES ('remote', 'Remote', 'remote', 'remote_http')", + ) + .execute(&pool) + .await + .expect("insert remote legacy client"); + let format_rules = r#"{"stdio":{"command_field":"command"}}"#; + sqlx::query( + "INSERT INTO client ( + id, name, identifier, config_path, connection_mode, format_rules + ) VALUES ('local', 'Local', 'local', '/tmp/client.json', 'manual', ?)", + ) + .bind(format_rules) + .execute(&pool) + .await + .expect("insert local legacy client"); + + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("upgrade legacy config database"); + + assert!( + table_columns(&pool, "llm_provider") + .await + .contains(&"is_default".into()) + ); + let server_columns = table_columns(&pool, "server_meta").await; + assert!(server_columns.contains(&"upstream_name".into())); + assert!(server_columns.contains(&"upstream_title".into())); + let remote: (String, String, i64) = sqlx::query_as( + "SELECT connection_mode, registration_origin, runtime_observed + FROM client WHERE id = 'remote'", + ) + .fetch_one(&pool) + .await + .expect("load normalized remote client"); + assert_eq!(remote, ("manual".into(), "runtime_initialize".into(), 1)); + let local: (String, String, String) = sqlx::query_as( + "SELECT connection_mode, registration_origin, transports + FROM client WHERE id = 'local'", + ) + .fetch_one(&pool) + .await + .expect("load normalized local client"); + assert_eq!( + local, + ( + "local_config_detected".into(), + "config_detection".into(), + format_rules.into(), + ) + ); +} + +#[tokio::test] +async fn capability_migration_checksum_covers_the_rust_artifact() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare config database"); + let recorded: String = sqlx::query_scalar( + "SELECT checksum FROM mcpmate_schema_migrations + WHERE target = 'config' AND version = 10", + ) + .fetch_one(&pool) + .await + .expect("load capability migration checksum"); + let source = include_str!("../src/migrations/config/v0010_create_capability_catalog.rs"); + let mut digest = Sha256::new(); + digest.update((source.len() as u64).to_be_bytes()); + digest.update(source.as_bytes()); + + assert_eq!(recorded, format!("{:x}", digest.finalize())); +} + +#[tokio::test] +async fn rejects_unversioned_and_unknown_epoch_capability_schemas() { + let unversioned = memory_support::pool().await; + sqlx::query("CREATE TABLE capability_records (id TEXT PRIMARY KEY)") + .execute(&unversioned) + .await + .expect("create unversioned capability table"); + let unversioned_error = prepare_config_database(&unversioned, DatabaseSource::InMemory) + .await + .expect_err("unversioned capability storage must be rejected"); + assert!(unversioned_error.to_string().contains("clean rebuild is required")); + assert!(table_exists(&unversioned, "capability_records").await); + assert!(!table_exists(&unversioned, "mcpmate_schema_migrations").await); + + let unknown_epoch = memory_support::pool().await; + sqlx::query( + "CREATE TABLE capability_schema_metadata ( + singleton INTEGER PRIMARY KEY, + schema_epoch INTEGER NOT NULL + )", + ) + .execute(&unknown_epoch) + .await + .expect("create capability schema metadata"); + sqlx::query("INSERT INTO capability_schema_metadata (singleton, schema_epoch) VALUES (1, 999)") + .execute(&unknown_epoch) + .await + .expect("insert unknown capability epoch"); + let epoch_error = prepare_config_database(&unknown_epoch, DatabaseSource::InMemory) + .await + .expect_err("unknown capability epoch must be rejected"); + assert!(epoch_error.to_string().contains("epoch 999 is not supported")); + assert!(!table_exists(&unknown_epoch, "mcpmate_schema_migrations").await); +} + +#[tokio::test] +async fn rejects_partial_or_unversioned_current_capability_storage() { + let unversioned = memory_support::pool().await; + sqlx::query("CREATE TABLE surface_reconciliation_jobs (job_id TEXT PRIMARY KEY)") + .execute(&unversioned) + .await + .expect("create unversioned current capability table"); + let error = prepare_config_database(&unversioned, DatabaseSource::InMemory) + .await + .expect_err("unversioned current capability storage must be rejected"); + assert!(error.to_string().contains("clean rebuild is required")); + assert!(!table_exists(&unversioned, "mcpmate_schema_migrations").await); + + let partial_epoch = memory_support::pool().await; + sqlx::query( + "CREATE TABLE capability_schema_metadata ( + singleton INTEGER PRIMARY KEY, + schema_epoch INTEGER NOT NULL + )", + ) + .execute(&partial_epoch) + .await + .expect("create capability schema metadata"); + sqlx::query("INSERT INTO capability_schema_metadata (singleton, schema_epoch) VALUES (1, 4)") + .execute(&partial_epoch) + .await + .expect("insert current capability epoch"); + let error = prepare_config_database(&partial_epoch, DatabaseSource::InMemory) + .await + .expect_err("incomplete current capability storage must be rejected"); + assert!(error.to_string().contains("incomplete capability schema epoch 4")); + assert!(!table_exists(&partial_epoch, "mcpmate_schema_migrations").await); +} + +async fn convert_prepared_database_to_epoch_four(pool: &sqlx::SqlitePool) { + sqlx::query( + "CREATE TABLE capability_schema_metadata ( + singleton INTEGER PRIMARY KEY, + schema_epoch INTEGER NOT NULL + )", + ) + .execute(pool) + .await + .expect("create current capability metadata"); + sqlx::query("INSERT INTO capability_schema_metadata (singleton, schema_epoch) VALUES (1, 4)") + .execute(pool) + .await + .expect("record current capability epoch"); + sqlx::query("DELETE FROM mcpmate_schema_migrations WHERE target = 'config'") + .execute(pool) + .await + .expect("remove config ledger to model pre-ledger storage"); + sqlx::query("DELETE FROM mcpmate_schema_migration_state WHERE target = 'config'") + .execute(pool) + .await + .expect("remove config ledger state to model pre-ledger storage"); +} + +#[tokio::test] +async fn adopts_complete_epoch_four_capability_storage_without_losing_data() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare current schema fixture"); + sqlx::query( + "INSERT INTO capability_server_snapshots ( + server_id, server_name, config_fingerprint, record_format_version, + catalog_revision, snapshot_state, initialize_payload, observed_at, + committed_at + ) VALUES ( + 'server-a', 'Docs', 'fingerprint', 1, 7, 'ready', '{}', + '2026-08-06T00:00:00Z', '2026-08-06T00:00:00Z' + )", + ) + .execute(&pool) + .await + .expect("insert current epoch catalog data"); + convert_prepared_database_to_epoch_four(&pool).await; + + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("adopt complete epoch-four catalog"); + + let revision: i64 = + sqlx::query_scalar("SELECT catalog_revision FROM capability_server_snapshots WHERE server_id = 'server-a'") + .fetch_one(&pool) + .await + .expect("load preserved catalog data"); + assert_eq!(revision, 7); + let version: i64 = sqlx::query_scalar( + "SELECT version FROM mcpmate_schema_migrations WHERE target = 'config' ORDER BY version DESC LIMIT 1", + ) + .fetch_one(&pool) + .await + .expect("load adopted migration version"); + assert_eq!(version, 10); +} + +#[tokio::test] +async fn rejects_epoch_four_catalog_with_complete_table_names_but_corrupt_structure() { + let pool = memory_support::pool().await; + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("prepare current schema fixture"); + convert_prepared_database_to_epoch_four(&pool).await; + sqlx::query("ALTER TABLE capability_refs RENAME TO capability_refs_valid") + .execute(&pool) + .await + .expect("retain valid table under a non-contract name"); + sqlx::query("CREATE TABLE capability_refs (ref_id TEXT PRIMARY KEY)") + .execute(&pool) + .await + .expect("create corrupt current table"); + + let error = prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect_err("corrupt epoch-four catalog must be rejected"); + assert!(error.to_string().contains("does not match the versioned contract")); + let recorded: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM mcpmate_schema_migrations WHERE target = 'config'") + .fetch_one(&pool) + .await + .expect("count rolled-back migration records"); + assert_eq!(recorded, 0); + assert!(table_exists(&pool, "capability_refs_valid").await); +} + +#[tokio::test] +async fn replaces_only_empty_legacy_secure_store_storage() { + let empty = memory_support::pool().await; + sqlx::query( + "CREATE TABLE secure_store_secrets ( + alias TEXT PRIMARY KEY, + kind TEXT NOT NULL, + encrypted_value TEXT NOT NULL + )", + ) + .execute(&empty) + .await + .expect("create empty legacy secure store"); + prepare_config_database(&empty, DatabaseSource::InMemory) + .await + .expect("replace empty legacy secure store"); + let columns = table_columns(&empty, "secure_store_secrets").await; + for required in ["provider_id", "provider_kind", "key_nonce", "encrypted_key"] { + assert!(columns.iter().any(|column| column == required)); + } + + let nonempty = memory_support::pool().await; + sqlx::query( + "CREATE TABLE secure_store_secrets ( + alias TEXT PRIMARY KEY, + kind TEXT NOT NULL, + encrypted_value TEXT NOT NULL + )", + ) + .execute(&nonempty) + .await + .expect("create nonempty legacy secure store"); + sqlx::query( + "INSERT INTO secure_store_secrets (alias, kind, encrypted_value) + VALUES ('legacy', 'api_key', 'ciphertext')", + ) + .execute(&nonempty) + .await + .expect("insert legacy secret"); + let error = prepare_config_database(&nonempty, DatabaseSource::InMemory) + .await + .expect_err("nonempty legacy secure store must not be replaced"); + assert!(error.to_string().contains("cannot be safely upgraded")); + let retained: String = + sqlx::query_scalar("SELECT encrypted_value FROM secure_store_secrets WHERE alias = 'legacy'") + .fetch_one(&nonempty) + .await + .expect("retain legacy secret after rollback"); + assert_eq!(retained, "ciphertext"); + assert!(!table_exists(&nonempty, "mcpmate_schema_migrations").await); +} + +#[tokio::test] +async fn rejects_nonempty_secure_store_with_incompatible_constraints() { + let pool = memory_support::pool().await; + sqlx::query( + "CREATE TABLE secure_store_secrets ( + alias TEXT, + kind TEXT, + label TEXT, + origin_server_id TEXT, + origin_server_name TEXT, + origin_server_kind TEXT, + origin_source TEXT, + origin_field_group TEXT, + origin_field_key TEXT, + origin_field_index INTEGER, + origin_field_path TEXT, + provider_id TEXT, + provider_kind TEXT, + version INTEGER, + key_nonce TEXT, + encrypted_key TEXT, + nonce TEXT, + encrypted_value TEXT, + key_wrap_alg TEXT, + encryption_alg TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP + )", + ) + .execute(&pool) + .await + .expect("create structurally incompatible secure store"); + sqlx::query( + "INSERT INTO secure_store_secrets ( + alias, kind, provider_id, provider_kind, version, key_nonce, + encrypted_key, nonce, encrypted_value, key_wrap_alg, + encryption_alg, created_at, updated_at + ) VALUES ( + 'legacy', 'api_key', 'provider', 'local', 1, 'key-nonce', + 'encrypted-key', 'nonce', 'ciphertext', 'AES-256-GCM', + 'AES-256-GCM', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + )", + ) + .execute(&pool) + .await + .expect("insert incompatible secure store record"); + + let error = prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect_err("nonempty incompatible secure store must be rejected"); + assert!(error.to_string().contains("cannot be safely upgraded")); + let retained: String = + sqlx::query_scalar("SELECT encrypted_value FROM secure_store_secrets WHERE alias = 'legacy'") + .fetch_one(&pool) + .await + .expect("retain secret after rollback"); + assert_eq!(retained, "ciphertext"); + assert!(!table_exists(&pool, "mcpmate_schema_migrations").await); +} + +#[tokio::test] +async fn replaces_only_empty_incompatible_secure_store_companion_tables() { + let pool = memory_support::pool().await; + sqlx::query("CREATE TABLE secure_store_usages (id TEXT PRIMARY KEY, alias TEXT)") + .execute(&pool) + .await + .expect("create incompatible secure store usage table"); + + prepare_config_database(&pool, DatabaseSource::InMemory) + .await + .expect("replace empty incompatible secure store tables"); + + let columns = table_columns(&pool, "secure_store_usages").await; + for required in ["server_id", "location_kind", "created_at", "updated_at"] { + assert!(columns.iter().any(|column| column == required)); + } + let foreign_keys: Vec = + sqlx::query_scalar("SELECT \"table\" FROM pragma_foreign_key_list('secure_store_usages')") + .fetch_all(&pool) + .await + .expect("inspect secure store usage foreign keys"); + assert!(foreign_keys.iter().any(|table| table == "secure_store_secrets")); +} diff --git a/backend/crates/mcpmate-migrations/tests/support/file.rs b/backend/crates/mcpmate-migrations/tests/support/file.rs new file mode 100644 index 00000000..3b223c40 --- /dev/null +++ b/backend/crates/mcpmate-migrations/tests/support/file.rs @@ -0,0 +1,16 @@ +use sqlx::{SqlitePool, sqlite::SqlitePoolOptions}; +use std::path::Path; + +pub async fn pool(path: &Path) -> SqlitePool { + let url = format!("sqlite://{}?mode=rwc", path.display()); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("connect file-backed SQLite database"); + sqlx::query("PRAGMA foreign_keys = ON") + .execute(&pool) + .await + .expect("enable SQLite foreign keys"); + pool +} diff --git a/backend/crates/mcpmate-migrations/tests/support/memory.rs b/backend/crates/mcpmate-migrations/tests/support/memory.rs new file mode 100644 index 00000000..1243e6d3 --- /dev/null +++ b/backend/crates/mcpmate-migrations/tests/support/memory.rs @@ -0,0 +1,14 @@ +use sqlx::{SqlitePool, sqlite::SqlitePoolOptions}; + +pub async fn pool() -> SqlitePool { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .expect("connect in-memory SQLite database"); + sqlx::query("PRAGMA foreign_keys = ON") + .execute(&pool) + .await + .expect("enable SQLite foreign keys"); + pool +} diff --git a/backend/crates/mcpmate-secrets/Cargo.toml b/backend/crates/mcpmate-secrets/Cargo.toml index c437b168..c72bfb07 100644 --- a/backend/crates/mcpmate-secrets/Cargo.toml +++ b/backend/crates/mcpmate-secrets/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Loocor "] [dependencies] anyhow = "1.0" base64 = "0.22.1" +mcpmate-migrations = { path = "../mcpmate-migrations" } ring = "0.17.14" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/backend/crates/mcpmate-secrets/src/database.rs b/backend/crates/mcpmate-secrets/src/database.rs index 6b037ec2..077ee6ac 100644 --- a/backend/crates/mcpmate-secrets/src/database.rs +++ b/backend/crates/mcpmate-secrets/src/database.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use sqlx::{Pool, Row, Sqlite, sqlite::SqliteRow}; use tracing::warn; @@ -17,32 +17,6 @@ pub(crate) struct SecretUsageRows { pub unsupported_count_by_alias: HashMap, } -const SECURE_STORE_SECRETS_TABLE: &str = "secure_store_secrets"; -const REQUIRED_SECRET_COLUMNS: &[&str] = &[ - "alias", - "kind", - "label", - "origin_server_id", - "origin_server_name", - "origin_server_kind", - "origin_source", - "origin_field_group", - "origin_field_key", - "origin_field_index", - "origin_field_path", - "provider_id", - "provider_kind", - "version", - "key_nonce", - "encrypted_key", - "nonce", - "encrypted_value", - "key_wrap_alg", - "encryption_alg", - "created_at", - "updated_at", -]; - pub(crate) struct SecretInsert<'a> { pub alias: &'a str, pub kind: &'a str, @@ -61,60 +35,7 @@ pub(crate) struct SecretUpdate<'a> { } pub(crate) async fn ensure_schema(pool: &Pool) -> Result<()> { - ensure_secure_store_secrets_schema(pool).await?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS secure_store_usages ( - id TEXT PRIMARY KEY, - alias TEXT NOT NULL, - server_id TEXT NOT NULL, - location_kind TEXT NOT NULL, - location_name TEXT, - location_index INTEGER, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (alias) REFERENCES secure_store_secrets (alias) ON DELETE CASCADE, - UNIQUE(alias, server_id, location_kind, location_name, location_index) - ) - "#, - ) - .execute(pool) - .await - .context("create secure_store_usages table")?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS secure_store_password_config ( - id INTEGER PRIMARY KEY CHECK (id = 1), - password_hash TEXT NOT NULL, - hash_salt TEXT NOT NULL, - hash_iterations INTEGER NOT NULL DEFAULT 600000, - protection_scope TEXT NOT NULL DEFAULT '["startup"]', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ) - .execute(pool) - .await - .context("create secure_store_password_config table")?; - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS secure_store_provider_config ( - id INTEGER PRIMARY KEY CHECK (id = 1), - provider_mode TEXT NOT NULL DEFAULT 'operating_system', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ) - .execute(pool) - .await - .context("create secure_store_provider_config table")?; - - Ok(()) + mcpmate_migrations::verify_config_database(pool).await } pub(crate) async fn secure_store_secret_count(pool: &Pool) -> Result { @@ -124,104 +45,6 @@ pub(crate) async fn secure_store_secret_count(pool: &Pool) -> Result) -> Result<()> { - if !table_exists(pool, SECURE_STORE_SECRETS_TABLE).await? { - create_secure_store_secrets_table(pool).await?; - return Ok(()); - } - - let columns = table_columns(pool, SECURE_STORE_SECRETS_TABLE).await?; - let missing_columns = REQUIRED_SECRET_COLUMNS - .iter() - .filter(|column| !columns.iter().any(|existing| existing == **column)) - .copied() - .collect::>(); - if missing_columns.is_empty() { - return Ok(()); - } - - let legacy_record_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM secure_store_secrets") - .fetch_one(pool) - .await - .context("count legacy secure store secrets")?; - - if legacy_record_count == 0 { - sqlx::query("DROP TABLE secure_store_secrets") - .execute(pool) - .await - .context("drop empty legacy secure_store_secrets table")?; - create_secure_store_secrets_table(pool).await?; - return Ok(()); - } - - bail!( - "outdated secure_store_secrets schema contains {legacy_record_count} legacy secret record(s); missing column(s): {}; reset the secure store data or run an explicit migration tool before using encrypted secrets", - missing_columns.join(", ") - ); -} - -async fn table_exists( - pool: &Pool, - table_name: &str, -) -> Result { - let exists: Option = - sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?1") - .bind(table_name) - .fetch_optional(pool) - .await - .with_context(|| format!("inspect sqlite table '{table_name}'"))?; - Ok(exists.is_some()) -} - -async fn table_columns( - pool: &Pool, - table_name: &str, -) -> Result> { - let rows = sqlx::query(&format!("PRAGMA table_info({table_name})")) - .fetch_all(pool) - .await - .with_context(|| format!("inspect sqlite table columns for '{table_name}'"))?; - rows.into_iter() - .map(|row| row.try_get("name").context("read sqlite table column name")) - .collect() -} - -async fn create_secure_store_secrets_table(pool: &Pool) -> Result<()> { - let secrets_schema = format!( - r#" - CREATE TABLE IF NOT EXISTS secure_store_secrets ( - alias TEXT PRIMARY KEY, - kind TEXT NOT NULL, - label TEXT, - origin_server_id TEXT, - origin_server_name TEXT, - origin_server_kind TEXT, - origin_source TEXT, - origin_field_group TEXT, - origin_field_key TEXT, - origin_field_index INTEGER, - origin_field_path TEXT, - provider_id TEXT NOT NULL, - provider_kind TEXT NOT NULL, - version INTEGER NOT NULL, - key_nonce TEXT NOT NULL, - encrypted_key TEXT NOT NULL, - nonce TEXT NOT NULL, - encrypted_value TEXT NOT NULL, - key_wrap_alg TEXT NOT NULL DEFAULT '{AEAD_ALGORITHM}', - encryption_alg TEXT NOT NULL DEFAULT '{AEAD_ALGORITHM}', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ); - sqlx::query(&secrets_schema) - .execute(pool) - .await - .context("create secure_store_secrets table")?; - Ok(()) -} - pub(crate) async fn insert_secret( pool: &Pool, input: SecretInsert<'_>, @@ -804,87 +627,3 @@ fn secret_origin_from_row(row: &sqlx::sqlite::SqliteRow) -> Result Pool { - SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool") - } - - async fn create_legacy_secret_table(pool: &Pool) { - sqlx::query( - r#" - CREATE TABLE secure_store_secrets ( - alias TEXT PRIMARY KEY, - kind TEXT NOT NULL, - label TEXT, - provider_id TEXT NOT NULL, - provider_kind TEXT NOT NULL, - version INTEGER NOT NULL, - nonce TEXT NOT NULL, - encrypted_value TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ) - .execute(pool) - .await - .expect("create legacy secrets table"); - } -} diff --git a/backend/crates/mcpmate-secrets/src/store.rs b/backend/crates/mcpmate-secrets/src/store.rs index 7d25217e..1ee26382 100644 --- a/backend/crates/mcpmate-secrets/src/store.rs +++ b/backend/crates/mcpmate-secrets/src/store.rs @@ -606,6 +606,18 @@ mod tests { SecretStoreRotationError, }; use sqlx::{Row, sqlite::SqlitePoolOptions}; + + async fn prepared_pool() -> Pool { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .expect("sqlite pool"); + mcpmate_migrations::prepare_config_database(&pool, mcpmate_migrations::DatabaseSource::InMemory) + .await + .expect("prepare config schema"); + pool + } use tempfile::TempDir; #[derive(Debug)] @@ -725,11 +737,7 @@ mod tests { #[tokio::test] #[serial_test::serial] async fn initialization_with_failing_root_key_provider_does_not_fallback() { - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let err = LocalSecretStore::initialize_with_root_key_provider(db_pool, Arc::new(FailingRootKeyProvider)) .await @@ -742,11 +750,7 @@ mod tests { #[serial_test::serial] async fn initialization_with_existing_secrets_does_not_create_missing_local_root_key() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; LocalSecretStore::ensure_schema(&db_pool).await.expect("ensure schema"); sqlx::query( r#" @@ -796,11 +800,7 @@ mod tests { #[serial_test::serial] async fn development_root_key_provider_metadata_is_stored_with_secret() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool, temp_dir.path().join("secrets").join("local-root.key"), @@ -827,11 +827,7 @@ mod tests { #[serial_test::serial] async fn create_secret_stores_origin_metadata() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool, temp_dir.path().join("secrets").join("local-root.key"), @@ -878,11 +874,7 @@ mod tests { #[serial_test::serial] async fn list_usages_skips_unknown_location_rows() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool, temp_dir.path().join("secrets").join("local-root.key"), @@ -954,11 +946,7 @@ mod tests { #[serial_test::serial] async fn delete_secret_without_force_blocks_unknown_location_rows() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool, temp_dir.path().join("secrets").join("local-root.key"), @@ -1014,11 +1002,7 @@ mod tests { #[serial_test::serial] async fn update_secret_rejects_kind_changes() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool, temp_dir.path().join("secrets").join("local-root.key"), @@ -1068,11 +1052,7 @@ mod tests { #[serial_test::serial] async fn create_secret_uses_per_record_envelope_keys() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let store = LocalSecretStore::initialize_with_development_root_key( db_pool.clone(), temp_dir.path().join("secrets").join("local-root.key"), @@ -1117,11 +1097,7 @@ mod tests { #[serial_test::serial] async fn rotate_provider_rewraps_records_updates_metadata_and_persists_mode() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let provider_a = Arc::new(TestRootKeyProvider::new( temp_dir.path().join("secrets-a").join("local-root.key"), "provider-a", @@ -1174,11 +1150,7 @@ mod tests { #[serial_test::serial] async fn rotate_provider_rejects_corrupted_current_record_without_mutation() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let provider_a = Arc::new(TestRootKeyProvider::new( temp_dir.path().join("secrets-a").join("local-root.key"), "provider-a", @@ -1228,11 +1200,7 @@ mod tests { #[serial_test::serial] async fn rotate_provider_overwrites_stale_target_root_material() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let provider_a = Arc::new(TestRootKeyProvider::new( temp_dir.path().join("secrets-a").join("local-root.key"), "provider-a", @@ -1269,11 +1237,7 @@ mod tests { #[serial_test::serial] async fn rotate_provider_returns_committed_store_without_target_reload() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let provider_a = Arc::new(TestRootKeyProvider::new( temp_dir.path().join("secrets-a").join("local-root.key"), "provider-a", @@ -1310,11 +1274,7 @@ mod tests { #[serial_test::serial] async fn rotate_provider_target_failure_keeps_current_state() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let provider_a = Arc::new(TestRootKeyProvider::new( temp_dir.path().join("secrets-a").join("local-root.key"), "provider-a", @@ -1345,11 +1305,7 @@ mod tests { #[serial_test::serial] async fn passphrase_rotation_rewraps_records() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let passphrase_path = temp_dir.path().join("secrets").join("passphrase-wrapped-key.json"); let old_provider = Arc::new(crate::PassphraseRootKeyProvider::new( &passphrase_path, @@ -1383,11 +1339,7 @@ mod tests { #[serial_test::serial] async fn passphrase_rotation_persistence_failure_restores_root_material() { let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let passphrase_path = temp_dir.path().join("secrets").join("passphrase-wrapped-key.json"); let old_provider = Arc::new(crate::PassphraseRootKeyProvider::new( &passphrase_path, @@ -1456,11 +1408,7 @@ mod tests { use std::os::unix::fs::PermissionsExt; let temp_dir = TempDir::new().expect("temp dir"); - let db_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("sqlite pool"); + let db_pool = prepared_pool().await; let secrets_dir = temp_dir.path().join("secrets"); let passphrase_path = secrets_dir.join("passphrase-wrapped-key.json"); let old_provider = Arc::new(crate::PassphraseRootKeyProvider::new( diff --git a/backend/src/api/handlers/audit.rs b/backend/src/api/handlers/audit.rs index 465322c4..f064400e 100644 --- a/backend/src/api/handlers/audit.rs +++ b/backend/src/api/handlers/audit.rs @@ -243,6 +243,7 @@ mod tests { .connect_with(options) .await .expect("connect audit db"); + crate::test_helpers::prepare_audit_database(&audit_pool).await; let audit_database = Arc::new(AuditDatabase { pool: audit_pool.clone(), diff --git a/backend/src/api/handlers/client/handlers.rs b/backend/src/api/handlers/client/handlers.rs index 390ffb77..bd717319 100644 --- a/backend/src/api/handlers/client/handlers.rs +++ b/backend/src/api/handlers/client/handlers.rs @@ -1607,6 +1607,7 @@ mod tests { .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&db_pool).await; initialize_server_tables(&db_pool).await.expect("init server tables"); initialize_client_table(&db_pool).await.expect("init client table"); crate::config::database::initialize_capability_catalog(&db_pool) diff --git a/backend/src/api/handlers/client/surface_reviews.rs b/backend/src/api/handlers/client/surface_reviews.rs index 9f2e5af8..1e9ba324 100644 --- a/backend/src/api/handlers/client/surface_reviews.rs +++ b/backend/src/api/handlers/client/surface_reviews.rs @@ -1346,6 +1346,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("server schema"); diff --git a/backend/src/api/handlers/onboarding.rs b/backend/src/api/handlers/onboarding.rs index b75a8255..6b2ab7a0 100644 --- a/backend/src/api/handlers/onboarding.rs +++ b/backend/src/api/handlers/onboarding.rs @@ -421,6 +421,7 @@ mod tests { .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&db_pool).await; initialize_server_tables(&db_pool).await.expect("init server tables"); initialize_profile_tables(&db_pool).await.expect("init profile tables"); initialize_client_table(&db_pool).await.expect("init client table"); diff --git a/backend/src/api/handlers/profile/capabilities.rs b/backend/src/api/handlers/profile/capabilities.rs index 18dcee6d..ae6b7ea0 100644 --- a/backend/src/api/handlers/profile/capabilities.rs +++ b/backend/src/api/handlers/profile/capabilities.rs @@ -506,6 +506,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); diff --git a/backend/src/api/handlers/profile/mgmt.rs b/backend/src/api/handlers/profile/mgmt.rs index 218f58fc..17503dec 100644 --- a/backend/src/api/handlers/profile/mgmt.rs +++ b/backend/src/api/handlers/profile/mgmt.rs @@ -744,6 +744,7 @@ mod tests { .connect("sqlite::memory:") .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); diff --git a/backend/src/api/handlers/profile/token_estimate.rs b/backend/src/api/handlers/profile/token_estimate.rs index 74ed1c58..77bc493d 100644 --- a/backend/src/api/handlers/profile/token_estimate.rs +++ b/backend/src/api/handlers/profile/token_estimate.rs @@ -270,6 +270,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("create test database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); diff --git a/backend/src/api/handlers/secrets.rs b/backend/src/api/handlers/secrets.rs index fd2f70b7..1a4a5584 100644 --- a/backend/src/api/handlers/secrets.rs +++ b/backend/src/api/handlers/secrets.rs @@ -1006,6 +1006,7 @@ mod tests { .execute(&db_pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&db_pool).await; crate::config::server::init::initialize_server_tables(&db_pool) .await .expect("init server tables"); diff --git a/backend/src/api/handlers/server/basic.rs b/backend/src/api/handlers/server/basic.rs index 704f173c..d406db36 100644 --- a/backend/src/api/handlers/server/basic.rs +++ b/backend/src/api/handlers/server/basic.rs @@ -1344,6 +1344,7 @@ mod tests { .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&db_pool).await; crate::config::server::init::initialize_server_tables(&db_pool) .await .expect("init server tables"); diff --git a/backend/src/api/handlers/server/capability.rs b/backend/src/api/handlers/server/capability.rs index 453a43b4..ff7c963d 100644 --- a/backend/src/api/handlers/server/capability.rs +++ b/backend/src/api/handlers/server/capability.rs @@ -764,6 +764,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize schema"); diff --git a/backend/src/api/handlers/server/common.rs b/backend/src/api/handlers/server/common.rs index e0586e17..a0009274 100644 --- a/backend/src/api/handlers/server/common.rs +++ b/backend/src/api/handlers/server/common.rs @@ -856,6 +856,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/api/handlers/server/crud.rs b/backend/src/api/handlers/server/crud.rs index 609bceda..a93b6320 100644 --- a/backend/src/api/handlers/server/crud.rs +++ b/backend/src/api/handlers/server/crud.rs @@ -1623,6 +1623,7 @@ for line in sys.stdin: .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&db_pool).await; crate::config::initialization::run_initialization(&db_pool) .await .expect("initialize database"); diff --git a/backend/src/api/handlers/server/preview.rs b/backend/src/api/handlers/server/preview.rs index f0da5c9d..396ee04c 100644 --- a/backend/src/api/handlers/server/preview.rs +++ b/backend/src/api/handlers/server/preview.rs @@ -417,6 +417,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init server tables"); pool } diff --git a/backend/src/audit/logger.rs b/backend/src/audit/logger.rs index 3fc0f2cc..4d520262 100644 --- a/backend/src/audit/logger.rs +++ b/backend/src/audit/logger.rs @@ -113,6 +113,7 @@ mod tests { .connect_with(options) .await .expect("connect"); + crate::test_helpers::prepare_audit_database(&pool).await; let store = Arc::new(AuditStore::new(pool)); AuditService::new(store).await.expect("audit service") } diff --git a/backend/src/audit/storage.rs b/backend/src/audit/storage.rs index c859d29d..beba88a7 100644 --- a/backend/src/audit/storage.rs +++ b/backend/src/audit/storage.rs @@ -34,71 +34,13 @@ impl AuditStore { } pub async fn initialize(&self) -> Result<()> { - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS audit_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, - action TEXT NOT NULL, - status TEXT NOT NULL, - occurred_at_ms INTEGER NOT NULL, - actor TEXT, - request_id TEXT, - client_id TEXT, - profile_id TEXT, - server_id TEXT, - session_id TEXT, - protocol_version TEXT, - http_method TEXT, - route TEXT, - mcp_method TEXT, - target TEXT, - direction TEXT, - error_code TEXT, - error_message TEXT, - detail TEXT, - duration_ms INTEGER, - data_json TEXT, - task_id TEXT, - related_task_id TEXT, - progress_token TEXT - ) - "#, - ) - .execute(&self.pool) - .await - .context("Failed to create audit_events table")?; - - for statement in [ - "CREATE INDEX IF NOT EXISTS idx_audit_events_occurred_at ON audit_events (occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_category_action ON audit_events (category, action, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_status ON audit_events (status, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_server_id ON audit_events (server_id, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_profile_id ON audit_events (profile_id, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_client_id ON audit_events (client_id, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_session_id ON audit_events (session_id, occurred_at_ms DESC, id DESC)", - "CREATE INDEX IF NOT EXISTS idx_audit_events_task_id ON audit_events (task_id, occurred_at_ms DESC, id DESC)", - ] { - sqlx::query(statement) - .execute(&self.pool) - .await - .with_context(|| format!("Failed to execute audit index statement: {statement}"))?; - } - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS audit_policy ( - id INTEGER PRIMARY KEY CHECK (id = 1), - policy TEXT NOT NULL, - sweep_interval_secs INTEGER NOT NULL, - updated_at_ms INTEGER NOT NULL - ) - "#, - ) - .execute(&self.pool) - .await - .context("Failed to create audit_policy table")?; - + mcpmate_migrations::verify_audit_database(&self.pool) + .await + .context("Migrate audit database before creating the audit store")?; + sqlx::query("SELECT 1 FROM audit_policy LIMIT 1") + .execute(&self.pool) + .await + .context("Audit database was not migrated before creating the audit store")?; Ok(()) } @@ -594,6 +536,15 @@ mod tests { .connect_with(options) .await .expect("connect"); + mcpmate_migrations::prepare_audit_database( + &pool, + mcpmate_migrations::DatabaseSource::File { + path: &path, + existed_before_open: false, + }, + ) + .await + .expect("prepare audit database"); let store = AuditStore::new(pool); store.initialize().await.expect("initialize audit store"); store diff --git a/backend/src/clients/service/core.rs b/backend/src/clients/service/core.rs index a0c71637..3df032c8 100644 --- a/backend/src/clients/service/core.rs +++ b/backend/src/clients/service/core.rs @@ -1336,6 +1336,7 @@ mod render_definition_tests { .await .expect("sqlite pool"), ); + crate::test_helpers::prepare_config_database(pool.as_ref()).await; initialize_client_table(pool.as_ref()).await.expect("init client table"); let template_root = TemplateRoot::new(temp_dir.path().join("client-templates")); @@ -1400,6 +1401,7 @@ mod render_definition_tests { .connect("sqlite::memory:") .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::client::init::initialize_client_table(&pool) .await .expect("init client table"); @@ -1477,6 +1479,7 @@ mod render_definition_tests { .connect("sqlite::memory:") .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::client::init::initialize_client_table(&pool) .await .expect("init client table"); @@ -1506,6 +1509,7 @@ mod render_definition_tests { .connect("sqlite::memory:") .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::client::init::initialize_client_table(&pool) .await .expect("init client table"); diff --git a/backend/src/clients/service/list.rs b/backend/src/clients/service/list.rs index 7869bdf1..e6b3e29e 100644 --- a/backend/src/clients/service/list.rs +++ b/backend/src/clients/service/list.rs @@ -142,6 +142,7 @@ mod tests { .expect("sqlite pool"), ); + crate::test_helpers::prepare_config_database(pool.as_ref()).await; initialize_server_tables(pool.as_ref()) .await .expect("init server tables"); diff --git a/backend/src/clients/service/query.rs b/backend/src/clients/service/query.rs index caf62c08..0603198e 100644 --- a/backend/src/clients/service/query.rs +++ b/backend/src/clients/service/query.rs @@ -282,6 +282,7 @@ mod tests { .await .expect("sqlite pool"), ); + crate::test_helpers::prepare_config_database(pool.as_ref()).await; crate::config::initialization::run_initialization(pool.as_ref()) .await .expect("initialize database"); diff --git a/backend/src/clients/service/state.rs b/backend/src/clients/service/state.rs index 52e95402..ce8dcfb2 100644 --- a/backend/src/clients/service/state.rs +++ b/backend/src/clients/service/state.rs @@ -940,6 +940,7 @@ mod tests { .expect("sqlite pool"), ); + crate::test_helpers::prepare_config_database(pool.as_ref()).await; crate::config::initialization::run_initialization(pool.as_ref()) .await .expect("initialize database"); diff --git a/backend/src/config/audit_database.rs b/backend/src/config/audit_database.rs index 127c5253..75880d29 100644 --- a/backend/src/config/audit_database.rs +++ b/backend/src/config/audit_database.rs @@ -22,6 +22,7 @@ impl AuditDatabase { pub async fn new() -> Result { let database_url = global_paths().audit_database_url(); let path = global_paths().audit_database_path(); + let existed = path.exists(); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) @@ -57,6 +58,19 @@ impl AuditDatabase { .await .context("Failed to configure busy_timeout for audit database")?; + if let Some(backup_path) = mcpmate_migrations::prepare_audit_database( + &pool, + mcpmate_migrations::DatabaseSource::File { + path: &path, + existed_before_open: existed, + }, + ) + .await + .context("Failed to migrate audit database")? + { + tracing::info!(path = %backup_path.display(), "Created audit database backup before migration"); + } + Ok(Self { pool, path }) } diff --git a/backend/src/config/client/init.rs b/backend/src/config/client/init.rs index 6230ab29..a66087f4 100644 --- a/backend/src/config/client/init.rs +++ b/backend/src/config/client/init.rs @@ -1,362 +1,23 @@ use anyhow::Result; use sqlx::{Pool, Sqlite}; -use tracing; -use crate::common::constants::database::tables; - -const DEFAULT_BACKUP_POLICY: &str = "keep_n"; -const DEFAULT_BACKUP_LIMIT: i64 = 5; -const DEFAULT_CAPABILITY_SOURCE: &str = "activated"; -const DEFAULT_CONNECTION_MODE: &str = "local_config_detected"; -const DEFAULT_GOVERNANCE_KIND: &str = "passive"; -const DEFAULT_REGISTRATION_ORIGIN: &str = "manual"; pub(crate) const CLIENT_RUNTIME_SETTINGS_TABLE: &str = "client_runtime_settings"; -pub(crate) const CLIENT_TEMPLATE_RUNTIME_TABLE: &str = "client_template_runtime"; -pub(crate) const CLIENT_WRITEBACK_POLICY_TABLE: &str = "client_writeback_policy"; -pub(crate) const DEFAULT_CONFIG_MODE_SETTING_KEY: &str = "default_config_mode"; pub(crate) const DEFAULT_CONFIG_MODE: &str = "unify"; -const OPTIONAL_CONFIG_MODE_SCHEMA_FRAGMENT: &str = - "config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent'))"; -/// Initialize client configuration state table. +/// Verify client storage after the config migration stream has run. pub async fn initialize_client_table(pool: &Pool) -> Result<()> { - tracing::debug!("Initializing client management table"); - - migrate_client_table_for_optional_config_mode(pool).await?; - - sqlx::query(&format!( - r#" - CREATE TABLE IF NOT EXISTS {table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - display_name TEXT, - identifier TEXT NOT NULL UNIQUE, - config_path TEXT, - -- Management mode: unify|hosted|transparent; NULL means use default mode - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - -- Transport protocol: auto|sse|stdio|streamable_http (default: auto) - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - -- Client version string (optional) - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT '{default_policy}' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT {default_backup_limit}, - capability_source TEXT NOT NULL DEFAULT '{default_capability_source}' CHECK ( - capability_source IN ('activated', 'profiles', 'custom') - ), - unify_route_mode TEXT NOT NULL DEFAULT 'broker_only' CHECK ( - unify_route_mode IN ('broker_only', 'server_level', 'capability_level') - ), - governance_kind TEXT NOT NULL DEFAULT '{default_governance_kind}' CHECK ( - governance_kind IN ('passive', 'active') - ), - connection_mode TEXT NOT NULL DEFAULT '{default_connection_mode}' CHECK ( - connection_mode IN ('local_config_detected', 'manual') - ), - registration_origin TEXT NOT NULL DEFAULT '{default_registration_origin}' CHECK ( - registration_origin IN ('manual', 'config_detection', 'runtime_initialize') - ), - runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), - template_identifier TEXT, - selected_profile_ids TEXT, - custom_profile_id TEXT, - approval_status TEXT NOT NULL DEFAULT 'approved' CHECK ( - approval_status IN ('pending', 'approved', 'suspended') - ), - attachment_state TEXT NOT NULL DEFAULT 'not_applicable' CHECK ( - attachment_state IN ('attached', 'detached', 'not_applicable') - ), - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - table = tables::CLIENT, - default_policy = DEFAULT_BACKUP_POLICY, - default_backup_limit = DEFAULT_BACKUP_LIMIT, - default_capability_source = DEFAULT_CAPABILITY_SOURCE, - default_governance_kind = DEFAULT_GOVERNANCE_KIND, - default_connection_mode = DEFAULT_CONNECTION_MODE, - default_registration_origin = DEFAULT_REGISTRATION_ORIGIN, - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create {} table: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to create {} table: {}", tables::CLIENT, e) - })?; - - ensure_column( - pool, - tables::CLIENT, - "capability_source", - "TEXT NOT NULL DEFAULT 'activated' CHECK (capability_source IN ('activated', 'profiles', 'custom'))", - ) - .await?; - ensure_column( - pool, - tables::CLIENT, - "unify_route_mode", - "TEXT NOT NULL DEFAULT 'broker_only' CHECK (unify_route_mode IN ('broker_only', 'server_level', 'capability_level'))", - ) - .await?; - ensure_column(pool, tables::CLIENT, "selected_profile_ids", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "custom_profile_id", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "display_name", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "config_path", "TEXT").await?; - ensure_column( - pool, - tables::CLIENT, - "governance_kind", - "TEXT NOT NULL DEFAULT 'passive' CHECK (governance_kind IN ('passive', 'active'))", - ) - .await?; - ensure_column( - pool, - tables::CLIENT, - "connection_mode", - "TEXT NOT NULL DEFAULT 'local_config_detected' CHECK (connection_mode IN ('local_config_detected', 'manual'))", - ) - .await?; - ensure_column( - pool, - tables::CLIENT, - "registration_origin", - "TEXT NOT NULL DEFAULT 'manual' CHECK (registration_origin IN ('manual', 'config_detection', 'runtime_initialize'))", - ) - .await?; - ensure_column( - pool, - tables::CLIENT, - "runtime_observed", - "INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1))", - ) - .await?; - ensure_column(pool, tables::CLIENT, "template_identifier", "TEXT").await?; - sqlx::query(&format!( - r#" - CREATE TABLE IF NOT EXISTS {table} ( - identifier TEXT PRIMARY KEY, - payload_json TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - table = CLIENT_TEMPLATE_RUNTIME_TABLE, - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create {} table: {}", CLIENT_TEMPLATE_RUNTIME_TABLE, e); - anyhow::anyhow!("Failed to create {} table: {}", CLIENT_TEMPLATE_RUNTIME_TABLE, e) - })?; - - sqlx::query(&format!( - r#" - CREATE TABLE IF NOT EXISTS {table} ( - client_identifier TEXT PRIMARY KEY, - merge_strategy TEXT NOT NULL CHECK (merge_strategy IN ('replace', 'deep_merge')), - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (client_identifier) REFERENCES {client_table}(identifier) ON DELETE CASCADE - ) - "#, - table = CLIENT_WRITEBACK_POLICY_TABLE, - client_table = tables::CLIENT, - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create {} table: {}", CLIENT_WRITEBACK_POLICY_TABLE, e); - anyhow::anyhow!("Failed to create {} table: {}", CLIENT_WRITEBACK_POLICY_TABLE, e) - })?; - - ensure_column( - pool, - tables::CLIENT, - "approval_status", - "TEXT NOT NULL DEFAULT 'approved' CHECK (approval_status IN ('pending', 'approved', 'suspended'))", - ) - .await?; - ensure_column(pool, tables::CLIENT, "template_id", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "template_version", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "approval_metadata", "TEXT").await?; - - // Template configuration fields (persisted from template at initialization) - ensure_column(pool, tables::CLIENT, "config_format", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "protocol_revision", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "container_type", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "container_keys", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "storage_kind", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "storage_adapter", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "storage_path_strategy", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "merge_strategy", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "keep_original_config", "INTEGER").await?; - ensure_column(pool, tables::CLIENT, "managed_source", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "transports", "TEXT").await?; - ensure_column(pool, tables::CLIENT, "config_file_parse", "TEXT").await?; - ensure_column( - pool, - tables::CLIENT, - "attachment_state", - "TEXT NOT NULL DEFAULT 'not_applicable' CHECK (attachment_state IN ('attached', 'detached', 'not_applicable'))", - ) - .await?; - - sqlx::query(&format!( - r#" - CREATE TABLE IF NOT EXISTS {table} ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - CHECK (key != '{default_mode_key}' OR value IN ('unify', 'hosted', 'transparent')) - ) - "#, - table = CLIENT_RUNTIME_SETTINGS_TABLE, - default_mode_key = DEFAULT_CONFIG_MODE_SETTING_KEY, - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create {} table: {}", CLIENT_RUNTIME_SETTINGS_TABLE, e); - anyhow::anyhow!("Failed to create {} table: {}", CLIENT_RUNTIME_SETTINGS_TABLE, e) - })?; - - sqlx::query(&format!( - "INSERT OR IGNORE INTO {table} (key, value) VALUES (?, ?)", - table = CLIENT_RUNTIME_SETTINGS_TABLE, - )) - .bind(DEFAULT_CONFIG_MODE_SETTING_KEY) - .bind(DEFAULT_CONFIG_MODE) - .execute(pool) - .await - .map_err(|e| { - tracing::error!( - "Failed to initialize {}.{}: {}", - CLIENT_RUNTIME_SETTINGS_TABLE, - DEFAULT_CONFIG_MODE_SETTING_KEY, - e - ); - anyhow::anyhow!( - "Failed to initialize {}.{}: {}", - CLIENT_RUNTIME_SETTINGS_TABLE, - DEFAULT_CONFIG_MODE_SETTING_KEY, - e - ) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET capability_source = ? WHERE capability_source IS NULL OR capability_source = ''", - table = tables::CLIENT, - )) - .bind(DEFAULT_CAPABILITY_SOURCE) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to backfill {} capability_source: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to backfill {} capability_source: {}", tables::CLIENT, e) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET config_mode = NULL WHERE config_mode = ''", - table = tables::CLIENT - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to normalize {} config_mode: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to normalize {} config_mode: {}", tables::CLIENT, e) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET display_name = name WHERE display_name IS NULL OR display_name = ''", - table = tables::CLIENT - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to backfill {} display_name: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to backfill {} display_name: {}", tables::CLIENT, e) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET governance_kind = CASE \ - WHEN config_mode IS NOT NULL AND TRIM(config_mode) <> '' THEN 'active' \ - WHEN transport IS NOT NULL AND TRIM(transport) <> '' AND transport <> 'auto' THEN 'active' \ - WHEN client_version IS NOT NULL AND TRIM(client_version) <> '' THEN 'active' \ - WHEN backup_policy IS NOT NULL AND backup_policy <> 'keep_n' THEN 'active' \ -WHEN backup_limit IS NOT NULL AND backup_limit <> {default_backup_limit} THEN 'active' \ - WHEN capability_source IS NOT NULL AND capability_source <> 'activated' THEN 'active' \ - WHEN selected_profile_ids IS NOT NULL AND TRIM(selected_profile_ids) <> '' THEN 'active' \ - WHEN custom_profile_id IS NOT NULL AND TRIM(custom_profile_id) <> '' THEN 'active' \ - WHEN approval_status = 'suspended' THEN 'active' \ - ELSE ? END \ - WHERE governance_kind IS NULL OR governance_kind = ''", - table = tables::CLIENT, - default_backup_limit = DEFAULT_BACKUP_LIMIT, - )) - .bind(DEFAULT_GOVERNANCE_KIND) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to backfill {} governance_kind: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to backfill {} governance_kind: {}", tables::CLIENT, e) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET template_identifier = identifier WHERE template_identifier IS NULL OR template_identifier = ''", - table = tables::CLIENT - )) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to backfill {} template_identifier: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to backfill {} template_identifier: {}", tables::CLIENT, e) - })?; - - sqlx::query(&format!( - "UPDATE {table} SET \ - runtime_observed = CASE \ - WHEN connection_mode = 'remote_http' THEN 1 \ - ELSE COALESCE(runtime_observed, 0) \ - END, \ - registration_origin = CASE \ - WHEN connection_mode = 'remote_http' THEN 'runtime_initialize' \ - WHEN registration_origin IS NULL OR registration_origin = '' OR registration_origin = ? THEN \ - CASE \ - WHEN COALESCE(runtime_observed, 0) = 1 THEN 'runtime_initialize' \ - WHEN config_path IS NOT NULL AND TRIM(config_path) <> '' THEN 'config_detection' \ - ELSE ? \ - END \ - ELSE registration_origin \ - END, \ - connection_mode = CASE \ - WHEN config_path IS NOT NULL AND TRIM(config_path) <> '' \ - THEN 'local_config_detected' \ - ELSE 'manual' \ - END", - table = tables::CLIENT, - )) - .bind(DEFAULT_REGISTRATION_ORIGIN) - .bind(DEFAULT_REGISTRATION_ORIGIN) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to normalize {} connection state: {}", tables::CLIENT, e); - anyhow::anyhow!("Failed to normalize {} connection state: {}", tables::CLIENT, e) - })?; - - migrate_client_table_constraints(pool).await?; - - tracing::debug!("{} table initialized", tables::CLIENT); + mcpmate_migrations::verify_config_database(pool).await?; + sqlx::query("SELECT 1 FROM client_runtime_settings LIMIT 1") + .execute(pool) + .await + .map_err(|error| anyhow::anyhow!("client storage was not migrated: {error}"))?; Ok(()) } pub async fn resolve_default_client_config_mode(pool: &Pool) -> Result { crate::system::settings::get_default_config_mode(pool) .await - .map_err(|err| anyhow::anyhow!(err.to_string())) + .map_err(|error| anyhow::anyhow!(error.to_string())) } pub fn effective_client_config_mode<'a>( @@ -380,639 +41,47 @@ pub async fn set_default_client_config_mode( ) -> Result<()> { crate::system::settings::set_default_config_mode(pool, mode) .await - .map_err(|err| anyhow::anyhow!(err.to_string())) -} - -async fn migrate_client_table_for_optional_config_mode(pool: &Pool) -> Result<()> { - let table_exists: Option = sqlx::query_scalar(&format!( - "SELECT name FROM sqlite_master WHERE type='table' AND name='{}'", - tables::CLIENT - )) - .fetch_optional(pool) - .await?; - - if table_exists.is_none() { - return Ok(()); - } - - let create_sql: Option = sqlx::query_scalar(&format!( - "SELECT sql FROM sqlite_master WHERE type='table' AND name='{}'", - tables::CLIENT - )) - .fetch_optional(pool) - .await?; - - let Some(create_sql) = create_sql else { - return Ok(()); - }; - - if create_sql.contains(OPTIONAL_CONFIG_MODE_SCHEMA_FRAGMENT) - && !create_sql.contains("config_mode TEXT NOT NULL DEFAULT 'hosted'") - { - return Ok(()); - } - - tracing::info!( - "Migrating {} table to allow unset config_mode for default-mode fallback", - tables::CLIENT - ); - - let migration_result = async { - let mut tx = pool.begin().await?; - let temp_table = format!("{}_config_mode_nullable", tables::CLIENT); - - sqlx::query(&format!( - r#" - CREATE TABLE {temp_table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - identifier TEXT NOT NULL UNIQUE, - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT '{default_policy}' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT {default_backup_limit}, - capability_source TEXT NOT NULL DEFAULT '{default_capability_source}' CHECK ( - capability_source IN ('activated', 'profiles', 'custom') - ), - selected_profile_ids TEXT, - custom_profile_id TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - temp_table = temp_table, - default_policy = DEFAULT_BACKUP_POLICY, - default_backup_limit = DEFAULT_BACKUP_LIMIT, - default_capability_source = DEFAULT_CAPABILITY_SOURCE, - )) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!( - r#" - INSERT INTO {temp_table} ( - id, name, identifier, config_mode, transport, client_version, - backup_policy, backup_limit, capability_source, selected_profile_ids, - custom_profile_id, created_at, updated_at - ) - SELECT - id, name, identifier, - config_mode, transport, client_version, - backup_policy, backup_limit, capability_source, selected_profile_ids, - custom_profile_id, created_at, updated_at - FROM {table} - "#, - temp_table = temp_table, - table = tables::CLIENT, - )) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!("DROP TABLE {table}", table = tables::CLIENT)) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!( - "ALTER TABLE {temp_table} RENAME TO {table}", - temp_table = temp_table, - table = tables::CLIENT, - )) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok::<(), sqlx::Error>(()) - } - .await; - - match migration_result { - Ok(()) => Ok(()), - Err(error) => Err(anyhow::anyhow!(error)), - } + .map_err(|error| anyhow::anyhow!(error.to_string())) } -async fn migrate_client_table_constraints(pool: &Pool) -> Result<()> { - let create_sql: Option = sqlx::query_scalar(&format!( - "SELECT sql FROM sqlite_master WHERE type='table' AND name='{}'", - tables::CLIENT - )) - .fetch_optional(pool) - .await?; - - let Some(create_sql) = create_sql else { - return Ok(()); - }; - - let needs_sse_transport = create_sql.contains("transport IN ('auto', 'stdio', 'streamable_http')"); - let needs_connection_mode_cleanup = create_sql.contains("'remote_http'"); - - if !needs_sse_transport && !needs_connection_mode_cleanup { - return Ok(()); - } - - tracing::info!("Migrating {} client table constraints", tables::CLIENT); - - let transports_source_expression = if column_exists(pool, tables::CLIENT, "format_rules").await? { - "COALESCE(NULLIF(transports, ''), format_rules)" - } else { - "transports" - }; - - let migration_result = async { - let mut tx = pool.begin().await?; - let temp_table = format!("{}_constraints_current", tables::CLIENT); - - sqlx::query(&format!( - r#" - CREATE TABLE {temp_table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - display_name TEXT, - identifier TEXT NOT NULL UNIQUE, - config_path TEXT, - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT '{default_policy}' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT {default_backup_limit}, - capability_source TEXT NOT NULL DEFAULT '{default_capability_source}' CHECK ( - capability_source IN ('activated', 'profiles', 'custom') - ), - unify_route_mode TEXT NOT NULL DEFAULT 'broker_only' CHECK ( - unify_route_mode IN ('broker_only', 'server_level', 'capability_level') - ), - governance_kind TEXT NOT NULL DEFAULT '{default_governance_kind}' CHECK ( - governance_kind IN ('passive', 'active') - ), - connection_mode TEXT NOT NULL DEFAULT '{default_connection_mode}' CHECK ( - connection_mode IN ('local_config_detected', 'manual') - ), - registration_origin TEXT NOT NULL DEFAULT '{default_registration_origin}' CHECK ( - registration_origin IN ('manual', 'config_detection', 'runtime_initialize') - ), - runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), - template_identifier TEXT, - selected_profile_ids TEXT, - custom_profile_id TEXT, - approval_status TEXT NOT NULL DEFAULT 'approved' CHECK ( - approval_status IN ('pending', 'approved', 'suspended') - ), - template_id TEXT, - template_version TEXT, - approval_metadata TEXT, - config_format TEXT, - protocol_revision TEXT, - container_type TEXT, - container_keys TEXT, - storage_kind TEXT, - storage_adapter TEXT, - storage_path_strategy TEXT, - merge_strategy TEXT, - keep_original_config INTEGER, - managed_source TEXT, - transports TEXT, - config_file_parse TEXT, - attachment_state TEXT NOT NULL DEFAULT 'not_applicable' CHECK ( - attachment_state IN ('attached', 'detached', 'not_applicable') - ), - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - temp_table = temp_table, - default_policy = DEFAULT_BACKUP_POLICY, - default_backup_limit = DEFAULT_BACKUP_LIMIT, - default_capability_source = DEFAULT_CAPABILITY_SOURCE, - default_governance_kind = DEFAULT_GOVERNANCE_KIND, - default_connection_mode = DEFAULT_CONNECTION_MODE, - default_registration_origin = DEFAULT_REGISTRATION_ORIGIN, - )) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!( - r#" - INSERT INTO {temp_table} ( - id, name, display_name, identifier, config_path, config_mode, transport, - client_version, backup_policy, backup_limit, capability_source, unify_route_mode, governance_kind, - connection_mode, registration_origin, runtime_observed, - template_identifier, selected_profile_ids, custom_profile_id, - approval_status, template_id, template_version, approval_metadata, config_format, protocol_revision, - container_type, container_keys, storage_kind, storage_adapter, storage_path_strategy, - merge_strategy, keep_original_config, managed_source, transports, config_file_parse, - attachment_state, - created_at, updated_at - ) - SELECT - id, name, display_name, identifier, config_path, config_mode, transport, - client_version, backup_policy, backup_limit, capability_source, unify_route_mode, governance_kind, - CASE - WHEN config_path IS NOT NULL AND TRIM(config_path) <> '' - THEN 'local_config_detected' - ELSE 'manual' - END, - CASE - WHEN connection_mode = 'remote_http' THEN 'runtime_initialize' - ELSE registration_origin - END, - CASE - WHEN connection_mode = 'remote_http' THEN 1 - ELSE runtime_observed - END, - template_identifier, selected_profile_ids, custom_profile_id, - approval_status, template_id, template_version, approval_metadata, config_format, protocol_revision, - container_type, container_keys, storage_kind, storage_adapter, storage_path_strategy, - merge_strategy, keep_original_config, managed_source, {transports_source_expression}, config_file_parse, - attachment_state, - created_at, updated_at - FROM {table} - "#, - temp_table = temp_table, - table = tables::CLIENT, - transports_source_expression = transports_source_expression, - )) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!("DROP TABLE {table}", table = tables::CLIENT)) - .execute(&mut *tx) - .await?; - - sqlx::query(&format!( - "ALTER TABLE {temp_table} RENAME TO {table}", - temp_table = temp_table, - table = tables::CLIENT, - )) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok::<(), sqlx::Error>(()) - } - .await; - - match migration_result { - Ok(()) => Ok(()), - Err(error) => Err(anyhow::anyhow!(error)), - } -} - -async fn ensure_column( - pool: &Pool, - table: &str, - column: &str, - definition: &str, -) -> Result<()> { - let stmt = format!( - "ALTER TABLE {table} ADD COLUMN {column} {definition}", - table = table, - column = column, - definition = definition - ); - - match sqlx::query(&stmt).execute(pool).await { - Ok(_) => { - tracing::debug!("Added column {}.{}", table, column); - Ok(()) - } - Err(sqlx::Error::Database(db_err)) if db_err.message().contains("duplicate column name") => { - tracing::trace!("Column {}.{} already exists", table, column); - Ok(()) - } - Err(e) => { - tracing::error!("Failed to add column {}.{}: {}", table, column, e); - Err(anyhow::anyhow!("Failed to add column {}.{}: {}", table, column, e)) - } - } -} - -async fn column_exists( - pool: &Pool, - table: &str, - column: &str, -) -> Result { - let rows: Vec = sqlx::query_scalar(&format!( - "SELECT name FROM pragma_table_info('{}')", - table.replace('\'', "''") - )) - .fetch_all(pool) - .await - .map_err(|e| anyhow::anyhow!("Failed to inspect {} columns: {}", table, e))?; - - Ok(rows.into_iter().any(|name| name == column)) -} - -/// Ensures the on-disk system settings store exists (JSON). Does not create or touch any SQLite -/// `system_settings` table; schema changes for existing installs are handled out-of-band. +/// Ensures the on-disk system settings store exists. It has no durable SQLite schema ownership. pub async fn initialize_system_settings(pool: &Pool) -> Result<()> { - tracing::debug!("Initializing system settings store"); - crate::system::settings::initialize_settings_file(pool) .await - .map_err(|err| anyhow::anyhow!(err.to_string())) + .map_err(|error| anyhow::anyhow!(error.to_string())) } #[cfg(test)] mod tests { - use super::{ - initialize_client_table, initialize_system_settings, resolve_default_client_config_mode, - set_default_client_config_mode, - }; - use crate::clients::models::FirstContactBehavior; - use crate::common::constants::database::tables; + use super::*; use sqlx::sqlite::SqlitePoolOptions; - async fn setup_raw_pool() -> sqlx::Pool { + async fn pool() -> Pool { SqlitePoolOptions::new() .max_connections(1) .connect("sqlite::memory:") .await - .expect("failed to create sqlite pool") - } - - async fn setup_pool() -> sqlx::Pool { - let pool = setup_raw_pool().await; - - initialize_client_table(&pool) - .await - .expect("failed to initialize client tables"); - initialize_system_settings(&pool) - .await - .expect("failed to initialize system settings store"); - - pool - } - - #[tokio::test] - async fn default_client_config_mode_defaults_to_unify() { - let pool = setup_pool().await; - - let mode = resolve_default_client_config_mode(&pool) - .await - .expect("failed to resolve default client config mode"); - - assert_eq!(mode, "unify"); + .unwrap() } #[tokio::test] - async fn default_client_config_mode_can_be_updated_and_read_back() { - let pool = setup_pool().await; - - set_default_client_config_mode(&pool, "unify") - .await - .expect("failed to persist unify mode"); - assert_eq!( - resolve_default_client_config_mode(&pool) - .await - .expect("failed to resolve unify mode"), - "unify" - ); - - set_default_client_config_mode(&pool, "transparent") - .await - .expect("failed to persist transparent mode"); - assert_eq!( - resolve_default_client_config_mode(&pool) + async fn initializes_current_client_schema_through_migrations() { + let pool = pool().await; + crate::test_helpers::prepare_config_database(&pool).await; + initialize_client_table(&pool).await.unwrap(); + let mode: String = + sqlx::query_scalar("SELECT value FROM client_runtime_settings WHERE key = 'default_config_mode'") + .fetch_one(&pool) .await - .expect("failed to resolve transparent mode"), - "transparent" - ); - } - - #[tokio::test] - async fn initialize_system_settings_file_creates_default_behavior() { - let pool = setup_pool().await; - - let settings = crate::system::settings::get_settings(&pool) - .await - .expect("failed to read system settings"); - - assert_eq!(settings.first_contact_behavior, FirstContactBehavior::Review); - assert_eq!(settings.api_port, crate::common::constants::ports::API_PORT); - assert_eq!(settings.mcp_port, crate::common::constants::ports::MCP_PORT); - assert_eq!(settings.inspector_timeout_ms, 8_000); - assert_eq!(settings.default_config_mode, "unify"); - } - - #[tokio::test] - async fn initialize_client_table_normalizes_legacy_remote_http_rows() { - let pool = setup_raw_pool().await; - - sqlx::query(&format!( - r#" - CREATE TABLE {table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - identifier TEXT NOT NULL UNIQUE, - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT 'keep_n' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT 5, - connection_mode TEXT NOT NULL DEFAULT 'local_config_detected' CHECK ( - connection_mode IN ('local_config_detected', 'remote_http', 'manual') - ), - registration_origin TEXT NOT NULL DEFAULT 'manual' CHECK ( - registration_origin IN ('manual', 'config_detection', 'runtime_initialize') - ), - runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - table = tables::CLIENT, - )) - .execute(&pool) - .await - .expect("create legacy client table"); - - sqlx::query(&format!( - "INSERT INTO {table} (id, name, identifier, connection_mode) VALUES (?, ?, ?, ?)", - table = tables::CLIENT, - )) - .bind("client_legacy") - .bind("Legacy Runtime") - .bind("legacy.runtime") - .bind("remote_http") - .execute(&pool) - .await - .expect("insert legacy remote_http row"); - - initialize_client_table(&pool).await.expect("initialize client table"); - - let (connection_mode, registration_origin, runtime_observed, unify_route_mode): (String, String, i64, String) = - sqlx::query_as(&format!( - "SELECT connection_mode, registration_origin, runtime_observed, unify_route_mode FROM {table} WHERE identifier = ?", - table = tables::CLIENT, - )) - .bind("legacy.runtime") - .fetch_one(&pool) - .await - .expect("fetch normalized row"); - - assert_eq!(connection_mode, "manual"); - assert_eq!(registration_origin, "runtime_initialize"); - assert_eq!(runtime_observed, 1); - assert_eq!(unify_route_mode, "broker_only"); - - let create_sql: String = sqlx::query_scalar(&format!( - "SELECT sql FROM sqlite_master WHERE type='table' AND name='{}'", - tables::CLIENT, - )) - .fetch_one(&pool) - .await - .expect("fetch normalized schema"); - - assert!(!create_sql.contains("'remote_http'")); - } - - #[tokio::test] - async fn initialize_client_table_derives_local_mode_from_legacy_config_path() { - let pool = setup_raw_pool().await; - - sqlx::query(&format!( - r#" - CREATE TABLE {table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - identifier TEXT NOT NULL UNIQUE, - config_path TEXT, - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT 'keep_n' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT 5, - connection_mode TEXT NOT NULL DEFAULT 'manual' CHECK ( - connection_mode IN ('local_config_detected', 'manual') - ), - registration_origin TEXT NOT NULL DEFAULT 'manual' CHECK ( - registration_origin IN ('manual', 'config_detection', 'runtime_initialize') - ), - runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - table = tables::CLIENT, - )) - .execute(&pool) - .await - .expect("create legacy client table"); - - let legacy_config_path = "/tmp/mcpmate-legacy-client.json"; - sqlx::query(&format!( - "INSERT INTO {table} (id, name, identifier, config_path, connection_mode) VALUES (?, ?, ?, ?, ?)", - table = tables::CLIENT, - )) - .bind("client_legacy_path") - .bind("Legacy Config Path") - .bind("legacy.path") - .bind(legacy_config_path) - .bind("manual") - .execute(&pool) - .await - .expect("insert legacy config path row"); - - initialize_client_table(&pool).await.expect("initialize client table"); - - let (connection_mode, registration_origin, config_path): (String, String, String) = sqlx::query_as(&format!( - "SELECT connection_mode, registration_origin, config_path FROM {table} WHERE identifier = ?", - table = tables::CLIENT, - )) - .bind("legacy.path") - .fetch_one(&pool) - .await - .expect("fetch normalized row"); - - assert_eq!(connection_mode, "local_config_detected"); - assert_eq!(registration_origin, "config_detection"); - assert_eq!(config_path, legacy_config_path); + .unwrap(); + assert_eq!(mode, "unify"); } #[tokio::test] - async fn initialize_client_table_preserves_legacy_format_rules_during_constraint_rebuild() { - let pool = setup_raw_pool().await; - - sqlx::query(&format!( - r#" - CREATE TABLE {table} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - identifier TEXT NOT NULL UNIQUE, - config_path TEXT, - config_mode TEXT CHECK (config_mode IN ('unify','hosted','transparent')), - transport TEXT NOT NULL DEFAULT 'auto' CHECK ( - transport IN ('auto', 'sse', 'stdio', 'streamable_http') - ), - client_version TEXT, - backup_policy TEXT NOT NULL DEFAULT 'keep_n' CHECK ( - backup_policy IN ('keep_last', 'keep_n', 'off') - ), - backup_limit INTEGER DEFAULT 5, - connection_mode TEXT NOT NULL DEFAULT 'local_config_detected' CHECK ( - connection_mode IN ('local_config_detected', 'remote_http', 'manual') - ), - registration_origin TEXT NOT NULL DEFAULT 'manual' CHECK ( - registration_origin IN ('manual', 'config_detection', 'runtime_initialize') - ), - runtime_observed INTEGER NOT NULL DEFAULT 0 CHECK (runtime_observed IN (0, 1)), - format_rules TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - table = tables::CLIENT, - )) - .execute(&pool) - .await - .expect("create legacy client table"); - - let legacy_format_rules = r#"{"stdio":{"command_field":"command","args_field":"args","env_field":"env"}}"#; - let legacy_config_path = "/tmp/mcpmate-legacy-rules.json"; - sqlx::query(&format!( - "INSERT INTO {table} (id, name, identifier, config_path, connection_mode, format_rules) VALUES (?, ?, ?, ?, ?, ?)", - table = tables::CLIENT, - )) - .bind("client_legacy_rules") - .bind("Legacy Format Rules") - .bind("legacy.rules") - .bind(legacy_config_path) - .bind("manual") - .bind(legacy_format_rules) - .execute(&pool) - .await - .expect("insert legacy format rules row"); - - initialize_client_table(&pool).await.expect("initialize client table"); - - let (connection_mode, transports): (String, String) = sqlx::query_as(&format!( - "SELECT connection_mode, transports FROM {table} WHERE identifier = ?", - table = tables::CLIENT, - )) - .bind("legacy.rules") - .fetch_one(&pool) - .await - .expect("fetch migrated row"); - - assert_eq!(connection_mode, "local_config_detected"); - assert_eq!(transports, legacy_format_rules); + async fn keeps_mode_helpers_stable() { + assert_eq!(effective_client_config_mode(Some(" hosted "), "unify"), "hosted"); + assert_eq!(effective_client_config_mode(Some(" "), "unify"), "unify"); + assert!(is_managed_client_config_mode("unify")); + assert!(!is_managed_client_config_mode("transparent")); } } diff --git a/backend/src/config/database.rs b/backend/src/config/database.rs index a44669b0..62f66051 100644 --- a/backend/src/config/database.rs +++ b/backend/src/config/database.rs @@ -96,8 +96,10 @@ impl Database { pub async fn new() -> Result { // Get database URL from environment or use default in user directory let database_url = get_database_url()?; + let connection_options = + sqlite_connect_options(&database_url).context("Failed to configure SQLite connection options")?; let db_path = if database_url.starts_with("sqlite:") { - PathBuf::from(database_url.strip_prefix("sqlite:").unwrap()) + connection_options.get_filename().to_path_buf() } else { global_paths().database_path() }; @@ -145,8 +147,6 @@ impl Database { // Connect to the database tracing::debug!("Connecting to database with max 5 connections"); - let connection_options = - sqlite_connect_options(&database_url).context("Failed to configure SQLite connection options")?; let pool = match SqlitePoolOptions::new() .max_connections(5) .connect_with(connection_options) @@ -162,6 +162,18 @@ impl Database { } }; + if let Some(backup_path) = mcpmate_migrations::prepare_config_database( + &pool, + mcpmate_migrations::DatabaseSource::File { + path: &db_path, + existed_before_open: db_exists, + }, + ) + .await? + { + tracing::info!(path = %backup_path.display(), "Created database backup before migration"); + } + // Initialize naming store as early as possible so other components can rely on it naming::initialize(pool.clone()); @@ -285,6 +297,17 @@ mod tests { use super::*; use sqlx::sqlite::SqlitePoolOptions; + #[test] + fn sqlite_connection_options_resolve_file_path() { + for (database_url, expected) in [ + ("sqlite://data.db", Path::new("data.db")), + ("sqlite://data.db?mode=rwc", Path::new("data.db")), + ("sqlite://data%20set.db", Path::new("data set.db")), + ] { + assert_eq!(sqlite_connect_options(database_url).unwrap().get_filename(), expected); + } + } + #[tokio::test] async fn main_database_connections_enable_wal_busy_timeout_and_foreign_keys() { let directory = tempfile::tempdir().unwrap(); @@ -324,6 +347,7 @@ mod tests { .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; initialize_capability_catalog(&pool).await.unwrap(); for table in [ @@ -350,6 +374,7 @@ mod tests { .connect("sqlite::memory:") .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; initialize_capability_catalog(&pool).await.unwrap(); sqlx::query( r#" diff --git a/backend/src/config/import.rs b/backend/src/config/import.rs index e96c1018..2bb3f4c3 100644 --- a/backend/src/config/import.rs +++ b/backend/src/config/import.rs @@ -186,6 +186,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; run_initialization(&pool).await.expect("initialize schema"); (temp_dir, pool) diff --git a/backend/src/config/initialization.rs b/backend/src/config/initialization.rs index 23984347..f66e5a90 100644 --- a/backend/src/config/initialization.rs +++ b/backend/src/config/initialization.rs @@ -15,6 +15,8 @@ use super::database::initialize_capability_catalog; pub async fn run_initialization(pool: &Pool) -> Result<()> { tracing::info!("Running database initialization"); + mcpmate_migrations::verify_config_database(pool).await?; + // Initialize server-related tables tracing::debug!("Initializing server-related tables"); initialize_server_tables(pool).await?; diff --git a/backend/src/config/llm/crud.rs b/backend/src/config/llm/crud.rs index 00912b98..99b60ae2 100644 --- a/backend/src/config/llm/crud.rs +++ b/backend/src/config/llm/crud.rs @@ -190,6 +190,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect sqlite"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_llm_tables(&pool).await.expect("init llm tables"); pool } diff --git a/backend/src/config/llm/init.rs b/backend/src/config/llm/init.rs index 90fe3d43..5b0e6894 100644 --- a/backend/src/config/llm/init.rs +++ b/backend/src/config/llm/init.rs @@ -1,41 +1,7 @@ use anyhow::Result; use sqlx::{Pool, Sqlite}; -use crate::config::server::init::ensure_column; - pub async fn initialize_llm_tables(pool: &Pool) -> Result<()> { - tracing::debug!("Initializing LLM provider database tables"); - create_llm_provider_table(pool).await?; - tracing::debug!("LLM provider database tables initialized successfully"); - Ok(()) -} - -async fn create_llm_provider_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating llm_provider table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS llm_provider ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - provider_type TEXT NOT NULL, - base_url TEXT NOT NULL, - model_id TEXT NOT NULL, - secret_alias TEXT, - default_params_json TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create llm_provider table: {}", e); - anyhow::anyhow!("Failed to create llm_provider table: {}", e) - })?; - - tracing::debug!("llm_provider table created or already exists"); - ensure_column(pool, "llm_provider", "is_default", "BOOLEAN NOT NULL DEFAULT 0").await?; + mcpmate_migrations::verify_config_database(pool).await?; Ok(()) } diff --git a/backend/src/config/profile/capability_ref.rs b/backend/src/config/profile/capability_ref.rs index 5987177e..84a19b65 100644 --- a/backend/src/config/profile/capability_ref.rs +++ b/backend/src/config/profile/capability_ref.rs @@ -275,6 +275,7 @@ mod tests { .connect("sqlite::memory:") .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); @@ -331,6 +332,7 @@ mod tests { .connect("sqlite::memory:") .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); diff --git a/backend/src/config/profile/init.rs b/backend/src/config/profile/init.rs index 27eedc6d..bd5da5c5 100644 --- a/backend/src/config/profile/init.rs +++ b/backend/src/config/profile/init.rs @@ -1,557 +1,14 @@ -// Profile database initialization -// Contains functions for initializing profile-related database tables - use anyhow::Result; use sqlx::{Pool, Sqlite}; -use tracing; use crate::common::constants::database::tables; -/// Initialize all profile-related database tables +/// Verify Profile authoring storage after the config migration stream has run. pub async fn initialize_profile_tables(pool: &Pool) -> Result<()> { - tracing::debug!("Initializing profile-related database tables"); - - create_profile_table(pool).await?; - create_profile_server_relationships_table(pool).await?; - create_server_tools_table(pool).await?; - create_server_tools_index(pool).await?; - create_server_prompts_table(pool).await?; - create_server_prompts_index(pool).await?; - create_server_resources_table(pool).await?; - create_server_resources_index(pool).await?; - create_server_resource_templates_table(pool).await?; - create_server_resource_templates_index(pool).await?; - create_server_issued_resources_table(pool).await?; - create_server_issued_resources_index(pool).await?; - create_profile_capability_refs_table(pool).await?; - create_direct_exposure_refs_table(pool).await?; - create_direct_exposure_servers_table(pool).await?; - - verify_profile_tables(pool).await?; - - tracing::debug!("Profile-related database tables initialized successfully"); - Ok(()) -} - -/// Create profile table if it doesn't exist -async fn create_profile_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating {} table if it doesn't exist", tables::PROFILE); - - let create_sql = format!( - r#" - CREATE TABLE IF NOT EXISTS {} ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - description TEXT, - type TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'user', - multi_select BOOLEAN NOT NULL DEFAULT 0, - priority INTEGER NOT NULL DEFAULT 0, - is_active BOOLEAN NOT NULL DEFAULT 0, - is_default BOOLEAN NOT NULL DEFAULT 0, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - tables::PROFILE - ); - - sqlx::query(&create_sql).execute(pool).await.map_err(|e| { - tracing::error!("Failed to create {} table: {}", tables::PROFILE, e); - anyhow::anyhow!("Failed to create {} table: {}", tables::PROFILE, e) - })?; - - tracing::debug!("{} table created or already exists", tables::PROFILE); - Ok(()) -} - -/// Create profile-level server relationships if they do not exist. -async fn create_profile_server_relationships_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating profile_server_relationships table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS profile_server_relationships ( - profile_id TEXT NOT NULL, - server_id TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - new_ref_policy TEXT NOT NULL CHECK (new_ref_policy IN ('follow', 'review')), - FOREIGN KEY (profile_id) REFERENCES profile (id) ON DELETE CASCADE, - PRIMARY KEY(profile_id, server_id) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create profile_server_relationships table: {}", e); - anyhow::anyhow!("Failed to create profile_server_relationships table: {}", e) - })?; - - tracing::debug!("profile_server_relationships table created or already exists"); - Ok(()) -} - -/// Create server_tools table if it doesn't exist -async fn create_server_tools_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_tools table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_tools ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - tool_name TEXT NOT NULL, - unique_name TEXT NOT NULL, - description TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, tool_name), - UNIQUE(unique_name) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_tools table: {}", e); - anyhow::anyhow!("Failed to create server_tools table: {}", e) - })?; - - tracing::debug!("server_tools table created or already exists"); - Ok(()) -} - -/// Create indexes on server_tools table for performance -async fn create_server_tools_index(pool: &Pool) -> Result<()> { - tracing::debug!("Creating indexes on server_tools table for performance"); - - // Index for lookup by server_id and tool_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_tools_lookup - ON server_tools(server_id, tool_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_tools lookup: {}", e); - anyhow::anyhow!("Failed to create index on server_tools lookup: {}", e) - })?; - - // Index for lookup by unique_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_tools_unique_name - ON server_tools(unique_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_tools unique_name: {}", e); - anyhow::anyhow!("Failed to create index on server_tools unique_name: {}", e) - })?; - - // Index for lookup by server_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_tools_server_name - ON server_tools(server_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_tools server_name: {}", e); - anyhow::anyhow!("Failed to create index on server_tools server_name: {}", e) - })?; - - tracing::debug!("Indexes on server_tools table created or already exists"); - Ok(()) + mcpmate_migrations::verify_config_database(pool).await?; + verify_profile_tables(pool).await } -/// Create server_prompts table if it doesn't exist (shadow table for indexing) -async fn create_server_prompts_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_prompts table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_prompts ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - prompt_name TEXT NOT NULL, - unique_name TEXT NOT NULL, - description TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, prompt_name), - UNIQUE(unique_name) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_prompts table: {}", e); - anyhow::anyhow!("Failed to create server_prompts table: {}", e) - })?; - - tracing::debug!("server_prompts table created or already exists"); - Ok(()) -} - -/// Create indexes on server_prompts table for performance -async fn create_server_prompts_index(pool: &Pool) -> Result<()> { - tracing::debug!("Creating indexes on server_prompts table for performance"); - - // Index for lookup by server_id and prompt_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_prompts_lookup - ON server_prompts(server_id, prompt_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_prompts lookup: {}", e); - anyhow::anyhow!("Failed to create index on server_prompts lookup: {}", e) - })?; - - // Index for lookup by unique_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_prompts_unique_name - ON server_prompts(unique_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_prompts unique_name: {}", e); - anyhow::anyhow!("Failed to create index on server_prompts unique_name: {}", e) - })?; - - // Index for lookup by server_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_prompts_server_name - ON server_prompts(server_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_prompts server_name: {}", e); - anyhow::anyhow!("Failed to create index on server_prompts server_name: {}", e) - })?; - - tracing::debug!("Indexes on server_prompts table created or already exists"); - Ok(()) -} - -/// Create server_resources table if it doesn't exist (shadow table for indexing) -async fn create_server_resources_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_resources table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_resources ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - resource_uri TEXT NOT NULL, - unique_uri TEXT NOT NULL, - name TEXT, - description TEXT, - mime_type TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, resource_uri), - UNIQUE(unique_uri) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_resources table: {}", e); - anyhow::anyhow!("Failed to create server_resources table: {}", e) - })?; - - tracing::debug!("server_resources table created or already exists"); - Ok(()) -} - -/// Create indexes on server_resources table for performance -async fn create_server_resources_index(pool: &Pool) -> Result<()> { - tracing::debug!("Creating indexes on server_resources table for performance"); - - // Index for lookup by server_id and resource_uri - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resources_lookup - ON server_resources(server_id, resource_uri) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resources lookup: {}", e); - anyhow::anyhow!("Failed to create index on server_resources lookup: {}", e) - })?; - - // Index for lookup by unique_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resources_unique_uri - ON server_resources(unique_uri) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resources unique_uri: {}", e); - anyhow::anyhow!("Failed to create index on server_resources unique_uri: {}", e) - })?; - - // Index for lookup by server_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resources_server_name - ON server_resources(server_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resources server_name: {}", e); - anyhow::anyhow!("Failed to create index on server_resources server_name: {}", e) - })?; - - tracing::debug!("Indexes on server_resources table created or already exists"); - Ok(()) -} - -/// Create server_resource_templates table if it doesn't exist (shadow table for indexing) -async fn create_server_resource_templates_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_resource_templates table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_resource_templates ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - uri_template TEXT NOT NULL, - unique_name TEXT NOT NULL, - route_uri TEXT, - name TEXT NOT NULL, - description TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, uri_template), - UNIQUE(unique_name), - UNIQUE(route_uri) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_resource_templates table: {}", e); - anyhow::anyhow!("Failed to create server_resource_templates table: {}", e) - })?; - - tracing::debug!("server_resource_templates table created or already exists"); - Ok(()) -} - -/// Create indexes on server_resource_templates table for performance -async fn create_server_resource_templates_index(pool: &Pool) -> Result<()> { - tracing::debug!("Creating indexes on server_resource_templates table for performance"); - - // Index for lookup by server_id and uri_template - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resource_templates_lookup - ON server_resource_templates(server_id, uri_template) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resource_templates lookup: {}", e); - anyhow::anyhow!("Failed to create index on server_resource_templates lookup: {}", e) - })?; - - // Index for lookup by unique_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resource_templates_unique_name - ON server_resource_templates(unique_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resource_templates unique_name: {}", e); - anyhow::anyhow!("Failed to create index on server_resource_templates unique_name: {}", e) - })?; - - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resource_templates_route_uri - ON server_resource_templates(route_uri) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resource_templates route_uri: {}", e); - anyhow::anyhow!("Failed to create index on server_resource_templates route_uri: {}", e) - })?; - - // Index for lookup by server_name - sqlx::query( - r#" - CREATE INDEX IF NOT EXISTS idx_server_resource_templates_server_name - ON server_resource_templates(server_name) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create index on server_resource_templates server_name: {}", e); - anyhow::anyhow!("Failed to create index on server_resource_templates server_name: {}", e) - })?; - - tracing::debug!("Indexes on server_resource_templates table created or already exists"); - Ok(()) -} - -async fn create_server_issued_resources_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_issued_resources table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_issued_resources ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - resource_uri TEXT NOT NULL, - unique_uri TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, resource_uri), - UNIQUE(unique_uri) - ) - "#, - ) - .execute(pool) - .await - .map_err(|error| { - tracing::error!("Failed to create server_issued_resources table: {}", error); - anyhow::anyhow!("Failed to create server_issued_resources table: {}", error) - })?; - - Ok(()) -} - -async fn create_server_issued_resources_index(pool: &Pool) -> Result<()> { - for statement in [ - r#" - CREATE INDEX IF NOT EXISTS idx_server_issued_resources_lookup - ON server_issued_resources(server_id, resource_uri) - "#, - r#" - CREATE INDEX IF NOT EXISTS idx_server_issued_resources_unique_uri - ON server_issued_resources(unique_uri) - "#, - ] { - sqlx::query(statement).execute(pool).await.map_err(|error| { - tracing::error!("Failed to create server_issued_resources index: {}", error); - anyhow::anyhow!("Failed to create server_issued_resources index: {}", error) - })?; - } - - Ok(()) -} - -async fn create_profile_capability_refs_table(pool: &Pool) -> Result<()> { - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS profile_capability_refs ( - profile_id TEXT NOT NULL, - ref_id TEXT NOT NULL, - enabled BOOLEAN NOT NULL, - FOREIGN KEY (profile_id) REFERENCES profile (id) ON DELETE CASCADE, - FOREIGN KEY (ref_id) REFERENCES capability_refs (ref_id) ON DELETE CASCADE, - PRIMARY KEY(profile_id, ref_id) - ) - "#, - ) - .execute(pool) - .await - .map_err(|error| anyhow::anyhow!("Failed to create profile_capability_refs table: {error}"))?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_profile_capability_refs_ref ON profile_capability_refs(ref_id)") - .execute(pool) - .await - .map_err(|error| anyhow::anyhow!("Failed to index profile_capability_refs: {error}"))?; - Ok(()) -} - -async fn create_direct_exposure_refs_table(pool: &Pool) -> Result<()> { - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS direct_exposure_refs ( - consumer_id TEXT NOT NULL, - ref_id TEXT NOT NULL, - enabled BOOLEAN NOT NULL, - FOREIGN KEY (consumer_id) REFERENCES client (identifier) ON DELETE CASCADE, - FOREIGN KEY (ref_id) REFERENCES capability_refs (ref_id) ON DELETE CASCADE, - PRIMARY KEY(consumer_id, ref_id) - ) - "#, - ) - .execute(pool) - .await - .map_err(|error| anyhow::anyhow!("Failed to create direct_exposure_refs table: {error}"))?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_direct_exposure_refs_ref ON direct_exposure_refs(ref_id)") - .execute(pool) - .await - .map_err(|error| anyhow::anyhow!("Failed to index direct_exposure_refs: {error}"))?; - Ok(()) -} - -async fn create_direct_exposure_servers_table(pool: &Pool) -> Result<()> { - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS direct_exposure_servers ( - consumer_id TEXT NOT NULL, - server_id TEXT NOT NULL, - new_ref_policy TEXT NOT NULL CHECK (new_ref_policy IN ('follow', 'review')), - FOREIGN KEY (consumer_id) REFERENCES client (identifier) ON DELETE CASCADE, - PRIMARY KEY(consumer_id, server_id) - ) - "#, - ) - .execute(pool) - .await - .map_err(|error| anyhow::anyhow!("Failed to create direct_exposure_servers table: {error}"))?; - Ok(()) -} - -/// Verify that all profile tables were created successfully async fn verify_profile_tables(pool: &Pool) -> Result<()> { for table in [ tables::PROFILE, @@ -569,148 +26,8 @@ async fn verify_profile_tables(pool: &Pool) -> Result<()> { "SELECT name FROM sqlite_master WHERE type='table' AND name='{table}'" )) .fetch_optional(pool) - .await - .map_err(|e| { - tracing::error!("Failed to verify {} table: {}", table, e); - anyhow::anyhow!("Failed to verify {} table: {}", table, e) - })? - .ok_or_else(|| { - let err = format!("{table} table not found after creation"); - tracing::error!("{}", err); - anyhow::anyhow!(err) - })?; - - tracing::debug!("Verified {} table exists", table); + .await? + .ok_or_else(|| anyhow::anyhow!("{table} table not found after migration"))?; } - Ok(()) } - -#[cfg(test)] -mod tests { - use sqlx::sqlite::SqlitePoolOptions; - - use super::*; - - #[tokio::test] - async fn resource_registry_schema_contains_template_routes_and_issued_resources() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("connect in-memory database"); - crate::config::server::init::initialize_server_tables(&pool) - .await - .expect("initialize server tables"); - initialize_profile_tables(&pool) - .await - .expect("initialize profile tables"); - - let template_columns = - sqlx::query_scalar::<_, String>("SELECT name FROM pragma_table_info('server_resource_templates')") - .fetch_all(&pool) - .await - .expect("load template columns"); - assert!(template_columns.iter().any(|column| column == "route_uri")); - - let issued_columns = - sqlx::query_scalar::<_, String>("SELECT name FROM pragma_table_info('server_issued_resources')") - .fetch_all(&pool) - .await - .expect("load issued resource columns"); - for expected in [ - "id", - "server_id", - "server_name", - "resource_uri", - "unique_uri", - "created_at", - "last_seen_at", - ] { - assert!( - issued_columns.iter().any(|column| column == expected), - "missing issued resource column {expected}" - ); - } - - let issued_indexes = - sqlx::query_scalar::<_, String>("SELECT name FROM pragma_index_list('server_issued_resources')") - .fetch_all(&pool) - .await - .expect("load issued resource indexes"); - assert!( - issued_indexes - .iter() - .any(|index| index == "idx_server_issued_resources_lookup") - ); - assert!( - issued_indexes - .iter() - .any(|index| index == "idx_server_issued_resources_unique_uri") - ); - } - - #[tokio::test] - async fn authoring_schema_uses_capability_refs_without_legacy_capability_tables() { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await - .expect("connect in-memory database"); - crate::config::server::init::initialize_server_tables(&pool) - .await - .expect("initialize server tables"); - crate::config::client::init::initialize_client_table(&pool) - .await - .expect("initialize client table"); - crate::config::database::initialize_capability_catalog(&pool) - .await - .expect("initialize capability catalog"); - initialize_profile_tables(&pool) - .await - .expect("initialize profile tables"); - - for table in [ - "profile_capability_refs", - "profile_server_relationships", - "direct_exposure_refs", - "direct_exposure_servers", - ] { - let exists: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") - .bind(table) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(exists, 1, "missing authoring table {table}"); - } - for legacy in [ - "profile_tool", - "profile_prompt", - "profile_resource", - "profile_resource_template", - ] { - let exists: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") - .bind(legacy) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(exists, 0, "legacy authoring table {legacy} must not exist"); - } - - let profile_server_columns = - sqlx::query_scalar::<_, String>("SELECT name FROM pragma_table_info('profile_server_relationships')") - .fetch_all(&pool) - .await - .unwrap(); - assert!(profile_server_columns.iter().any(|column| column == "enabled")); - assert!(profile_server_columns.iter().any(|column| column == "new_ref_policy")); - let direct_server_columns = - sqlx::query_scalar::<_, String>("SELECT name FROM pragma_table_info('direct_exposure_servers')") - .fetch_all(&pool) - .await - .unwrap(); - assert!(direct_server_columns.iter().any(|column| column == "new_ref_policy")); - } -} diff --git a/backend/src/config/profile/server.rs b/backend/src/config/profile/server.rs index acf92fc5..9e51389a 100644 --- a/backend/src/config/profile/server.rs +++ b/backend/src/config/profile/server.rs @@ -154,6 +154,7 @@ mod tests { .connect("sqlite::memory:") .await .unwrap(); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); diff --git a/backend/src/config/server/capabilities.rs b/backend/src/config/server/capabilities.rs index 016a2ce4..25328443 100644 --- a/backend/src/config/server/capabilities.rs +++ b/backend/src/config/server/capabilities.rs @@ -1874,6 +1874,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -3271,6 +3272,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -3310,6 +3312,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -3418,6 +3421,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -3609,6 +3613,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -3693,6 +3698,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/config/server/crud.rs b/backend/src/config/server/crud.rs index f586edc1..6bb01aa8 100644 --- a/backend/src/config/server/crud.rs +++ b/backend/src/config/server/crud.rs @@ -211,6 +211,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init tables"); pool } diff --git a/backend/src/config/server/import.rs b/backend/src/config/server/import.rs index c21fd871..45bdb2e5 100644 --- a/backend/src/config/server/import.rs +++ b/backend/src/config/server/import.rs @@ -820,6 +820,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -867,6 +868,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/config/server/init.rs b/backend/src/config/server/init.rs index 6c14b50d..7ad826ad 100644 --- a/backend/src/config/server/init.rs +++ b/backend/src/config/server/init.rs @@ -1,346 +1,27 @@ -// Server database initialization -// Contains functions for initializing server-related database tables - use anyhow::Result; use sqlx::{Pool, Sqlite}; -use tracing; use crate::common::constants::database::tables; -/// Initialize all server-related database tables +/// Verify server storage and perform recurring startup cleanup. +/// Durable schema is owned by `mcpmate-migrations`. pub async fn initialize_server_tables(pool: &Pool) -> Result<()> { - tracing::debug!("Initializing server-related database tables"); - - create_server_config_table(pool).await?; - create_server_args_table(pool).await?; - create_server_env_table(pool).await?; - create_server_headers_table(pool).await?; - create_server_meta_table(pool).await?; - create_server_namespace_issue_table(pool).await?; - create_server_oauth_config_table(pool).await?; - create_server_oauth_tokens_table(pool).await?; - + mcpmate_migrations::verify_config_database(pool).await?; verify_server_tables(pool).await?; - cleanup_pending_import_servers(pool).await?; - - tracing::debug!("Server-related database tables initialized successfully"); - Ok(()) -} - -async fn create_server_namespace_issue_table(pool: &Pool) -> Result<()> { - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_namespace_issue ( - server_id TEXT PRIMARY KEY, - issue_kind TEXT NOT NULL, - capability_kind TEXT, - external_identifier TEXT, - upstream_value TEXT, - conflicting_server_id TEXT, - conflicting_upstream_value TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - FOREIGN KEY (conflicting_server_id) REFERENCES server_config (id) ON DELETE CASCADE - ) - "#, - ) - .execute(pool) - .await?; - Ok(()) + cleanup_pending_import_servers(pool).await } async fn cleanup_pending_import_servers(pool: &Pool) -> Result<()> { - let result = sqlx::query( - r#" - DELETE FROM server_config - WHERE pending_import = 1 - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to clean pending_import server records: {}", e); - anyhow::anyhow!("Failed to clean pending_import server records: {}", e) - })?; - + let result = sqlx::query("DELETE FROM server_config WHERE pending_import = 1") + .execute(pool) + .await?; let removed = result.rows_affected(); if removed > 0 { tracing::info!(removed, "Removed stale pending_import server records during startup"); } - - Ok(()) -} - -/// Create server_config table if it doesn't exist -async fn create_server_config_table(pool: &Pool) -> Result<()> { - use crate::common::constants::transport; - - tracing::debug!("Creating server_config table if it doesn't exist"); - - let create_sql = format!( - r#" - CREATE TABLE IF NOT EXISTS server_config ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - server_type TEXT NOT NULL CHECK ( - server_type IN ('{}', '{}', '{}') - ), - command TEXT, - url TEXT, - source TEXT, - enabled BOOLEAN NOT NULL DEFAULT 1, - unify_direct_exposure_eligible BOOLEAN NOT NULL DEFAULT 0, - pending_import BOOLEAN NOT NULL DEFAULT 0, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - transport::STDIO, - transport::SSE, - transport::STREAMABLE_HTTP - ); - - sqlx::query(&create_sql).execute(pool).await.map_err(|e| { - tracing::error!("Failed to create server_config table: {}", e); - anyhow::anyhow!("Failed to create server_config table: {}", e) - })?; - - tracing::debug!("server_config table created or already exists"); - ensure_column(pool, "server_config", "pending_import", "BOOLEAN NOT NULL DEFAULT 0").await?; - ensure_column( - pool, - "server_config", - "unify_direct_exposure_eligible", - "BOOLEAN NOT NULL DEFAULT 0", - ) - .await?; - ensure_column(pool, "server_config", "source", "TEXT").await?; - Ok(()) -} - -/// Create server_args table if it doesn't exist -async fn create_server_args_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_args table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_args ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - arg_index INTEGER NOT NULL, - arg_value TEXT NOT NULL, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, arg_index) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_args table: {}", e); - anyhow::anyhow!("Failed to create server_args table: {}", e) - })?; - - tracing::debug!("server_args table created or already exists"); - Ok(()) -} - -/// Create server_env table if it doesn't exist -async fn create_server_env_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_env table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_env ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - env_key TEXT NOT NULL, - env_value TEXT NOT NULL, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, env_key) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_env table: {}", e); - anyhow::anyhow!("Failed to create server_env table: {}", e) - })?; - - tracing::debug!("server_env table created or already exists"); Ok(()) } -/// Create server_headers table (HTTP default headers) if it doesn't exist -async fn create_server_headers_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_headers table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_headers ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - header_key TEXT NOT NULL, - header_value TEXT NOT NULL, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id, header_key) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_headers table: {}", e); - anyhow::anyhow!("Failed to create server_headers table: {}", e) - })?; - - tracing::debug!("server_headers table created or already exists"); - Ok(()) -} - -/// Create server_meta table if it doesn't exist -async fn create_server_meta_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_meta table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_meta ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL, - server_name TEXT NOT NULL, - author TEXT, - category TEXT, - description TEXT, - extras_json TEXT, - icons_json TEXT, - protocol_version TEXT, - rating INTEGER, - recommended_scenario TEXT, - registry_meta_json TEXT, - registry_version TEXT, - repository TEXT, - upstream_name TEXT, - upstream_title TEXT, - server_version TEXT, - website TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE, - UNIQUE(server_id) - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_meta table: {}", e); - anyhow::anyhow!("Failed to create server_meta table: {}", e) - })?; - - tracing::debug!("server_meta table created or already exists"); - - // Backfill new columns when upgrading an existing development database - ensure_column(pool, "server_meta", "registry_version", "TEXT").await?; - ensure_column(pool, "server_meta", "registry_meta_json", "TEXT").await?; - ensure_column(pool, "server_meta", "extras_json", "TEXT").await?; - ensure_column(pool, "server_meta", "upstream_name", "TEXT").await?; - ensure_column(pool, "server_meta", "upstream_title", "TEXT").await?; - - Ok(()) -} - -async fn create_server_oauth_config_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_oauth_config table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_oauth_config ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL UNIQUE, - authorization_endpoint TEXT NOT NULL, - token_endpoint TEXT NOT NULL, - client_id TEXT NOT NULL, - client_secret TEXT, - scopes TEXT, - redirect_uri TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_oauth_config table: {}", e); - anyhow::anyhow!("Failed to create server_oauth_config table: {}", e) - })?; - - Ok(()) -} - -async fn create_server_oauth_tokens_table(pool: &Pool) -> Result<()> { - tracing::debug!("Creating server_oauth_tokens table if it doesn't exist"); - - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS server_oauth_tokens ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL UNIQUE, - access_token TEXT NOT NULL, - refresh_token TEXT, - token_type TEXT NOT NULL DEFAULT 'bearer', - expires_at TEXT, - scope TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (server_id) REFERENCES server_config (id) ON DELETE CASCADE - ) - "#, - ) - .execute(pool) - .await - .map_err(|e| { - tracing::error!("Failed to create server_oauth_tokens table: {}", e); - anyhow::anyhow!("Failed to create server_oauth_tokens table: {}", e) - })?; - - Ok(()) -} - -pub async fn ensure_column( - pool: &Pool, - table: &str, - column: &str, - definition: &str, -) -> Result<()> { - let stmt = format!( - "ALTER TABLE {table} ADD COLUMN {column} {definition}", - table = table, - column = column, - definition = definition - ); - match sqlx::query(&stmt).execute(pool).await { - Ok(_) => { - tracing::debug!("Added column {}.{}", table, column); - Ok(()) - } - Err(sqlx::Error::Database(db_err)) if db_err.message().contains("duplicate column name") => { - tracing::trace!("Column {}.{} already exists", table, column); - Ok(()) - } - Err(e) => { - tracing::error!("Failed to add column {}.{}: {}", table, column, e); - Err(anyhow::anyhow!("Failed to add column {}.{}: {}", table, column, e)) - } - } -} - -/// Verify that all server tables were created successfully async fn verify_server_tables(pool: &Pool) -> Result<()> { for table in [ tables::SERVER_CONFIG, @@ -355,20 +36,9 @@ async fn verify_server_tables(pool: &Pool) -> Result<()> { "SELECT name FROM sqlite_master WHERE type='table' AND name='{table}'" )) .fetch_optional(pool) - .await - .map_err(|e| { - tracing::error!("Failed to verify {} table: {}", table, e); - anyhow::anyhow!("Failed to verify {} table: {}", table, e) - })? - .ok_or_else(|| { - let err = format!("{table} table not found after creation"); - tracing::error!("{}", err); - anyhow::anyhow!(err) - })?; - - tracing::debug!("Verified {} table exists", table); + .await? + .ok_or_else(|| anyhow::anyhow!("{table} table not found after migration"))?; } - Ok(()) } @@ -416,55 +86,19 @@ mod tests { #[tokio::test] async fn initialize_server_tables_removes_pending_import_records() { let pool = setup_pool().await; + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("initialize tables"); - upsert_server(&pool, &build_server("serv_visible", "visible-server", false)) .await - .expect("insert visible server"); + .unwrap(); upsert_server(&pool, &build_server("serv_pending", "pending-server", true)) .await - .expect("insert pending server"); - - initialize_server_tables(&pool) - .await - .expect("reinitialize tables and cleanup pending records"); - + .unwrap(); + initialize_server_tables(&pool).await.expect("reinitialize tables"); let remaining_names = sqlx::query_scalar::<_, String>("SELECT name FROM server_config ORDER BY name ASC") .fetch_all(&pool) .await - .expect("list remaining servers"); - + .unwrap(); assert_eq!(remaining_names, vec!["visible-server".to_string()]); } - - #[tokio::test] - async fn initialize_server_tables_adds_observed_identity_columns_to_existing_meta_table() { - let pool = setup_pool().await; - sqlx::query( - r#" - CREATE TABLE server_meta ( - id TEXT PRIMARY KEY, - server_id TEXT NOT NULL UNIQUE, - server_name TEXT NOT NULL, - registry_version TEXT, - registry_meta_json TEXT, - extras_json TEXT - ) - "#, - ) - .execute(&pool) - .await - .expect("create legacy server_meta table"); - - initialize_server_tables(&pool) - .await - .expect("upgrade existing server tables"); - - let columns = sqlx::query_scalar::<_, String>("SELECT name FROM pragma_table_info('server_meta')") - .fetch_all(&pool) - .await - .expect("list server_meta columns"); - assert!(columns.iter().any(|column| column == "upstream_name")); - assert!(columns.iter().any(|column| column == "upstream_title")); - } } diff --git a/backend/src/config/server/meta.rs b/backend/src/config/server/meta.rs index 53c89830..748adc2c 100644 --- a/backend/src/config/server/meta.rs +++ b/backend/src/config/server/meta.rs @@ -233,6 +233,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/config/server/namespace_repair.rs b/backend/src/config/server/namespace_repair.rs index fd9cd3d3..b0830a4a 100644 --- a/backend/src/config/server/namespace_repair.rs +++ b/backend/src/config/server/namespace_repair.rs @@ -446,6 +446,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); diff --git a/backend/src/config/server/oauth.rs b/backend/src/config/server/oauth.rs index 2e2d3fa7..b0e67824 100644 --- a/backend/src/config/server/oauth.rs +++ b/backend/src/config/server/oauth.rs @@ -202,6 +202,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init tables"); pool } diff --git a/backend/src/config/server/tools.rs b/backend/src/config/server/tools.rs index 7fc56a5e..321c6bbf 100644 --- a/backend/src/config/server/tools.rs +++ b/backend/src/config/server/tools.rs @@ -333,6 +333,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/core/capability/connection_provider.rs b/backend/src/core/capability/connection_provider.rs index 3852e813..20b3e766 100644 --- a/backend/src/core/capability/connection_provider.rs +++ b/backend/src/core/capability/connection_provider.rs @@ -632,6 +632,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/core/capability/read_service.rs b/backend/src/core/capability/read_service.rs index f2bf1ba1..1813aadb 100644 --- a/backend/src/core/capability/read_service.rs +++ b/backend/src/core/capability/read_service.rs @@ -2650,6 +2650,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/core/capability/resource_registry.rs b/backend/src/core/capability/resource_registry.rs index 836b6d25..dffb5e86 100644 --- a/backend/src/core/capability/resource_registry.rs +++ b/backend/src/core/capability/resource_registry.rs @@ -378,6 +378,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory registry"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); @@ -624,6 +625,7 @@ mod tests { ) .await .expect("connect registry database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); diff --git a/backend/src/core/capability/resource_uri.rs b/backend/src/core/capability/resource_uri.rs index a69df23c..0725317d 100644 --- a/backend/src/core/capability/resource_uri.rs +++ b/backend/src/core/capability/resource_uri.rs @@ -1050,6 +1050,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect registry database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -1637,6 +1638,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect registry database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/core/capability/runtime.rs b/backend/src/core/capability/runtime.rs index 9f90e0a8..4ac0e574 100644 --- a/backend/src/core/capability/runtime.rs +++ b/backend/src/core/capability/runtime.rs @@ -1680,6 +1680,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect in-memory database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); @@ -1931,6 +1932,7 @@ mod tests { .connect_with(connect()) .await .expect("open first database instance"); + crate::test_helpers::prepare_config_database(&first_pool).await; crate::config::initialization::run_initialization(&first_pool) .await .expect("initialize first database instance"); diff --git a/backend/src/core/foundation/loader.rs b/backend/src/core/foundation/loader.rs index 01420ccb..d8c77c80 100644 --- a/backend/src/core/foundation/loader.rs +++ b/backend/src/core/foundation/loader.rs @@ -613,6 +613,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; run_initialization(&pool).await.expect("initialize schema"); let db_path = temp_dir.path().join("test.db"); diff --git a/backend/src/core/oauth/manager.rs b/backend/src/core/oauth/manager.rs index 7dfb86ad..1a31a05f 100644 --- a/backend/src/core/oauth/manager.rs +++ b/backend/src/core/oauth/manager.rs @@ -1298,6 +1298,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init tables"); initialize_llm_tables(&pool).await.expect("init llm tables"); OAuthManager::new(pool) @@ -1314,6 +1315,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init tables"); initialize_llm_tables(&pool).await.expect("init llm tables"); let store = Arc::new( diff --git a/backend/src/core/pool/sync.rs b/backend/src/core/pool/sync.rs index 700fc0a5..93266a96 100644 --- a/backend/src/core/pool/sync.rs +++ b/backend/src/core/pool/sync.rs @@ -249,6 +249,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; run_initialization(&pool).await.expect("initialize schema"); let db_path = temp_dir.path().join("test.db"); diff --git a/backend/src/core/profile/visibility.rs b/backend/src/core/profile/visibility.rs index cc3e06f6..b219fbfb 100644 --- a/backend/src/core/profile/visibility.rs +++ b/backend/src/core/profile/visibility.rs @@ -1136,6 +1136,7 @@ mod tests { .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_server_tables(&pool).await.expect("init server tables"); initialize_client_table(&pool).await.expect("init client table"); crate::config::database::initialize_capability_catalog(&pool) diff --git a/backend/src/core/proxy/init.rs b/backend/src/core/proxy/init.rs index 4d703b8a..9d2e1e9e 100644 --- a/backend/src/core/proxy/init.rs +++ b/backend/src/core/proxy/init.rs @@ -411,20 +411,8 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; run_initialization(&pool).await.expect("initialize schema"); - sqlx::query( - r#" - CREATE TABLE secure_store_provider_config ( - id INTEGER PRIMARY KEY CHECK (id = 1), - provider_mode TEXT NOT NULL DEFAULT 'operating_system', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - "#, - ) - .execute(&pool) - .await - .expect("create test secret provider config"); sqlx::query("INSERT INTO secure_store_provider_config (id, provider_mode) VALUES (1, 'local_file')") .execute(&pool) .await diff --git a/backend/src/core/proxy/server/gateway.rs b/backend/src/core/proxy/server/gateway.rs index 6f84be37..904d6baa 100644 --- a/backend/src/core/proxy/server/gateway.rs +++ b/backend/src/core/proxy/server/gateway.rs @@ -2351,6 +2351,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("sqlite pool"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::initialization::run_initialization(&pool) .await .expect("initialize database"); @@ -2989,6 +2990,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; initialize_client_table(&pool).await.expect("init client table"); initialize_system_settings(&pool) .await diff --git a/backend/src/core/proxy/server/resources.rs b/backend/src/core/proxy/server/resources.rs index 1b2f03ef..d65b00f8 100644 --- a/backend/src/core/proxy/server/resources.rs +++ b/backend/src/core/proxy/server/resources.rs @@ -207,6 +207,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect route database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/core/secrets.rs b/backend/src/core/secrets.rs index 7eae1aff..fb1dfbe5 100644 --- a/backend/src/core/secrets.rs +++ b/backend/src/core/secrets.rs @@ -538,6 +538,7 @@ mod tests { .execute(&pool) .await .expect("enable foreign keys"); + crate::test_helpers::prepare_config_database(&pool).await; server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/inspector/calls.rs b/backend/src/inspector/calls.rs index 5c11aba6..a053af13 100644 --- a/backend/src/inspector/calls.rs +++ b/backend/src/inspector/calls.rs @@ -596,6 +596,7 @@ mod tests { .connect("sqlite::memory:") .await .expect("connect registry database"); + crate::test_helpers::prepare_config_database(&pool).await; crate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize server tables"); diff --git a/backend/src/test_helpers.rs b/backend/src/test_helpers.rs index 79155273..10a97210 100644 --- a/backend/src/test_helpers.rs +++ b/backend/src/test_helpers.rs @@ -1,4 +1,17 @@ use mcpmate_secrets::store::SecretOriginInput; +use sqlx::{Pool, Sqlite}; + +pub async fn prepare_config_database(pool: &Pool) { + mcpmate_migrations::prepare_config_database(pool, mcpmate_migrations::DatabaseSource::InMemory) + .await + .expect("prepare test config database"); +} + +pub async fn prepare_audit_database(pool: &Pool) { + mcpmate_migrations::prepare_audit_database(pool, mcpmate_migrations::DatabaseSource::InMemory) + .await + .expect("prepare test audit database"); +} /// Build a `SecretOriginInput` for an OAuth-managed secret slot. pub fn oauth_secret_origin( diff --git a/backend/tests/capability_read_surface.rs b/backend/tests/capability_read_surface.rs index ae24a695..236b5a3d 100644 --- a/backend/tests/capability_read_surface.rs +++ b/backend/tests/capability_read_surface.rs @@ -108,6 +108,7 @@ async fn open_database(path: PathBuf) -> Arc { .connect_with(options) .await .expect("open test database"); + database_support::prepare_config(&pool).await; run_initialization(&pool).await.expect("initialize test database"); mcpmate::core::capability::naming::initialize(pool.clone()); mcpmate::core::capability::resolver::clear_cache().await; @@ -2629,3 +2630,5 @@ async fn isolated_restart_reset_parity_preserves_catalog_and_target_only_recover ] ); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/configuration_mode_transition.rs b/backend/tests/configuration_mode_transition.rs index ea2470d5..94b6176d 100644 --- a/backend/tests/configuration_mode_transition.rs +++ b/backend/tests/configuration_mode_transition.rs @@ -36,6 +36,7 @@ async fn fixture(explicit_mode: Option<&str>) -> Fixture { .connect("sqlite::memory:") .await .expect("create database"); + database_support::prepare_config(&pool).await; mcpmate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize servers"); @@ -520,3 +521,5 @@ async fn only_one_default_mode_transition_can_remain_pending() { assert!(error.as_database_error().is_some()); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/direct_exposure_management.rs b/backend/tests/direct_exposure_management.rs index ec211f0a..966353a8 100644 --- a/backend/tests/direct_exposure_management.rs +++ b/backend/tests/direct_exposure_management.rs @@ -18,6 +18,7 @@ async fn fixture() -> (sqlx::SqlitePool, ClientConfigService, CatalogRecord) { .connect("sqlite::memory:") .await .expect("create database"); + database_support::prepare_config(&pool).await; mcpmate::config::server::init::initialize_server_tables(&pool) .await .expect("initialize servers"); @@ -194,3 +195,5 @@ async fn retired_same_kind_refs_remain_as_intent_but_are_excluded_from_the_activ .expect("count published retired refs"); assert_eq!(published_refs, 0); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/inspector_smoke.rs b/backend/tests/inspector_smoke.rs index fa1643a3..7c24a4f9 100644 --- a/backend/tests/inspector_smoke.rs +++ b/backend/tests/inspector_smoke.rs @@ -100,6 +100,7 @@ async fn build_database_state(temp_dir: &TempDir) -> Arc { .execute(&db_pool) .await .expect("enable foreign keys"); + database_support::prepare_config(&db_pool).await; run_initialization(&db_pool).await.expect("initialize schema"); mcpmate::core::capability::naming::initialize(db_pool.clone()); mcpmate::core::capability::resolver::clear_cache().await; @@ -1417,3 +1418,5 @@ async fn inspector_tool_call_events_ws_unknown_call_closes() { let _ = ws.close(None).await; server.abort(); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/profile_surface_management.rs b/backend/tests/profile_surface_management.rs index dd73d430..511b0885 100644 --- a/backend/tests/profile_surface_management.rs +++ b/backend/tests/profile_surface_management.rs @@ -21,6 +21,7 @@ async fn init_management_pool() -> sqlx::SqlitePool { .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; mcpmate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); @@ -1166,3 +1167,5 @@ async fn direct_exposure_eligibility_and_affected_surfaces_commit_together() { .unwrap(); assert_eq!(active_entry_count, 0); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/secrets_store_api.rs b/backend/tests/secrets_store_api.rs index 9b7ece8a..8b861642 100644 --- a/backend/tests/secrets_store_api.rs +++ b/backend/tests/secrets_store_api.rs @@ -69,6 +69,7 @@ async fn build_test_context() -> (TempDir, Arc, Arc) .execute(&db_pool) .await .expect("enable foreign keys"); + database_support::prepare_config(&db_pool).await; run_initialization(&db_pool).await.expect("initialize schema"); let database = Arc::new(Database { @@ -124,6 +125,7 @@ async fn build_passphrase_test_context(master_password: &str) -> (TempDir, Arc (TempDir, Arc .execute(&db_pool) .await .expect("enable foreign keys"); + database_support::prepare_config(&db_pool).await; run_initialization(&db_pool).await.expect("initialize schema"); let database = Arc::new(Database { @@ -1047,6 +1050,7 @@ async fn provider_mode_persists_across_restart_simulation() { .execute(&db_pool) .await .expect("enable foreign keys"); + database_support::prepare_config(&db_pool).await; run_initialization(&db_pool).await.expect("initialize schema"); let database = Arc::new(Database { @@ -1269,3 +1273,5 @@ async fn list_secrets_reports_historical_usage_count() { assert_eq!(secret["used_by_count"], 0); assert_eq!(secret["historical_usage_count"], 1); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/support/client_writeback.rs b/backend/tests/support/client_writeback.rs index 254b11a8..97a98bcc 100644 --- a/backend/tests/support/client_writeback.rs +++ b/backend/tests/support/client_writeback.rs @@ -10,6 +10,8 @@ use sqlx::SqlitePool; use sqlx::sqlite::SqlitePoolOptions; use tempfile::TempDir; +use super::database; + pub const CLIENT_ID: &str = "writeback-client"; pub struct ClientWritebackFixture { @@ -31,6 +33,7 @@ impl ClientWritebackFixture { .connect("sqlite::memory:") .await .expect("create database"); + database::prepare_config(&pool).await; mcpmate::config::client::init::initialize_client_table(&pool) .await .expect("initialize clients"); diff --git a/backend/tests/support/database.rs b/backend/tests/support/database.rs new file mode 100644 index 00000000..356a318e --- /dev/null +++ b/backend/tests/support/database.rs @@ -0,0 +1,7 @@ +use mcpmate_migrations::{DatabaseSource, prepare_config_database}; + +pub async fn prepare_config(pool: &sqlx::SqlitePool) { + prepare_config_database(pool, DatabaseSource::InMemory) + .await + .expect("prepare config database through migrations"); +} diff --git a/backend/tests/support/mod.rs b/backend/tests/support/mod.rs index 73879e08..bc7a00c4 100644 --- a/backend/tests/support/mod.rs +++ b/backend/tests/support/mod.rs @@ -1 +1,2 @@ pub mod client_writeback; +pub mod database; diff --git a/backend/tests/support/runtime_database.rs b/backend/tests/support/runtime_database.rs index a15b9597..4450b50f 100644 --- a/backend/tests/support/runtime_database.rs +++ b/backend/tests/support/runtime_database.rs @@ -5,12 +5,16 @@ use mcpmate_capability_store::DerivedCapabilityCache; use sqlx::sqlite::SqlitePoolOptions; use tempfile::TempDir; +#[path = "database.rs"] +mod database; + pub async fn open_database(temp_dir: &TempDir) -> Arc { let pool = SqlitePoolOptions::new() .max_connections(4) .connect("sqlite::memory:") .await .expect("open test database"); + database::prepare_config(&pool).await; run_initialization(&pool).await.expect("initialize test database"); Arc::new(Database { pool, diff --git a/backend/tests/surface_materialization.rs b/backend/tests/surface_materialization.rs index f1d5990d..8b765363 100644 --- a/backend/tests/surface_materialization.rs +++ b/backend/tests/surface_materialization.rs @@ -140,6 +140,7 @@ async fn materializer_combines_owners_with_the_strictest_policy_and_restores_ite .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let first = record("first", "version one"); @@ -296,6 +297,7 @@ async fn resolved_proposal_is_idempotent_for_repeated_complete_trigger() { .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let capability = record("builtin", "version one"); @@ -380,6 +382,7 @@ async fn authoring_loader_combines_direct_exposure_and_builtin_records_without_s .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); mcpmate::config::server::init::initialize_server_tables(&pool) @@ -428,14 +431,10 @@ async fn authoring_loader_combines_direct_exposure_and_builtin_records_without_s )) .await .unwrap(); - for statement in [ - "CREATE TABLE profile_capability_refs (profile_id TEXT, ref_id TEXT, enabled INTEGER)", - "CREATE TABLE profile_server_relationships (profile_id TEXT, server_id TEXT, enabled BOOLEAN, new_ref_policy TEXT)", - "CREATE TABLE direct_exposure_refs (consumer_id TEXT, ref_id TEXT, enabled INTEGER)", - "CREATE TABLE direct_exposure_servers (consumer_id TEXT, server_id TEXT, new_ref_policy TEXT)", - ] { - sqlx::query(statement).execute(&pool).await.unwrap(); - } + sqlx::query("INSERT INTO client (id, name, identifier) VALUES ('client-a', 'Consumer A', 'consumer-a')") + .execute(&pool) + .await + .unwrap(); sqlx::query("INSERT INTO direct_exposure_refs VALUES ('consumer-a', ?, 1)") .bind(upstream.ref_id.as_str()) .execute(&pool) @@ -471,3 +470,5 @@ async fn authoring_loader_combines_direct_exposure_and_builtin_records_without_s assert_eq!(targets.len(), 1); transaction.rollback().await.unwrap(); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/surface_reconciliation.rs b/backend/tests/surface_reconciliation.rs index fdf636b4..ae43b123 100644 --- a/backend/tests/surface_reconciliation.rs +++ b/backend/tests/surface_reconciliation.rs @@ -56,6 +56,7 @@ async fn initialized_surface_pool() -> Pool { .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; mcpmate::config::server::init::initialize_server_tables(&pool) .await .unwrap(); @@ -94,6 +95,7 @@ async fn failed_outbox_delivery_remains_pending_for_retry() { .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; SqliteCapabilityCatalog::new(pool.clone()) .ensure_schema() .await @@ -130,6 +132,7 @@ async fn identical_catalog_observation_does_not_touch_surface_governance() { .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let first = record("stable definition"); @@ -581,3 +584,5 @@ async fn durable_worker_materializes_and_records_a_success_receipt() { assert_eq!(pending_missing, 1); assert_eq!(obsolete_versions, 1); } +#[path = "support/database.rs"] +mod database_support; diff --git a/backend/tests/surface_runtime.rs b/backend/tests/surface_runtime.rs index af74bd6c..35f9e4fc 100644 --- a/backend/tests/surface_runtime.rs +++ b/backend/tests/surface_runtime.rs @@ -70,6 +70,7 @@ async fn published_tool_surface() -> ( .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let initialize: InitializeResult = serde_json::from_value(json!({ @@ -207,6 +208,7 @@ async fn resource_reads_resolve_only_static_or_template_routes_pinned_by_the_act .connect("sqlite::memory:") .await .unwrap(); + database_support::prepare_config(&pool).await; let catalog = SqliteCapabilityCatalog::new(pool.clone()); catalog.ensure_schema().await.unwrap(); let resource: Resource = serde_json::from_value(json!({ @@ -331,3 +333,5 @@ async fn resource_reads_resolve_only_static_or_template_routes_pinned_by_the_act .is_err() ); } +#[path = "support/database.rs"] +mod database_support;