From ee874e05290413fe2639fb7e30ac4bfb4384802a Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 21:43:16 -0600 Subject: [PATCH 01/14] Migrate self-hosted Supabase off MinIO --- .gitignore | 9 +- README.md | 4 + eslint.config.js | 2 +- supabase/.env | 92 -- supabase/.env.example | 389 ++++++ supabase/.gitignore | 16 + supabase/LICENSE | 201 +++ supabase/MIGRATION.md | 232 ++++ supabase/README.md | 133 ++ supabase/UPSTREAM | 8 + supabase/docker-compose.github.yml | 8 + supabase/docker-compose.pg15.yml | 18 + supabase/docker-compose.pg17.yml | 19 + supabase/docker-compose.yml | 539 +++++--- supabase/migrate-storage.py | 137 ++ supabase/revisit.sql | 15 + supabase/run.sh | 297 +++++ supabase/tests/migrate-storage.spec.py | 83 ++ supabase/update.sh | 692 +++++++++++ supabase/upgrades.json | 39 + supabase/utils/add-new-auth-keys.sh | 226 ++++ supabase/utils/generate-keys.sh | 138 +++ supabase/utils/upgrade-pg17.sh | 822 ++++++++++++ supabase/volumes/api/envoy/cds.yaml | 223 ++++ .../volumes/api/envoy/docker-entrypoint.sh | 35 + supabase/volumes/api/envoy/envoy.yaml | 27 + supabase/volumes/api/envoy/lds.template.yaml | 1101 +++++++++++++++++ supabase/volumes/api/kong.yml | 241 ---- supabase/volumes/db/init/data.sql | 0 supabase/volumes/db/jwt.sql | 2 - supabase/volumes/db/pooler.sql | 6 + supabase/volumes/db/realtime.sql | 4 + supabase/volumes/db/webhooks.sql | 4 + supabase/volumes/functions/deno.jsonc | 6 + supabase/volumes/functions/hello/index.ts | 37 + supabase/volumes/functions/main/index.ts | 232 ++++ supabase/volumes/pooler/pooler.exs | 30 + tests/supabase-live.spec.ts | 113 ++ 38 files changed, 5697 insertions(+), 483 deletions(-) delete mode 100644 supabase/.env create mode 100644 supabase/.env.example create mode 100644 supabase/.gitignore create mode 100644 supabase/LICENSE create mode 100644 supabase/MIGRATION.md create mode 100644 supabase/README.md create mode 100644 supabase/UPSTREAM create mode 100644 supabase/docker-compose.github.yml create mode 100644 supabase/docker-compose.pg15.yml create mode 100644 supabase/docker-compose.pg17.yml create mode 100644 supabase/migrate-storage.py create mode 100644 supabase/revisit.sql create mode 100755 supabase/run.sh create mode 100644 supabase/tests/migrate-storage.spec.py create mode 100755 supabase/update.sh create mode 100644 supabase/upgrades.json create mode 100644 supabase/utils/add-new-auth-keys.sh create mode 100644 supabase/utils/generate-keys.sh create mode 100644 supabase/utils/upgrade-pg17.sh create mode 100644 supabase/volumes/api/envoy/cds.yaml create mode 100755 supabase/volumes/api/envoy/docker-entrypoint.sh create mode 100644 supabase/volumes/api/envoy/envoy.yaml create mode 100644 supabase/volumes/api/envoy/lds.template.yaml delete mode 100644 supabase/volumes/api/kong.yml delete mode 100644 supabase/volumes/db/init/data.sql create mode 100644 supabase/volumes/db/pooler.sql create mode 100644 supabase/volumes/db/realtime.sql create mode 100644 supabase/volumes/functions/deno.jsonc create mode 100644 supabase/volumes/functions/hello/index.ts create mode 100644 supabase/volumes/functions/main/index.ts create mode 100644 supabase/volumes/pooler/pooler.exs create mode 100644 tests/supabase-live.spec.ts diff --git a/.gitignore b/.gitignore index 0289daf5b5..04ccce548e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,9 +33,10 @@ docsLibraries/ public/__revisit-widget src/public/__revisit-widget -supabase/volumes/* -!supabase/volumes/db/ -supabase/volumes/db/data -!supabase/volumes/api/ +supabase/volumes/db/data.bak.* +supabase/volumes/db/data_migration/ +supabase/volumes/db/pg17_upgrade_bin_*.tar.gz +supabase/volumes/db/pgsodium_root.key.bak.* +supabase/.env.old coverage/ diff --git a/README.md b/README.md index 849de05186..c24f6f4a14 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ To run this demo experiment locally, you will need to install Node.js on your co * To run locally, run `yarn serve`. * Go to [http://localhost:8080](http://localhost:8080) to view it in your browser. The page will reload when you make changes. +## Self-hosted Supabase + +See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for the bundled, version-pinned Supabase stack. Existing MinIO installations must migrate before starting the new configuration. + ## Adding Tests This repo uses two test types: diff --git a/eslint.config.js b/eslint.config.js index 24866873b8..805e90af95 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -18,7 +18,7 @@ const compat = new FlatCompat({ }); export default [{ - ignores: ['public/**', 'src/public/**'], + ignores: ['public/**', 'src/public/**', 'supabase/volumes/functions/**'], }, ...fixupConfigRules(compat.extends( 'airbnb-base', 'airbnb/rules/react', diff --git a/supabase/.env b/supabase/.env deleted file mode 100644 index 7e1170ea56..0000000000 --- a/supabase/.env +++ /dev/null @@ -1,92 +0,0 @@ -########## -# Secrets -# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION -############ - -POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password -JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long -ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE -SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated - - -############ -# Database - You can change these to any PostgreSQL database that has logical replication enabled. -############ - -POSTGRES_HOST=db -POSTGRES_DB=postgres -POSTGRES_PORT=5432 -# default user is postgres - - -############ -# API Proxy - Configuration for the Kong Reverse proxy. -############ - -KONG_HTTP_PORT=8000 -KONG_HTTPS_PORT=8443 - - -############ -# API - Configuration for PostgREST. -############ - -PGRST_DB_SCHEMAS=public,storage,graphql_public - - -############ -# Auth - Configuration for the GoTrue authentication server. -############ -ENABLE_GITHUB_OAUTH=true -GITHUB_OAUTH_CLIENT_ID=your-github-oauth-client-id -GITHUB_OAUTH_SECRET=your-github-oauth-client-secret -GITHUB_OAUTH_REDIRECT_URI=http://localhost:8000/auth/v1/callback - -## General -SITE_URL=http://localhost:3000 -ADDITIONAL_REDIRECT_URLS= -JWT_EXPIRY=3600 -DISABLE_SIGNUP=false -API_EXTERNAL_URL=http://localhost:8000 - -## Mailer Config -MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" -MAILER_URLPATHS_INVITE="/auth/v1/verify" -MAILER_URLPATHS_RECOVERY="/auth/v1/verify" -MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" - -## Email auth -ENABLE_EMAIL_SIGNUP=true -ENABLE_EMAIL_AUTOCONFIRM=false -SMTP_ADMIN_EMAIL=admin@example.com -SMTP_HOST=supabase-mail -SMTP_PORT=2500 -SMTP_USER=fake_mail_user -SMTP_PASS=fake_mail_password -SMTP_SENDER_NAME=fake_sender -ENABLE_ANONYMOUS_USERS=true - -## Phone auth -ENABLE_PHONE_SIGNUP=true -ENABLE_PHONE_AUTOCONFIRM=true - - -############ -# Studio - Configuration for the Dashboard -############ - -STUDIO_DEFAULT_ORGANIZATION=Default Organization -STUDIO_DEFAULT_PROJECT=Default Project - -# replace if you intend to use Studio outside of localhost -SUPABASE_PUBLIC_URL=http://localhost:8000 - -############ -# Logs - Configuration for Logflare -# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction -############ - -# Change vector.toml sinks to reflect this change -LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key diff --git a/supabase/.env.example b/supabase/.env.example new file mode 100644 index 0000000000..eb130e1cfe --- /dev/null +++ b/supabase/.env.example @@ -0,0 +1,389 @@ +############ +# Docker compose override files to layer on top of docker-compose.yml. +# Native docker compose COMPOSE_FILE: colon-separated list, base file first. +# Manage with: ./run.sh config add|remove +# +# Examples: +# COMPOSE_FILE=docker-compose.yml +# COMPOSE_FILE=docker-compose.yml:docker-compose.pg17.yml +# +############ +COMPOSE_FILE=docker-compose.yml + + +############ +# Secrets +# +# YOU MUST CHANGE ALL THE DEFAULT VALUES BELOW BEFORE STARTING +# THE CONTAINERS FOR THE FIRST TIME! +# +# Documentation: +# https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase +# +# To generate secrets and API keys: +# 1. sh utils/generate-keys.sh +# 2. sh utils/add-new-auth-keys.sh +# +############ + +# Postgres +POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password + +# Legacy symmetric HS256 key +JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long +# Legacy API keys (HS256-signed JWTs) +ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE +SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q + +# Asymmetric key pair (ES256) and opaque API keys +# +# Documentation: +# https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys +# +# To generate: +# sh ./utils/add-new-auth-keys.sh +# +# Opaque API key for client-side use (anon role). +SUPABASE_PUBLISHABLE_KEY= +# Opaque API key for server-side use (service_role). Never expose in client code. +SUPABASE_SECRET_KEY= +# JSON array of signing JWKs (EC private + legacy symmetric). +# Used by Auth. +JWT_KEYS= +# JWKS for token verification (EC public + legacy symmetric). +# Used by PostgREST, Realtime, Storage to verify tokens. +JWT_JWKS= + +# Access to Dashboard +DASHBOARD_USERNAME=supabase +DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated + +# Encryption key for securing Realtime and Supavisor communications. +# (Must be at least 64 characters; generate with: openssl rand -base64 48) +SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq + +# Encryption key used by Realtime for sensitive fields in the `_realtime` schema. +# (Must be exactly 16 characters; generate with: `openssl rand -hex 8`) +REALTIME_DB_ENC_KEY=supabaserealtime + +# Encryption key used by Supavisor for storing encrypted configuration. +# (Must be exactly 32 characters; generate with: openssl rand -hex 16) +VAULT_ENC_KEY=your-32-character-encryption-key + +# Encryption key for securing connection strings used by Studio against postgres-meta. +# (Must be at least 32 characters; generate with openssl rand -base64 24) +PG_META_CRYPTO_KEY=your-encryption-key-32-chars-min + +# API token for log ingestion used by Logflare and Vector. +# (Must be at least 32 characters; generate with openssl rand -base64 24) +LOGFLARE_PUBLIC_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-public +# API token used for Logflare management operations. Never expose client-side. +# (Must be at least 32 characters; generate with openssl rand -base64 24) +LOGFLARE_PRIVATE_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-private + +# Access key ID (username-like) for accessing the S3 protocol endpoint in Storage. +# (Generate with: openssl rand -hex 16) +S3_PROTOCOL_ACCESS_KEY_ID=625729a08b95bf1b7ff351a663f3a23c +# Secret key (password-like) used with S3_PROTOCOL_ACCESS_KEY_ID. +# (Generate with: openssl rand -hex 32) +S3_PROTOCOL_ACCESS_KEY_SECRET=850181e4652dd023b7a98c58ae0d2d34bd487ee0cc3254aed6eda37307425907 + + +############ +# URLs - Configure hostnames below to reflect your actual domain name +############ + +# Access to Dashboard and REST API +SUPABASE_PUBLIC_URL=http://localhost:8000 + +# Full external URL of the Auth service, used to construct OAuth callbacks, +# SAML endpoints, and email links +API_EXTERNAL_URL=http://localhost:8000/auth/v1 + +# See also the Auth section below for Site URL and Redirect URLs configuration + + +############ +# Database - Postgres configuration +############ + +# Using default user (postgres) +POSTGRES_HOST=db +POSTGRES_DB=postgres + +# Default configuration includes Supavisor exposing POSTGRES_PORT +# Postgres uses POSTGRES_PORT inside the container +# Documentation: +# https://supabase.com/docs/guides/self-hosting/accessing-postgres +POSTGRES_PORT=5432 + + +############ +# Database pooler +############ + +# Self-hosted Supabase uses Supavisor as the default database pooler. +# If you use the PgBouncer docker-compose override, Supavisor is disabled +# and the pooler settings below are used to configure PgBouncer instead. +# +# Supavisor exposes POSTGRES_PORT and POOLER_PROXY_PORT_TRANSACTION, +# POSTGRES_PORT is used for session mode pooling +# PgBouncer only exposes POOLER_PROXY_PORT_TRANSACTION. +# +# Port to use for transaction mode pooling connections +POOLER_PROXY_PORT_TRANSACTION=6543 + +# Maximum number of PostgreSQL connections Supavisor or PgBouncer opens per pool +POOLER_DEFAULT_POOL_SIZE=20 + +# Maximum number of client connections Supavisor or PgBouncer accepts per pool +POOLER_MAX_CLIENT_CONN=100 + +# Unique Supavisor tenant identifier +# Documentation: +# https://supabase.com/docs/guides/self-hosting/accessing-postgres +POOLER_TENANT_ID=your-tenant-id + +# Pool size for internal metadata storage used by Supavisor +# This is separate from client connections and used only by Supavisor itself +POOLER_DB_POOL_SIZE=5 + + +############ +# Studio - Configuration for the Dashboard +############ + +STUDIO_DEFAULT_ORGANIZATION=Default Organization +STUDIO_DEFAULT_PROJECT=Default Project + +# Add your OpenAI API key to enable AI Assistant +OPENAI_API_KEY=sk-proj-xxxxxxxx + + +############ +# Auth - Configuration for the authentication server +############ + +## General settings + +# Equivalent to "Site URL" and "Redirect URLs" platform configuration options +# Documentation: https://supabase.com/docs/guides/auth/redirect-urls +SITE_URL=http://localhost:3000 +ADDITIONAL_REDIRECT_URLS= + +JWT_EXPIRY=3600 +DISABLE_SIGNUP=false + +## Mailer Config +MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" +MAILER_URLPATHS_INVITE="/auth/v1/verify" +MAILER_URLPATHS_RECOVERY="/auth/v1/verify" +MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" + +## Email auth +ENABLE_EMAIL_SIGNUP=true +ENABLE_EMAIL_AUTOCONFIRM=false +SMTP_ADMIN_EMAIL=admin@example.com +SMTP_HOST=supabase-mail +SMTP_PORT=2500 +SMTP_USER=fake_mail_user +SMTP_PASS=fake_mail_password +SMTP_SENDER_NAME=fake_sender +ENABLE_ANONYMOUS_USERS=false + +## Phone auth +ENABLE_PHONE_SIGNUP=true +ENABLE_PHONE_AUTOCONFIRM=true + +## OAuth / Social login providers + +# Uncomment and fill in the providers you want to enable. +# You must ALSO uncomment the matching GOTRUE_EXTERNAL_* lines in docker-compose.yml +# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-oauth +# GOOGLE_ENABLED=false +# GOOGLE_CLIENT_ID= +# GOOGLE_SECRET= + +# GITHUB_ENABLED=false +# GITHUB_CLIENT_ID= +# GITHUB_SECRET= + +# AZURE_ENABLED=false +# AZURE_CLIENT_ID= +# AZURE_SECRET= + +# Phone / SMS provider configuration +# Uncomment to configure SMS delivery for phone auth and phone MFA. +# You must ALSO uncomment the matching GOTRUE_SMS_* lines in docker-compose.yml +# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa +# SMS_PROVIDER=twilio +# SMS_OTP_EXP=60 +# SMS_OTP_LENGTH=6 +# SMS_MAX_FREQUENCY=60s +# SMS_TEMPLATE=Your code is {{ .Code }} + +# SMS_TWILIO_ACCOUNT_SID= +# SMS_TWILIO_AUTH_TOKEN= +# SMS_TWILIO_MESSAGE_SERVICE_SID= + +# Test OTP: map phone numbers to fixed OTP codes for development +# Format: phone1:code1,phone2:code2 +# SMS_TEST_OTP= + +# Multi-factor authentication (MFA) +# Uncomment to change MFA defaults. +# You must ALSO uncomment the matching GOTRUE_MFA_* lines in docker-compose.yml + +# App Authenticator (TOTP) - enabled by default +# MFA_TOTP_ENROLL_ENABLED=true +# MFA_TOTP_VERIFY_ENABLED=true + +# Phone MFA - disabled by default (opt-in) +# MFA_PHONE_ENROLL_ENABLED=false +# MFA_PHONE_VERIFY_ENABLED=false + +# Maximum MFA factors a user can enroll +# MFA_MAX_ENROLLED_FACTORS=10 + +## SAML SSO + +# You must ALSO uncomment the matching GOTRUE_* lines in docker-compose.yml +# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso + +# SAML_ENABLED=true +# SAML_PRIVATE_KEY= + +# Optional: accept encrypted SAML assertions from IdPs (default: false) +# SAML_ALLOW_ENCRYPTED_ASSERTIONS=false + +# Optional: how long relay state tokens remain valid (default: 2m0s) +# SAML_RELAY_STATE_VALIDITY_PERIOD=2m0s + +# Optional: override the SAML entity ID / ACS base URL +# Defaults to API_EXTERNAL_URL if not set +# SAML_EXTERNAL_URL=https://supabase.example.com:8000/auth/v1 + +# Optional: rate limit on the ACS endpoint (requests per second, default: 15) +# SAML_RATE_LIMIT_ASSERTION=15 + + +############ +# Storage - Configuration for Storage +############ + +# Check the S3_PROTOCOL_ACCESS_KEY_ID/SECRET above, and +# refer to the documentation at: +# https://supabase.com/docs/guides/self-hosting/self-hosted-s3 +# to learn how to configure the S3 protocol endpoint + +# S3 bucket when using S3 backend, directory name when using 'file' +GLOBAL_S3_BUCKET=stub + +# Used for S3 protocol endpoint configuration +REGION=stub + +# Used by MinIO when added via: +# docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d +MINIO_ROOT_USER=supa-storage +# Root administrator password for the RustFS or MinIO server. +# (Must be 8+ characters; generate with: openssl rand -hex 16) +MINIO_ROOT_PASSWORD=secret1234 + +# Equivalent to project_ref as described here: +# https://supabase.com/docs/guides/storage/s3/authentication#session-token +STORAGE_TENANT_ID=stub + + +############ +# Functions - Configuration for Edge functions +############ + +# Documentation: +# https://supabase.com/docs/guides/self-hosting/self-hosted-functions + +# NOTE: VERIFY_JWT applies to all functions +FUNCTIONS_VERIFY_JWT=false + + +############ +# API - Configuration for PostgREST +############ + +# Postgres schemas exposed via the REST API +PGRST_DB_SCHEMAS=public,graphql_public + +# Max number of rows returned by a request +PGRST_DB_MAX_ROWS=1000 + +# Extra schemas added to the search_path of every request +PGRST_DB_EXTRA_SEARCH_PATH=public + + +############ +# Logs and Analytics +############ + +## Vector log collection and routing + +# Docker socket location - required for proper Vector operation +DOCKER_SOCKET_LOCATION=/var/run/docker.sock +# For Podman use the following: +# DOCKER_SOCKET_LOCATION=/run/podman/podman.sock + +## Analytics (Logflare) + +# Check the LOGFLARE_* access token configuration _above_. +# If Logflare has to be externally exposed - configure securely! + +# Google Cloud Project details +# Documentation: +# https://supabase.com/docs/reference/self-hosting-analytics/introduction +GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID +GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER + + +############ +# API gateway +############ + +# Host port the API gateway (Envoy by default) listens on. +API_GW_HTTP_PORT=8000 + +# Kong gateway override only (sh run.sh config add kong). KONG_HTTPS_PORT is +# Kong's built-in HTTPS listener; KONG_HTTP_PORT is kept as a fallback for +# API_GW_HTTP_PORT so existing .env files continue to work. +KONG_HTTP_PORT=8000 +KONG_HTTPS_PORT=8443 + +# Used internally by the API gateway - DO NOT use in any client or server code. +# Pre-signed ES256 JWT "API key" for anon role. +ANON_KEY_ASYMMETRIC= +# Pre-signed ES256 JWT "API key" for service_role. +SERVICE_ROLE_KEY_ASYMMETRIC= + + +############ +# imgproxy +############ + +# Enable webp support +IMGPROXY_AUTO_WEBP=true + + +############ +# TLS Proxy - Optional Caddy or Nginx reverse proxy with Let's Encrypt +############ + +# Documentation: +# https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https + +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.caddy.yml up -d +# docker compose -f docker-compose.yml -f docker-compose.nginx.yml up -d + +# Domain name for the proxy (must point to your server) +PROXY_DOMAIN=your-domain.example.com + +# Email for Let's Encrypt certificate notifications (nginx only, Caddy uses PROXY_DOMAIN). +# This should be a valid email, not a placeholder (otherwise Certbot may fail to start). +CERTBOT_EMAIL=admin@example.com diff --git a/supabase/.gitignore b/supabase/.gitignore new file mode 100644 index 0000000000..1f9c343619 --- /dev/null +++ b/supabase/.gitignore @@ -0,0 +1,16 @@ +volumes/db/data +volumes/storage +volumes/snippets +volumes/functions/** +!volumes/functions/deno.json* +!volumes/functions/main/ +volumes/functions/main/** +!volumes/functions/main/index.ts +!volumes/functions/hello/ +volumes/functions/hello/** +!volumes/functions/hello/index.ts +.env +test.http +docker-compose.override.yml +.supabase-version +backups diff --git a/supabase/LICENSE b/supabase/LICENSE new file mode 100644 index 0000000000..49ea124025 --- /dev/null +++ b/supabase/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Supabase + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md new file mode 100644 index 0000000000..3e25f34dac --- /dev/null +++ b/supabase/MIGRATION.md @@ -0,0 +1,232 @@ +# Migrate the bundled MinIO deployment + +This procedure targets the previous bundled stack (Postgres 15.8.1.060, +Supabase Storage v1.11.13, and MinIO) and moves it to the release in [UPSTREAM](UPSTREAM), +using Postgres 17 and Storage v1.74.0's default file backend. +Rehearse against a restored copy of your deployment first. New installations +use [README.md](README.md); hosted Supabase users are unaffected. + +The database upgrade and storage move are separate steps in **one maintenance +window**. Storage object IDs, versions, owners, timestamps, bucket definitions, +and policies remain in Postgres. `migrate-storage.py` exports object bytes through +the old API and installs their versioned files with HTTP metadata in the new +backend. It never writes database records or re-uploads objects through the API. + +Requirements: a Linux server, Docker Compose, Bash, curl, OpenSSL, Python 3.11+, +root access for the upstream database upgrade, and enough space for the database +backup/upgrade and two copies of object data. Pre-pull the target images before +starting the upgrade; allow at least 15 GB additional temporary space for the +upgrade images, extracted binaries, and compressed bundle (also inside the +Docker VM when using Docker Desktop). The destination filesystem must +support Linux extended attributes. Do not reuse MinIO's on-disk directory as +Supabase file storage. Custom S3 layouts, custom object-version separators, or +multiple Supabase stacks on one Docker host require a separately rehearsed plan. + +## 1. Inventory and export while the old stack is running + +Keep the old Compose/configuration files and images available for rollback. +Save cached MinIO server/client images if the registry can no longer supply them. +Record your database extensions, OAuth/SMTP settings, proxy configuration, +bucket limits, storage tenant, backend bucket, and any custom access policies. +Restrict application traffic and pause participant/designer/background writes +for the remainder of the procedure. Keep Supabase accessible to the migration +operator until export finishes. + +Copy `migrate-storage.py` from the new checkout to the **old deployment directory**. +From that old directory, create a private backup outside the deployment: + +```sh +umask 077 +mkdir ../supabase-migration-backup +cp .env docker-compose.yml ../supabase-migration-backup/ +cp -a volumes/api ../supabase-migration-backup/api +docker image save minio/minio minio/mc \ + -o ../supabase-migration-backup/minio-images.tar +docker exec supabase-db pg_dumpall -U supabase_admin \ + > ../supabase-migration-backup/database.sql +db_config_volume=$(docker inspect supabase-db --format \ + '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}') +test -n "$db_config_volume" +docker run --rm -v "$db_config_volume:/source:ro" \ + -v "$(cd ../supabase-migration-backup && pwd):/backup" \ + alpine:3.22 tar -cf /backup/db-config.tar -C /source . +``` + +Privately set `SUPABASE_SERVICE_ROLE_KEY` to the **old** `.env`'s `SERVICE_ROLE_KEY` +(for example, use `read -rs SUPABASE_SERVICE_ROLE_KEY` in Bash, then `export +SUPABASE_SERVICE_ROLE_KEY`; paste the key when prompted). Do not put the key in +shell history or in the application's `VITE_*` variables. + +```sh +python3 migrate-storage.py export ../supabase-migration-backup/objects \ + --url http://localhost:8000 +``` + +Use your actual gateway URL. Export includes all live objects in every Supabase +bucket, not only `revisit`, and fails on missing objects or size mismatches. +The export directory must not already exist. An interrupted export must be +repeated into a new directory; only a completed export contains `manifest.json`. +It does not copy orphaned MinIO objects absent from the database. + +Stop the old stack **using its old Compose file**, without deleting volumes: + +```sh +docker compose down +sudo cp -a volumes/db/data ../supabase-migration-backup/postgres15-data +sudo cp -a volumes/storage ../supabase-migration-backup/minio-data +``` + +Save the other deployment-specific configuration files as needed. Verify backup +completion before continuing. The physical database and encryption-volume +backups must belong to the same quiesced deployment. + +## 2. Install the new configuration; preserve existing secrets + +Copy the new checkout's `supabase/` configuration, scripts, and supporting files +into the deployment directory, excluding runtime data and `.env`. For example, +from the old deployment directory, with rsync installed: + +```sh +rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ + --exclude='volumes/db/data*' --exclude='volumes/storage*' \ + /path/to/new-checkout/supabase/ ./ +``` + +Use the actual path to the new checkout. Do not add `--delete`. Preserve the +existing Compose project name and `db-config` named volume. + +Merge new `.env.example` settings into your existing private `.env`. Keep the +existing database password, `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY`, and +encryption keys. **Do not run `generate-keys.sh` during migration:** rotating +these values can invalidate sessions or make encrypted data unreadable. + +- Set `COMPOSE_FILE=docker-compose.yml` and `ENABLE_ANONYMOUS_USERS=true`. +- Carry the previous `TENANT_ID` into `STORAGE_TENANT_ID`; keep `GLOBAL_S3_BUCKET` + and `REGION` consistent with the storage installation command below. +- Set the new gateway port `API_GW_HTTP_PORT` to your previous HTTP port. +- Supply newly required secrets such as `SECRET_KEY_BASE`, `REALTIME_DB_ENC_KEY`, + `VAULT_ENC_KEY`, and `PG_META_CRYPTO_KEY` if absent. Also generate fresh + `S3_PROTOCOL_ACCESS_KEY_ID` and `S3_PROTOCOL_ACCESS_KEY_SECRET` if missing; + these secure Storage’s S3 API even when its backend is local files. Generate + only missing values using the commands/lengths in `.env.example`; never use + the template’s default secrets or credentials. +- Preserve URLs, SMTP, signup settings, and OAuth provider credentials. The old + GitHub settings are preserved by `sh run.sh config add github`. This optional + override reads the existing `ENABLE_GITHUB_OAUTH`, `GITHUB_OAUTH_CLIENT_ID`, + `GITHUB_OAUTH_SECRET`, and `GITHUB_OAUTH_REDIRECT_URI` variables. Keep the same + callback URL and registered GitHub OAuth application. +- Keep legacy signing for this migration. Introducing asymmetric signing/key + rotation is optional later; legacy keys are supported by the pinned stack. + +Before modifying database files, pull the target images and check the two Node +services start on this host: + +```sh +sh run.sh pull +docker compose run --rm --no-deps studio node --version +docker compose run --rm --no-deps meta node --version +``` + +If either crashes, resolve host/image compatibility before upgrading. Our ARM64 +Docker Desktop rehearsal required the amd64 variants of these two images, as +noted in [README.md](README.md). + +Move the old MinIO data aside and create an empty file-storage directory: + +```sh +mv volumes/storage volumes/storage.minio-backup +mkdir volumes/storage +``` + +Do **not** apply `revisit.sql`: existing application schema, buckets, and policies +are retained by the database upgrade. + +## 3. Upgrade Postgres using the upstream script + +The upgrade script expects a running Postgres 15 container. Start **only the old +database** with the saved old configuration, pointing Compose at this deployment +directory so its relative paths still refer to the original data: + +```sh +docker compose --project-directory "$PWD" \ + -f ../supabase-migration-backup/docker-compose.yml up -d --wait db +docker exec supabase-db pg_isready -U postgres +# Our previous Compose did not initialize the upstream Realtime schema. +docker exec -i supabase-db psql -U supabase_admin -d postgres \ + -v ON_ERROR_STOP=1 < volumes/db/realtime.sql +sudo bash utils/upgrade-pg17.sh +``` + +Read all output, including migration warnings. This script uses upstream's +`pg_upgrade` procedure, preserves `data.bak.pg15` and the encryption key, applies +required role/extension changes, and starts the new stack. See +[upstream database upgrade guidance](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17). +On a host with multiple `db-config` volumes, stop and resolve the target before +running: this upstream script selects a matching volume by name. + +Confirm the script reaches **Upgrade complete** and all services start. Do not +rerun it blindly after a partial upgrade: it refuses an already upgraded database. +Resolve startup failures and use the preserved backups to rehearse again. + +The target image changes the libc collation version. While traffic is still +blocked, rebuild indexes in each retained database (include any custom databases): + +```sh +for database in postgres template1 _supabase; do + docker exec supabase-db psql -U supabase_admin -d "$database" \ + -v ON_ERROR_STOP=1 -c "REINDEX DATABASE \"$database\";" +done +``` + +The upstream script refreshes the version marker; that alone does not rebuild +indexes. See [PostgreSQL collation guidance](https://www.postgresql.org/docs/17/sql-altercollation.html). +Allow time and temporary disk space for this step. + +Keep application access blocked. Stop Storage and install exported files: + +```sh +docker compose stop storage +sudo python3 migrate-storage.py install ../supabase-migration-backup/objects \ + --destination volumes/storage --bucket stub --tenant stub +docker compose up -d storage +sh run.sh start +``` + +Replace `stub` with your preserved backend bucket and tenant values. The helper +checks all export hashes before writing, rejects unsafe paths/conflicting files, +and preserves Content-Type and Cache-Control using Linux extended attributes. +It can be rerun with the same verified export after fixing an installation error. + +## 4. Verify before reopening writes + +```sh +python3 migrate-storage.py verify ../supabase-migration-backup/objects \ + --url http://localhost:8000 +sh run.sh status +docker exec supabase-db psql -U supabase_admin -d postgres \ + -c 'SHOW server_version;' +printf 'ref=self-hosted/v0.8.2\n' > .supabase-version +``` + +Verification streams every migrated object through the new API and checks its +SHA-256, Content-Type, and Cache-Control, plus the original database object +metadata. Keep the legacy application URL/key unchanged when possible. Check Study Designer login, +existing participants/answers/configurations, recordings, exports, access +policies, and a new participant submission. Restart the stack and confirm both +old and new data persist. Retain database/file/encryption backups and old images +until the deployment has been accepted. Then reopen application writes. + +## Rollback + +While writes are still paused, stop the new stack without `-v`. Restore the saved +old Compose/API configuration and `.env`, the quiesced Postgres 15 data, the +MinIO directory, and the matching `db-config` encryption/configuration volume. +The database upgrade changes the volume's ownership/configuration, so restoring +only database files is insufficient. Use the physical backup and saved volume +archive (and load saved MinIO images with `docker image load -i`), or follow the +upstream rollback steps together with the original configuration. Start the old stack with the saved original images and verify +data and login before reopening writes. Keep the failed new data separately +for diagnosis. + +After new writes are accepted, restoring the old backups loses those writes. +Pause and plan how to retain them rather than treating rollback as lossless. diff --git a/supabase/README.md b/supabase/README.md new file mode 100644 index 0000000000..f9eb89c043 --- /dev/null +++ b/supabase/README.md @@ -0,0 +1,133 @@ +# Self-hosted Supabase for reVISit + +This directory bundles the default stack from **Supabase self-hosted/v0.8.2** +(see [UPSTREAM](UPSTREAM)). The Compose file and upstream configuration/scripts +are unchanged. Supabase Storage uses persistent local files; MinIO is not required. +For an existing deployment, follow [MIGRATION.md](MIGRATION.md) **before starting +this configuration**. Hosted Supabase deployments do not need this migration. + +## First-time setup + +Requirements: a Linux host with Docker Engine and Docker Compose v2 or newer, OpenSSL, +and Node.js 16+ (the upstream key helper can use Docker instead). For server +resources and operating-system details, follow the +[upstream installation guide](https://supabase.com/docs/guides/self-hosting/docker). +The storage filesystem must support Linux extended attributes. Mac shared +folders in Docker Desktop may reject these attributes; use a Docker-managed +Linux volume for a local rehearsal and back it up separately. The pinned ARM64 +Studio and postgres-meta images also crashed on our Apple Silicon Docker Desktop +host; their amd64 variants worked under emulation. Verify image startup on your +actual host before a migration. + +Run the commands below from this directory. The upstream stack uses fixed +container names: run only one copy per Docker host, or deliberately isolate it. + +1. Create your private configuration and generate fresh secrets: + + ```sh + cp .env.example .env + sh utils/generate-keys.sh --update-env + sh utils/add-new-auth-keys.sh --update-env + printf 'ref=self-hosted/v0.8.2\n' > .supabase-version + ``` + + These helpers print secrets; keep their output private. Never commit `.env`. + The second helper enables asymmetric signing in `docker-compose.yml` while + retaining verification of legacy JWTs. + +2. Edit `.env`: + - Set `ENABLE_ANONYMOUS_USERS=true` for participant sessions. + - Set `SUPABASE_PUBLIC_URL` to the Supabase base URL clients can reach + (`http://localhost:8000` locally). Set `API_EXTERNAL_URL` to its Auth + endpoint (`http://localhost:8000/auth/v1` locally), following the template. + - Set `SITE_URL` and `ADDITIONAL_REDIRECT_URLS` for your reVISit application. + - Review email confirmation and SMTP settings for Study Designer accounts. + For GitHub OAuth, append `ENABLE_GITHUB_OAUTH=true`, + `GITHUB_OAUTH_CLIENT_ID`, `GITHUB_OAUTH_SECRET`, and + `GITHUB_OAUTH_REDIRECT_URI` to `.env`, then run + `sh run.sh config add github`. The redirect URI is + `/auth/v1/callback`; register it in the GitHub OAuth app. + See [upstream OAuth guidance](https://supabase.com/docs/guides/self-hosting/auth). + - Keep `STORAGE_TENANT_ID=stub`, `GLOBAL_S3_BUCKET=stub`, and `REGION=stub` + unless you have an existing deployment with different values. + +3. Pull and start the stack: + + ```sh + sh run.sh pull + sh run.sh start + sh run.sh status + ``` + + Open Studio at `http://localhost:8000` and use your generated + `DASHBOARD_USERNAME` and `DASHBOARD_PASSWORD`. On a server, expose the API + through HTTPS and restrict administration/database ports. The upstream + pooler publishes ports 5432 and 6543; do not allow public access to them. + Follow [upstream HTTPS guidance](https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https). + +4. Initialize the reVISit table and private bucket **on a fresh database only**: + + ```sh + docker exec -i supabase-db psql -U supabase_admin -d postgres \ + -v ON_ERROR_STOP=1 < revisit.sql + ``` + + This applies the same schema and access model as the existing + [reVISit setup instructions](https://revisit.dev/docs/data-and-deployment/supabase/setup/). + Those policies allow anyone holding the public application key to read and + write reVISit records and objects. A private bucket prevents unauthenticated + public URLs; it does **not** provide participant-level isolation under these + policies. Keep your existing stricter policies when migrating. This upgrade + does not redesign authorization. + +5. Configure the `.env` at the root of your **reVISit application**: + + ```dotenv + VITE_STORAGE_ENGINE="supabase" + VITE_SUPABASE_URL="http://localhost:8000" + VITE_SUPABASE_ANON_KEY="" + ``` + + The application variable retains its existing name and accepts the new + publishable key. Never put `SERVICE_ROLE_KEY` or `SUPABASE_SECRET_KEY` in + application configuration. Restart `yarn serve` or rebuild/redeploy the app + after changing these build-time variables. + +6. Complete a short study, confirm its participant data in Studio, and restart + Supabase. Confirm the original data still opens and another participant can + submit answers. Exercise your configured Study Designer sign-in as well. + +## Storage and ongoing operations + +Files persist under `volumes/storage`; Postgres persists under `volumes/db/data`. +The `db-config` named volume contains encryption material and must also be backed +up. **Never use `docker compose down -v` on a deployment you intend to keep.** +Back up files and database together with writes paused, store backups outside +the server, test restores, and monitor disk capacity. File storage is suitable +for a single-server deployment; it does not replicate data or provide failover. +For deployments requiring object storage, use +[upstream S3 configuration](https://supabase.com/docs/guides/self-hosting/self-hosted-s3). + +Use `sh run.sh stop`, `start`, `status`, and `logs` to operate the stack. +Review future tagged updates using `sh update.sh --dry-run --to ` +and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). +This repository bundles the default stack and database upgrade support; +upstream updates can introduce additional files or changes to setup instructions. +Review the plan and reVISit-specific files before applying an update. + +## Local regression checks + +The migration helper requires Python 3.11+ and no third-party Python packages: + +```sh +python3 tests/migrate-storage.spec.py +``` + +The opt-in Chromium test is `../tests/supabase-live.spec.ts`. It needs an isolated +running stack, initialized `revisit.sql`, and the application's Supabase +environment variables. Set `PW_SUPABASE_LIVE=1` and +`PW_SUPABASE_SERVICE_ROLE_KEY` privately, then run the focused suite from the +repository root with `yarn exec playwright test tests/supabase-live.spec.ts --project chromium`. +Use `PW_SUPABASE_PARTICIPANT_ID` to retain a known UUID before migration and +`PW_SUPABASE_EXISTING_PARTICIPANT` to verify it afterwards. The test creates real +participant records; do not point it at production. diff --git a/supabase/UPSTREAM b/supabase/UPSTREAM new file mode 100644 index 0000000000..9e43450ea2 --- /dev/null +++ b/supabase/UPSTREAM @@ -0,0 +1,8 @@ +Supabase self-hosted/v0.8.2 +Commit: 564eab8ad7840b13324f68b1bfac074ef8d51c21 +Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.2/docker + +Compose, configuration, and upstream scripts are copied without edits. +Only the default stack and PostgreSQL migration support are bundled. +README.md, MIGRATION.md, revisit.sql, migrate-storage.py, tests/, and +docker-compose.github.yml are reVISit additions. diff --git a/supabase/docker-compose.github.yml b/supabase/docker-compose.github.yml new file mode 100644 index 0000000000..003f59dd16 --- /dev/null +++ b/supabase/docker-compose.github.yml @@ -0,0 +1,8 @@ +# Preserve the GitHub OAuth variables used by the previous reVISit stack. +services: + auth: + environment: + GOTRUE_EXTERNAL_GITHUB_ENABLED: ${ENABLE_GITHUB_OAUTH:-false} + GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: ${GITHUB_OAUTH_CLIENT_ID:-} + GOTRUE_EXTERNAL_GITHUB_SECRET: ${GITHUB_OAUTH_SECRET:-} + GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: ${GITHUB_OAUTH_REDIRECT_URI:-} diff --git a/supabase/docker-compose.pg15.yml b/supabase/docker-compose.pg15.yml new file mode 100644 index 0000000000..f87fe8c482 --- /dev/null +++ b/supabase/docker-compose.pg15.yml @@ -0,0 +1,18 @@ +# Postgres 15 override for self-hosted Supabase. +# +# Postgres 17 is now the default in docker-compose.yml. Use this override to pin +# Postgres 15, for two cases: +# 1. An existing Postgres 15 deployment that has not been upgraded yet. +# 2. As the rollback target for utils/upgrade-pg17.sh (PG 15 and PG 17 use +# different postgres UIDs, so rollback must start with the PG 15 image). +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d +# +# To upgrade an existing Postgres 15 database to Postgres 17 in place, see: +# - https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17 +# - utils/upgrade-pg17.sh + +services: + db: + image: supabase/postgres:15.8.1.085 diff --git a/supabase/docker-compose.pg17.yml b/supabase/docker-compose.pg17.yml new file mode 100644 index 0000000000..a8e37117e1 --- /dev/null +++ b/supabase/docker-compose.pg17.yml @@ -0,0 +1,19 @@ +# Postgres 17 override for self-hosted Supabase. +# +# NOTE: Postgres 17 is now the default in docker-compose.yml, so this override +# is redundant for new installs (kept for explicitness / backwards compatibility). +# Keep the image tag here in sync with the db image in docker-compose.yml. +# +# Usage (new install or after upgrade): +# docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d +# +# For upgrading an existing Postgres 15 database, run utils/upgrade-pg17.sh first. +# +# Note: PG 15 and PG 17 use different postgres UIDs. If starting fresh with a +# leftover db-config volume from PG 15, you may see +# "FATAL: invalid secret key". Either remove the old volume or fix ownership. +# See: docs/guides/self-hosting/postgres-upgrade-17 + +services: + db: + image: supabase/postgres:17.6.1.136 diff --git a/supabase/docker-compose.yml b/supabase/docker-compose.yml index 0d224b6414..a37d39dc7c 100644 --- a/supabase/docker-compose.yml +++ b/supabase/docker-compose.yml @@ -1,71 +1,114 @@ +# Usage +# Start: docker compose up -d +# Stop: docker compose down +# Dev mode: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml up -d +# Reset everything: sh reset.sh +# +# Notes: +# - Nested variable interpolation (${A:-${B}}) requires podman-compose >= 1.6.0 +# + name: supabase services: studio: container_name: supabase-studio - image: supabase/studio:2025.04.21-sha-173cc56 + image: supabase/studio:2026.09.07-sha-7996410 restart: unless-stopped healthcheck: test: [ - "CMD", - "node", - "-e", - "fetch('http://studio:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})" + "CMD-SHELL", + "node -e \"fetch('http://localhost:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})\"" ] timeout: 10s interval: 5s retries: 3 - depends_on: - analytics: - condition: service_healthy + start_period: 20s environment: + # Listen on all IPv4 interfaces + HOSTNAME: "0.0.0.0" + STUDIO_PG_META_URL: http://meta:8080 + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_DB: ${POSTGRES_DB} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + # See: https://supabase.com/docs/guides/self-hosting/remove-superuser-access + POSTGRES_USER_READ_WRITE: postgres + + PG_META_CRYPTO_KEY: ${PG_META_CRYPTO_KEY} + PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} + PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} + DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} + OPENAI_API_KEY: ${OPENAI_API_KEY} - SUPABASE_URL: http://kong:8000 + SUPABASE_URL: http://api-gw:8000 SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} SUPABASE_ANON_KEY: ${ANON_KEY} SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} AUTH_JWT_SECRET: ${JWT_SECRET} + SUPABASE_PUBLISHABLE_KEY: ${SUPABASE_PUBLISHABLE_KEY} + SUPABASE_SECRET_KEY: ${SUPABASE_SECRET_KEY} - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} - LOGFLARE_URL: http://analytics:4000 - NEXT_PUBLIC_ENABLE_LOGS: true - NEXT_ANALYTICS_BACKEND_PROVIDER: postgres + # See: docker-compose.logs.yml + ENABLED_FEATURES_LOGS_ALL: "false" - kong: - container_name: supabase-kong - image: kong:2.8.1 - restart: unless-stopped - ports: - - ${KONG_HTTP_PORT}:8000/tcp - - ${KONG_HTTPS_PORT}:8443/tcp + SNIPPETS_MANAGEMENT_FOLDER: /app/snippets + EDGE_FUNCTIONS_MANAGEMENT_FOLDER: /app/edge-functions volumes: - - ./volumes/api/kong.yml:/home/kong/temp.yml:ro,z + - ./volumes/snippets:/app/snippets:z + - ./volumes/functions:/app/edge-functions:ro,z + + # Envoy is the default API gateway + # See: https://github.com/orgs/supabase/discussions/48048 + api-gw: + container_name: supabase-envoy + image: envoyproxy/envoy:v1.39.1 + restart: unless-stopped + networks: + default: + # Expose `envoy` and `kong` as network aliases, so internal configs + # that reference either hostname resolve to whichever gateway is active. + aliases: + - envoy + - kong + healthcheck: + # Using a TCP port check because this image does not include curl or wget. + test: ["CMD-SHELL", "timeout 1 bash -c ' ~/kong.yml && /docker-entrypoint.sh kong docker-start' + entrypoint: ["/bin/sh", "/docker-entrypoint.sh"] auth: container_name: supabase-auth - image: supabase/gotrue:v2.171.0 + image: supabase/gotrue:v2.196.0 restart: unless-stopped healthcheck: test: @@ -82,8 +125,7 @@ services: retries: 3 depends_on: db: - condition: service_healthy - analytics: + # Disable this if you are using an external Postgres database condition: service_healthy environment: GOTRUE_API_HOST: 0.0.0.0 @@ -101,12 +143,25 @@ services: GOTRUE_JWT_AUD: authenticated GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated GOTRUE_JWT_EXP: ${JWT_EXPIRY} + + # Legacy symmetric HS256 key GOTRUE_JWT_SECRET: ${JWT_SECRET} + # JSON array of signing JWKs (EC private + legacy symmetric) + # For Podman, use: GOTRUE_JWT_KEYS: ${JWT_KEYS} + #GOTRUE_JWT_KEYS: ${JWT_KEYS:-[]} + + GOTRUE_JWT_ISSUER: ${API_EXTERNAL_URL} + GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} + # Uncomment to bypass nonce check in ID Token flow. Commonly set to true when using Google Sign In on mobile. + # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: "true" + + # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: "true" + # GOTRUE_SMTP_MAX_FREQUENCY: 1s GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} GOTRUE_SMTP_HOST: ${SMTP_HOST} GOTRUE_SMTP_PORT: ${SMTP_PORT} @@ -121,97 +176,314 @@ services: GOTRUE_EXTERNAL_PHONE_ENABLED: ${ENABLE_PHONE_SIGNUP} GOTRUE_SMS_AUTOCONFIRM: ${ENABLE_PHONE_AUTOCONFIRM} - GOTRUE_EXTERNAL_GITHUB_ENABLED: ${ENABLE_GITHUB_OAUTH} - GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: ${GITHUB_OAUTH_CLIENT_ID} - GOTRUE_EXTERNAL_GITHUB_SECRET: ${GITHUB_OAUTH_SECRET} - GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: ${GITHUB_OAUTH_REDIRECT_URI} + # Uncomment to enable OAuth / social login providers. + # GOTRUE_EXTERNAL_GOOGLE_ENABLED: ${GOOGLE_ENABLED} + # GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID} + # GOTRUE_EXTERNAL_GOOGLE_SECRET: ${GOOGLE_SECRET} + # GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI: ${API_EXTERNAL_URL}/callback + + # GOTRUE_EXTERNAL_GITHUB_ENABLED: ${GITHUB_ENABLED} + # GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID} + # GOTRUE_EXTERNAL_GITHUB_SECRET: ${GITHUB_SECRET} + # GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: ${API_EXTERNAL_URL}/callback + + # GOTRUE_EXTERNAL_AZURE_ENABLED: ${AZURE_ENABLED} + # GOTRUE_EXTERNAL_AZURE_CLIENT_ID: ${AZURE_CLIENT_ID} + # GOTRUE_EXTERNAL_AZURE_SECRET: ${AZURE_SECRET} + # GOTRUE_EXTERNAL_AZURE_REDIRECT_URI: ${API_EXTERNAL_URL}/callback + + # Uncomment to configure SMS delivery (phone auth and phone MFA). + # GOTRUE_SMS_PROVIDER: ${SMS_PROVIDER} + # GOTRUE_SMS_OTP_EXP: ${SMS_OTP_EXP} + # GOTRUE_SMS_OTP_LENGTH: ${SMS_OTP_LENGTH} + # GOTRUE_SMS_MAX_FREQUENCY: ${SMS_MAX_FREQUENCY} + # GOTRUE_SMS_TEMPLATE: ${SMS_TEMPLATE} + + # Twilio credentials (when SMS_PROVIDER=twilio) + # GOTRUE_SMS_TWILIO_ACCOUNT_SID: ${SMS_TWILIO_ACCOUNT_SID} + # GOTRUE_SMS_TWILIO_AUTH_TOKEN: ${SMS_TWILIO_AUTH_TOKEN} + # GOTRUE_SMS_TWILIO_MESSAGE_SERVICE_SID: ${SMS_TWILIO_MESSAGE_SERVICE_SID} + + # Test OTP mappings for development + # GOTRUE_SMS_TEST_OTP: ${SMS_TEST_OTP} + + # Uncomment to configure multi-factor authentication (MFA). + # GOTRUE_MFA_TOTP_ENROLL_ENABLED: ${MFA_TOTP_ENROLL_ENABLED} + # GOTRUE_MFA_TOTP_VERIFY_ENABLED: ${MFA_TOTP_VERIFY_ENABLED} + # GOTRUE_MFA_PHONE_ENROLL_ENABLED: ${MFA_PHONE_ENROLL_ENABLED} + # GOTRUE_MFA_PHONE_VERIFY_ENABLED: ${MFA_PHONE_VERIFY_ENABLED} + # GOTRUE_MFA_MAX_ENROLLED_FACTORS: ${MFA_MAX_ENROLLED_FACTORS} + + # SAML SSO + # See: https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso + # GOTRUE_SAML_ENABLED: ${SAML_ENABLED} + # GOTRUE_SAML_PRIVATE_KEY: ${SAML_PRIVATE_KEY} + # GOTRUE_SAML_ALLOW_ENCRYPTED_ASSERTIONS: ${SAML_ALLOW_ENCRYPTED_ASSERTIONS} + # GOTRUE_SAML_RELAY_STATE_VALIDITY_PERIOD: ${SAML_RELAY_STATE_VALIDITY_PERIOD} + # GOTRUE_SAML_RATE_LIMIT_ASSERTION: ${SAML_RATE_LIMIT_ASSERTION} + # Optional, defaults to API_EXTERNAL_URL if not set + # GOTRUE_SAML_EXTERNAL_URL: ${SAML_EXTERNAL_URL} + + # Uncomment to enable custom access token hook. + # See: https://supabase.com/docs/guides/auth/auth-hooks for + # full list of hooks and additional details about custom_access_token_hook + + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED: "true" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_SECRETS: "" + + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/mfa_verification_attempt" + + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/password_verification_attempt" + + # GOTRUE_HOOK_SEND_SMS_ENABLED: "false" + # GOTRUE_HOOK_SEND_SMS_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_SEND_SMS_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" + + # GOTRUE_HOOK_SEND_EMAIL_ENABLED: "false" + # GOTRUE_HOOK_SEND_EMAIL_URI: "http://host.docker.internal:54321/functions/v1/email_sender" + # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" rest: container_name: supabase-rest - image: postgrest/postgrest:v12.2.11 + image: postgrest/postgrest:v14.17 restart: unless-stopped depends_on: db: + # Disable this if you are using an external Postgres database condition: service_healthy - analytics: - condition: service_healthy + healthcheck: + test: [ "CMD", "postgrest", "--ready" ] + interval: 5s + timeout: 5s + retries: 3 environment: PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} + PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} PGRST_DB_ANON_ROLE: anon - PGRST_JWT_SECRET: ${JWT_SECRET} + + PGRST_ADMIN_SERVER_PORT: 3001 + PGRST_ADMIN_SERVER_HOST: localhost + + # PostgREST accepts a plain-text symmetric secret, a single JWK, or a JWKS. + # For Podman, use either PGRST_JWT_SECRET: ${JWT_SECRET} or + # PGRST_JWT_SECRET: ${JWT_JWKS} + PGRST_JWT_SECRET: ${JWT_JWKS:-${JWT_SECRET}} + PGRST_DB_USE_LEGACY_GUCS: "false" - PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} command: [ "postgrest" ] - meta: - container_name: supabase-meta - image: supabase/postgres-meta:v0.88.9 + realtime: + # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain + container_name: realtime-dev.supabase-realtime + image: supabase/realtime:v2.134.10 restart: unless-stopped depends_on: db: + # Disable this if you are using an external Postgres database condition: service_healthy - analytics: + healthcheck: + test: + [ + "CMD-SHELL", + "curl -sSfL --head -o /dev/null -H \"Authorization: Bearer ${ANON_KEY}\" http://localhost:4000/api/tenants/realtime-dev/health" + ] + timeout: 5s + interval: 30s + retries: 3 + start_period: 10s + environment: + PORT: 4000 + DB_HOST: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_USER: supabase_admin + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_NAME: ${POSTGRES_DB} + DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' + DB_ENC_KEY: ${REALTIME_DB_ENC_KEY:-supabaserealtime} + + # Legacy symmetric HS256 key + API_JWT_SECRET: ${JWT_SECRET} + + # JWKS for token verification (EC public + legacy symmetric). + # For Podman, use: API_JWT_JWKS: ${JWT_JWKS} + #API_JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} + + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + METRICS_JWT_SECRET: ${JWT_SECRET} + ERL_AFLAGS: -proto_dist inet_tcp + DNS_NODES: "''" + RLIMIT_NOFILE: "10000" + APP_NAME: realtime + SEED_SELF_HOST: "true" + RUN_JANITOR: "true" + DISABLE_HEALTHCHECK_LOGGING: "true" + + # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up + storage: + container_name: supabase-storage + image: supabase/storage-api:v1.74.0 + restart: unless-stopped + depends_on: + db: + # Disable this if you are using an external Postgres database condition: service_healthy + rest: + condition: service_started + imgproxy: + condition: service_started + healthcheck: + test: + [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://storage:5000/status" + ] + timeout: 5s + interval: 5s + retries: 3 + start_period: 10s environment: - PG_META_PORT: 8080 - PG_META_DB_HOST: ${POSTGRES_HOST} - PG_META_DB_PORT: ${POSTGRES_PORT} - PG_META_DB_NAME: ${POSTGRES_DB} - PG_META_DB_USER: supabase_admin - PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + ANON_KEY: ${ANON_KEY} + SERVICE_KEY: ${SERVICE_ROLE_KEY} + POSTGREST_URL: http://rest:3000 + + # Legacy symmetric HS256 key + AUTH_JWT_SECRET: ${JWT_SECRET} + + # JWKS for token verification (EC public + legacy symmetric). + # For Podman, use: JWT_JWKS: ${JWT_JWKS} + #JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} + + DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + STORAGE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} + REQUEST_ALLOW_X_FORWARDED_PATH: "true" + FILE_SIZE_LIMIT: 52428800 + STORAGE_BACKEND: file + # S3 bucket when using S3 backend, directory name when using 'file' + GLOBAL_S3_BUCKET: ${GLOBAL_S3_BUCKET} + # S3 Backend configuration + #GLOBAL_S3_ENDPOINT: https://your-s3-endpoint + #GLOBAL_S3_PROTOCOL: https + #GLOBAL_S3_FORCE_PATH_STYLE: "true" + #AWS_ACCESS_KEY_ID: your-access-key-id + #AWS_SECRET_ACCESS_KEY: your-secret-access-key + FILE_STORAGE_BACKEND_PATH: /var/lib/storage + TENANT_ID: ${STORAGE_TENANT_ID} + # TODO: https://github.com/supabase/storage-api/issues/55 + REGION: ${REGION} + ENABLE_IMAGE_TRANSFORMATION: "true" + IMGPROXY_URL: http://imgproxy:5001 + # S3 protocol endpoint configuration + S3_PROTOCOL_ACCESS_KEY_ID: ${S3_PROTOCOL_ACCESS_KEY_ID} + S3_PROTOCOL_ACCESS_KEY_SECRET: ${S3_PROTOCOL_ACCESS_KEY_SECRET} + volumes: + - ./volumes/storage:/var/lib/storage:z - analytics: - container_name: supabase-analytics - image: supabase/logflare:1.22.1 + imgproxy: + container_name: supabase-imgproxy + image: darthsim/imgproxy:v3.31.4 restart: unless-stopped - ports: - - 4000:4000 + volumes: + - ./volumes/storage:/var/lib/storage:z healthcheck: test: [ "CMD", - "curl", - "http://localhost:4000/health" + "imgproxy", + "health" ] timeout: 5s interval: 5s - retries: 10 + retries: 3 + environment: + IMGPROXY_BIND: ":5001" + IMGPROXY_LOCAL_FILESYSTEM_ROOT: / + IMGPROXY_USE_ETAG: "true" + IMGPROXY_AUTO_WEBP: ${IMGPROXY_AUTO_WEBP} + IMGPROXY_MAX_SRC_RESOLUTION: 16.8 + + meta: + container_name: supabase-meta + image: supabase/postgres-meta:v0.99.0 + restart: unless-stopped depends_on: db: + # Disable this if you are using an external Postgres database condition: service_healthy environment: - LOGFLARE_NODE_HOST: 127.0.0.1 - DB_USERNAME: supabase_admin - DB_DATABASE: _supabase - DB_HOSTNAME: ${POSTGRES_HOST} - DB_PORT: ${POSTGRES_PORT} - DB_PASSWORD: ${POSTGRES_PASSWORD} - DB_SCHEMA: _analytics - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} - LOGFLARE_SINGLE_TENANT: true - LOGFLARE_SUPABASE_MODE: true - LOGFLARE_MIN_CLUSTER_SIZE: 1 - LOGFLARE_LOG_LEVEL: warn + PG_META_PORT: 8080 + PG_META_DB_HOST: ${POSTGRES_HOST} + PG_META_DB_PORT: ${POSTGRES_PORT} + PG_META_DB_NAME: ${POSTGRES_DB} + PG_META_DB_USER: postgres + PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + CRYPTO_KEY: ${PG_META_CRYPTO_KEY} - POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase - POSTGRES_BACKEND_SCHEMA: _analytics - LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true + functions: + container_name: supabase-edge-functions + image: supabase/edge-runtime:v1.76.2 + restart: unless-stopped + volumes: + - ./volumes/functions:/home/deno/functions:z + - deno-cache:/root/.cache/deno + depends_on: + api-gw: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c ' - /bin/sh -c " - /usr/bin/mc alias set supa-minio http://minio:9000 supa-storage secret1234; - /usr/bin/mc mb --ignore-existing supa-minio/stub; - exit 0; - " - storage: - container_name: supabase-storage - image: supabase/storage-api:v1.11.13 - depends_on: - db: - condition: service_healthy - rest: - condition: service_started - minio: - condition: service_healthy + - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z healthcheck: test: [ "CMD", - "wget", - "--no-verbose", - "--tries=1", - "--spider", - "http://127.0.0.1:5000/status" + "curl", + "-sSfL", + "--head", + "-o", + "/dev/null", + "http://127.0.0.1:4000/api/health" ] + interval: 10s timeout: 5s - interval: 5s - retries: 3 - restart: unless-stopped + retries: 10 + start_period: 30s + depends_on: + db: + condition: service_healthy environment: - ANON_KEY: ${ANON_KEY} - SERVICE_KEY: ${SERVICE_ROLE_KEY} - POSTGREST_URL: http://rest:3000 - PGRST_JWT_SECRET: ${JWT_SECRET} - DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - FILE_SIZE_LIMIT: 52428800 - STORAGE_BACKEND: s3 - GLOBAL_S3_BUCKET: stub - GLOBAL_S3_ENDPOINT: http://minio:9000 - GLOBAL_S3_PROTOCOL: http - GLOBAL_S3_FORCE_PATH_STYLE: true - AWS_ACCESS_KEY_ID: supa-storage - AWS_SECRET_ACCESS_KEY: secret1234 - AWS_DEFAULT_REGION: stub - FILE_STORAGE_BACKEND_PATH: /var/lib/storage - TENANT_ID: stub - REGION: stub - ENABLE_IMAGE_TRANSFORMATION: "true" - volumes: - - ./volumes/storage:/var/lib/storage:z + PORT: 4000 + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + CLUSTER_POSTGRES: "true" + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + VAULT_ENC_KEY: ${VAULT_ENC_KEY} + API_JWT_SECRET: ${JWT_SECRET} + METRICS_JWT_SECRET: ${JWT_SECRET} + REGION: local + ERL_AFLAGS: -proto_dist inet_tcp + POOLER_TENANT_ID: ${POOLER_TENANT_ID} + POOLER_DEFAULT_POOL_SIZE: ${POOLER_DEFAULT_POOL_SIZE} + POOLER_MAX_CLIENT_CONN: ${POOLER_MAX_CLIENT_CONN} + POOLER_POOL_MODE: transaction + DB_POOL_SIZE: ${POOLER_DB_POOL_SIZE} + command: + [ + "/bin/sh", + "-c", + "/app/bin/migrate && /app/bin/supavisor eval \"$$(cat /etc/pooler/pooler.exs)\" && /app/bin/server" + ] volumes: db-config: + deno-cache: diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py new file mode 100644 index 0000000000..d6b5220181 --- /dev/null +++ b/supabase/migrate-storage.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Offline MinIO -> Supabase v1.74.0 file-backend migration. No database writes. + +Export runs with the old stack online and application writes stopped. Install +runs on Linux with the new Storage service stopped. See MIGRATION.md. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import urllib.parse +import urllib.request + + +def objects(): + sql = "SELECT coalesce(json_agg(o ORDER BY bucket_id, name), '[]'::json) FROM storage.objects o" + result = subprocess.run( + ["docker", "exec", "supabase-db", "psql", "-U", "supabase_admin", + "-d", "postgres", "-At", "-v", "ON_ERROR_STOP=1", "-c", sql], + check=True, capture_output=True, text=True, + ) + return json.loads(result.stdout) + + +def download(url, obj): + key = urllib.parse.quote(f"{obj['bucket_id']}/{obj['name']}", safe="/") + token = os.environ["SUPABASE_SERVICE_ROLE_KEY"] + request = urllib.request.Request( + f"{url.rstrip('/')}/storage/v1/object/authenticated/{key}", + headers={"apikey": token, "Authorization": f"Bearer {token}"}, + ) + return urllib.request.urlopen(request, timeout=120) + + +def digest(file): + with file.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def export(url, directory): + directory.mkdir(parents=True, exist_ok=False) + manifest = objects() + for index, obj in enumerate(manifest): + obj["file"] = f"{index:08d}.bin" + target = directory / obj["file"] + with download(url, obj) as response, target.open("wb") as output: + obj["content_type"] = response.headers.get("Content-Type", "application/octet-stream") + obj["cache_control"] = response.headers.get("Cache-Control", "no-cache") + shutil.copyfileobj(response, output) + obj["sha256"] = digest(target) + size = (obj.get("metadata") or {}).get("size") + if size is not None and target.stat().st_size != int(size): + raise ValueError("Downloaded object size differs from database metadata") + (directory / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(f"Exported {len(manifest)} objects") + + +def destination(root, bucket, tenant, obj): + # These match v1.74.0's default location and withOptionalVersion helpers. + parts = [bucket, tenant, obj["bucket_id"], *obj["name"].split("/")] + if obj.get("version"): + parts.append(obj["version"]) + if any(not part or part in (".", "..") or "/" in part or "\\" in part for part in parts): + raise ValueError("Unsafe storage object path") + target = root.joinpath(*parts) + if not target.resolve().is_relative_to(root.resolve()): + raise ValueError("Storage path escapes destination") + return target + + +def install(directory, root, bucket, tenant): + if not hasattr(os, "setxattr"): + raise RuntimeError("Install requires Linux extended attributes; see MIGRATION.md") + manifest = json.loads((directory / "manifest.json").read_text()) + # Validate the entire export before changing the destination. + for index, obj in enumerate(manifest): + if obj["file"] != f"{index:08d}.bin" or digest(directory / obj["file"]) != obj["sha256"]: + raise ValueError("Export checksum mismatch") + target = destination(root, bucket, tenant, obj) + if target.exists() and digest(target) != obj["sha256"]: + raise ValueError("Destination already contains different data") + for obj in manifest: + target = destination(root, bucket, tenant, obj) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(directory / obj["file"], target) + # Storage reads HTTP metadata from Linux extended attributes. + os.setxattr(target, "user.supabase.content-type", obj["content_type"].encode()) + os.setxattr(target, "user.supabase.cache-control", obj["cache_control"].encode()) + print(f"Installed {len(manifest)} objects; database metadata unchanged") + + +def verify(url, directory): + manifest = json.loads((directory / "manifest.json").read_text()) + current = objects() + exported_columns = set(manifest[0]) - {"file", "sha256", "content_type", "cache_control"} if manifest else set() + original = {obj["id"]: {key: obj[key] for key in exported_columns} for obj in manifest} + retained = {obj["id"]: {key: obj.get(key) for key in exported_columns} for obj in current} + if retained != original or len(current) != len(manifest): + raise ValueError("Database object metadata changed; verify before reopening writes") + for obj in manifest: + hasher = hashlib.sha256() + with download(url, obj) as response: + if response.headers.get("Content-Type") != obj["content_type"]: + raise ValueError("Object content type changed") + if response.headers.get("Cache-Control") != obj["cache_control"]: + raise ValueError("Object cache control changed") + while chunk := response.read(1024 * 1024): + hasher.update(chunk) + if hasher.hexdigest() != obj["sha256"]: + raise ValueError("Migrated object checksum mismatch") + print(f"Verified {len(manifest)} objects and unchanged database metadata") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=["export", "install", "verify"]) + parser.add_argument("directory", type=Path) + parser.add_argument("--url", default="http://localhost:8000") + parser.add_argument("--destination", type=Path) + parser.add_argument("--bucket", default="stub", help="GLOBAL_S3_BUCKET, not the application bucket") + parser.add_argument("--tenant", default="stub", help="STORAGE_TENANT_ID (previously TENANT_ID)") + args = parser.parse_args() + if args.operation == "install": + if args.destination is None: + parser.error("install requires --destination") + install(args.directory, args.destination, args.bucket, args.tenant) + elif args.operation == "export": + export(args.url, args.directory) + else: + verify(args.url, args.directory) + + +if __name__ == "__main__": + main() diff --git a/supabase/revisit.sql b/supabase/revisit.sql new file mode 100644 index 0000000000..635837594f --- /dev/null +++ b/supabase/revisit.sql @@ -0,0 +1,15 @@ +-- Match the existing documented access model; see README.md before applying. +BEGIN; +CREATE TABLE public.revisit ( + "createdAt" timestamp DEFAULT now(), + "studyId" varchar NOT NULL, + "docId" varchar NOT NULL, + data jsonb, + PRIMARY KEY ("studyId", "docId") +); +ALTER TABLE public.revisit ENABLE ROW LEVEL SECURITY; +GRANT SELECT, INSERT, UPDATE, DELETE ON public.revisit TO anon, authenticated, service_role; +CREATE POLICY allow_authenticated_read_write ON public.revisit FOR ALL TO anon, authenticated, service_role USING (true) WITH CHECK (true); +INSERT INTO storage.buckets (id, name, public) VALUES ('revisit', 'revisit', false); +CREATE POLICY revisit_storage_read_write ON storage.objects FOR ALL TO anon, authenticated, service_role USING (bucket_id = 'revisit') WITH CHECK (bucket_id = 'revisit'); +COMMIT; diff --git a/supabase/run.sh b/supabase/run.sh new file mode 100755 index 0000000000..397816869d --- /dev/null +++ b/supabase/run.sh @@ -0,0 +1,297 @@ +#!/bin/sh +# +# Manage the self-hosted Supabase docker compose stack. +# +# Override files are layered via docker compose's native COMPOSE_FILE env +# var in .env. Format: colon-separated list with docker-compose.yml first. +# +# Examples in .env: +# COMPOSE_FILE=docker-compose.yml +# COMPOSE_FILE=docker-compose.yml:docker-compose.pg17.yml +# +# Manage with: sh run.sh config add | config remove +# (accepts either a short name like 'pg17' or 'docker-compose.pg17.yml') +# +# Usage: +# sh run.sh start # docker compose up -d --wait +# sh run.sh stop # docker compose down +# sh run.sh restart [service] # restart the stack (or named services) +# sh run.sh restart --except ... # restart all services except the named ones +# sh run.sh recreate [service] # stop then start (or force-recreate one service) +# sh run.sh recreate --except ... # force-recreate all services except the named ones +# sh run.sh status # docker compose ps +# sh run.sh logs [service] # follow logs (all or one service) +# sh run.sh inspect # docker inspect on a service's container +# sh run.sh printenv # print a service's environment variables +# sh run.sh pull # pull images +# sh run.sh config # show the active COMPOSE_FILE list +# sh run.sh config add # add an override to COMPOSE_FILE in .env +# sh run.sh config remove # remove an override from COMPOSE_FILE in .env +# sh run.sh compose-config # dump fully-resolved docker compose config +# sh run.sh secrets # print key passwords and API keys from .env +# + +set -e + +cd "$(dirname "$0")" + +if [ ! -f docker-compose.yml ]; then + echo "ERROR: docker-compose.yml not found in $(pwd)" >&2 + exit 1 +fi + +# Normalize an override argument: +# pg17 -> docker-compose.pg17.yml +# docker-compose.pg17.yml -> docker-compose.pg17.yml +# ./docker-compose.pg17.yml -> docker-compose.pg17.yml +# docker-compose.yml -> error (base file, always implicit) +normalize_override() { + arg="${1#./}" + case "$arg" in + docker-compose.yml) + echo "ERROR: docker-compose.yml is the base file, always included" >&2 + return 1 + ;; + docker-compose.*.yml) + echo "$arg" + ;; + *) + echo "docker-compose.${arg}.yml" + ;; + esac +} + +# Read COMPOSE_FILE from .env (stripping quotes and CR). +read_compose_file() { + [ -f .env ] || return 0 + grep '^COMPOSE_FILE=' .env | head -n1 | cut -d= -f2- | tr -d "\r\"'" +} + +# Pretty-print the effective compose file list. +print_config() { + val="$1" + [ -z "$val" ] && val="docker-compose.yml" + echo "COMPOSE_FILE=$val" + echo "compose files:" + OLD_IFS=$IFS + IFS=: + for f in $val; do + echo " $f" + done + IFS=$OLD_IFS + echo "" +} + +# Update or append COMPOSE_FILE in .env. +write_compose_file() { + new_value="$1" + if [ ! -f .env ]; then + echo "ERROR: .env not found in $(pwd)" >&2 + exit 1 + fi + new_line="COMPOSE_FILE=$new_value" + if grep -q '^COMPOSE_FILE=' .env; then + sed -i.old -e "s|^COMPOSE_FILE=.*$|$new_line|" .env + rm -f .env.old + else + cat >> .env <. +# +# Examples: +# COMPOSE_FILE=docker-compose.yml +# COMPOSE_FILE=docker-compose.yml:docker-compose.pg17.yml +############ +$new_line +EOF + fi +} + +# Echoes the list of services (one per line) minus those passed as args. +# Warns on unknown names; returns 1 if no services remain. +services_except() { + all_services=$(docker compose config --services) + filtered="$all_services" + for ex in "$@"; do + echo "$all_services" | grep -qFx "$ex" \ + || echo "Warning: '$ex' is not a service in this project" >&2 + filtered=$(echo "$filtered" | grep -vFx "$ex" || true) + done + if [ -z "$filtered" ]; then + echo "No services left after applying --except" >&2 + return 1 + fi + printf '%s\n' "$filtered" +} + +CMD="${1:-help}" +[ "$#" -gt 0 ] && shift + +case "$CMD" in + start|up) + exec docker compose up -d --wait "$@" + ;; + stop|down) + exec docker compose down "$@" + ;; + restart) + if [ "${1:-}" = "--except" ]; then + shift + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") restart --except ..." >&2; exit 1; } + services=$(services_except "$@") || exit 1 + # shellcheck disable=SC2086 + exec docker compose restart $services + fi + exec docker compose restart "$@" + ;; + recreate) + if [ "${1:-}" = "--except" ]; then + shift + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") recreate --except ..." >&2; exit 1; } + services=$(services_except "$@") || exit 1 + # shellcheck disable=SC2086 + exec docker compose up -d --wait --force-recreate --no-deps $services + fi + if [ $# -eq 0 ]; then + docker compose down + exec docker compose up -d --wait + fi + # Single-service recreate: force-recreate the named services only, + # leave their dependencies running. + exec docker compose up -d --wait --force-recreate --no-deps "$@" + ;; + status|ps) + exec docker compose ps "$@" + ;; + logs) + exec docker compose logs -f "$@" + ;; + inspect) + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") inspect [docker-inspect-args]" >&2; exit 1; } + svc="$1"; shift + cid=$(docker compose ps -q "$svc") + [ -z "$cid" ] && { echo "Service '$svc' is not running" >&2; exit 1; } + exec docker inspect "$cid" "$@" + ;; + printenv) + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") printenv " >&2; exit 1; } + svc="$1" + cid=$(docker compose ps -q "$svc") + [ -z "$cid" ] && { echo "Service '$svc' is not running" >&2; exit 1; } + exec docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$cid" + ;; + pull) + exec docker compose pull "$@" + ;; + compose-config) + exec docker compose config "$@" + ;; + config) + sub="${1:-show}" + [ "$#" -gt 0 ] && shift + current=$(read_compose_file) + case "$sub" in + show) + print_config "$current" + ;; + add) + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") config add ..." >&2; exit 1; } + new_value="${current:-docker-compose.yml}" + changed=false + for arg in "$@"; do + file=$(normalize_override "$arg") || exit 1 + if [ ! -f "$file" ]; then + echo "ERROR: $file not found" >&2 + exit 1 + fi + case ":$new_value:" in + *":$file:"*) echo "Already present: $file" ;; + *) new_value="$new_value:$file"; changed=true ;; + esac + done + [ "$changed" = true ] && write_compose_file "$new_value" + print_config "$new_value" + ;; + remove|rm) + [ $# -eq 0 ] && { echo "Usage: $(basename "$0") config remove ..." >&2; exit 1; } + new_value="${current:-docker-compose.yml}" + changed=false + for arg in "$@"; do + file=$(normalize_override "$arg") || exit 1 + case ":$new_value:" in + *":$file:"*) + # Drop $file by rebuilding the colon list + tmp="" + OLD_IFS=$IFS + IFS=: + for tok in $new_value; do + [ "$tok" = "$file" ] || tmp="${tmp:+$tmp:}$tok" + done + IFS=$OLD_IFS + new_value="$tmp" + changed=true + ;; + *) echo "Not present: $file" ;; + esac + done + [ "$changed" = true ] && write_compose_file "$new_value" + print_config "$new_value" + ;; + *) + echo "Unknown config subcommand: $sub" >&2 + echo "Use: config | config add ... | config remove ..." >&2 + exit 1 + ;; + esac + ;; + secrets) + if [ ! -f .env ]; then + echo "ERROR: .env not found in $(pwd)" >&2 + exit 1 + fi + for var in POSTGRES_PASSWORD DASHBOARD_PASSWORD \ + SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY \ + S3_PROTOCOL_ACCESS_KEY_ID S3_PROTOCOL_ACCESS_KEY_SECRET; do + line=$(grep "^${var}=" .env | head -n1) + if [ -n "$line" ]; then + echo "$line" + else + echo "${var}=" + fi + done + echo "" + ;; + help|-h|--help) + cat < + +Commands: + start Start the stack (docker compose up -d --wait) + stop Stop the stack (docker compose down) + restart [service] Restart the stack (or named services) + restart --except ... + Restart all services except the named ones + recreate [service] Stop then start, or force-recreate one service (--no-deps) + recreate --except ... + Force-recreate all services except the named ones (--no-deps) + status Show service status + logs [service] Follow logs (optionally for a single service) + inspect Inspect a service's container (forwards extra args to docker inspect) + printenv Print a service's environment variables (one per line) + pull Pull all images + config Show the active COMPOSE_FILE list + config add Add an override to COMPOSE_FILE in .env (short name or full filename) + config remove Remove an override from COMPOSE_FILE in .env + compose-config Dump the fully-resolved docker compose config + secrets Show key passwords and API keys from .env + +EOF + ;; + *) + echo "Unknown command: $CMD" >&2 + echo "Run '$0 help' for usage." >&2 + exit 1 + ;; +esac diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py new file mode 100644 index 0000000000..4359ae94cc --- /dev/null +++ b/supabase/tests/migrate-storage.spec.py @@ -0,0 +1,83 @@ +import hashlib +import json +from io import BytesIO +from pathlib import Path +import runpy +import tempfile +import unittest +from unittest.mock import patch + + +migration = runpy.run_path(str(Path(__file__).parents[1] / "migrate-storage.py")) + + +class StorageMigrationTests(unittest.TestCase): + def setUp(self): + attributes = patch("os.setxattr", create=True) + attributes.start() + self.addCleanup(attributes.stop) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.export = Path(self.temp.name) / "export" + self.export.mkdir() + self.target = Path(self.temp.name) / "storage" + data = b"existing participant data" + (self.export / "00000000.bin").write_bytes(data) + self.obj = {"bucket_id": "revisit", "name": "study/participant data.json", + "version": "existing-version", "file": "00000000.bin", + "sha256": hashlib.sha256(data).hexdigest(), + "content_type": "application/json", "cache_control": "max-age=3600"} + + def manifest(self, objects): + (self.export / "manifest.json").write_text(json.dumps(objects)) + + def test_preserves_versioned_layout_and_http_metadata(self): + self.manifest([self.obj]) + with patch("os.setxattr", create=True) as attributes: + migration["install"](self.export, self.target, "stub", "stub") + result = self.target / "stub/stub/revisit/study/participant data.json/existing-version" + self.assertEqual(result.read_bytes(), b"existing participant data") + attributes.assert_any_call(result, "user.supabase.content-type", b"application/json") + attributes.assert_any_call(result, "user.supabase.cache-control", b"max-age=3600") + + def test_corruption_is_rejected_before_installing_any_object(self): + self.manifest([self.obj, {**self.obj, "file": "00000001.bin"}]) + (self.export / "00000001.bin").write_bytes(b"corrupted") + with self.assertRaisesRegex(ValueError, "checksum"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + + def test_verify_ignores_row_order_but_rejects_changed_metadata(self): + objects = [{**self.obj, "id": "first"}, + {**self.obj, "id": "second", "name": "another object"}] + self.manifest(objects) + + def download(url, obj): + response = BytesIO(b"existing participant data") + response.headers = {"Content-Type": obj["content_type"], + "Cache-Control": obj["cache_control"]} + return response + + with patch.dict(migration["verify"].__globals__, + objects=lambda: list(reversed(objects)), download=download): + migration["verify"]("http://example.test", self.export) + changed = [{**objects[0], "version": "changed"}, objects[1]] + with patch.dict(migration["verify"].__globals__, objects=lambda: changed): + with self.assertRaisesRegex(ValueError, "metadata changed"): + migration["verify"]("http://example.test", self.export) + + def test_rejects_traversal_and_conflicting_destination(self): + self.manifest([{**self.obj, "name": "../outside"}]) + with self.assertRaisesRegex(ValueError, "Unsafe"): + migration["install"](self.export, self.target, "stub", "stub") + self.manifest([self.obj]) + result = migration["destination"](self.target, "stub", "stub", self.obj) + result.parent.mkdir(parents=True) + result.write_bytes(b"other data") + with self.assertRaisesRegex(ValueError, "different data"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(result.read_bytes(), b"other data") + + +if __name__ == "__main__": + unittest.main() diff --git a/supabase/update.sh b/supabase/update.sh new file mode 100755 index 0000000000..cf3313b639 --- /dev/null +++ b/supabase/update.sh @@ -0,0 +1,692 @@ +#!/bin/sh +# +# Update an existing self-hosted Supabase deployment in place. +# +# The deployment directory mixes vendor-owned files (docker-compose.yml, the +# override files, volumes/*, scripts, .env.example) with user-owned state +# (.env, docker-compose.override.yml, volumes/db/data, volumes/storage, etc.). +# This script pulls a newer version of the Supabase files on top of yours +# using a 3-way merge against the version you started from, so local edits +# survive and genuine conflicts are surfaced rather than silently overwritten. +# +# The version you started from is recorded in .supabase-version (written by +# setup.sh). If it is missing, pass --from or follow the printed guidance. +# +# What it never touches: .env values you set, docker-compose.override.yml, and +# the data directories (volumes/db/data, volumes/storage, etc.). New keys from +# .env.example are appended to your .env; existing values are kept as-is. +# +# By default it updates to the latest self-hosted/v* release tag (or 'master' +# until the first tag exists). Pass --to to pin a specific tag/branch. +# +# Usage: +# sh update.sh # update to the latest release tag +# sh update.sh --dry-run # show what would change, write nothing +# sh update.sh --to # update to a specific tag/branch +# sh update.sh --from # base to merge from (if no .supabase-version) +# sh update.sh --yes # don't prompt, even on breaking changes +# +# Env: +# SUPABASE_REPO_URL Override the upstream repo (default: github supabase/supabase) +# +# Documentation: https://supabase.com/docs/guides/self-hosting/updating +# + +set -e + +cd "$(dirname "$0")" + +# Pipeline (see main at the bottom): +# resolve refs -> fetch base+target snapshots -> [report-only exit] +# -> build manifest gate -> confirm_gate (before any writes) +# -> backup → merge vendor files + .env keys -> summary → stamp +# +# Three trees for every vendor file path: +# - base - upstream at BASE_REF (.supabase-version ref=, or --from) +# - target - upstream at TARGET_REF (--to, or latest self-hosted/v* tag) +# - user's - the deployment directory (cwd); _not_ a git checkout +# +# Git is only used to fetch snapshots (fetch_snapshot) and to run git merge-file. +# +# Breaking-change gate (upgrades.json on the target snapshot): +# - Keyed by version (e.g. "0.7.0"); window = entries in (BASE_VER, TARGET_VER], +# where the bounds are the base/target refs reduced to bare semver +# (self-hosted/vX.Y.Z -> X.Y.Z) and compared with sort -V. +# - If a bound is not a release tag (a commit SHA, or "master"), that side of +# the window is left open and all applicable entries are shown with a warning. +# - Prompt when an entry has breaking:true or a gate script; runs before +# backup/merge so abort leaves the deployment untouched. +# - CHANGELOG.md is never parsed (update.sh only points users at it); routine +# "requires compose update" items are applied by the merge, not listed here. +# +# User-owned paths skipped during merge are defined in .gitignore (loaded from the +# target snapshot). git check-ignore --no-index applies negation rules (e.g. +# volumes/functions/** ignored except volumes/functions/main/index.ts). +# .git/ is excluded here only - never listed in .gitignore. +# +# .env is never 3-way merged: append missing keys from .env.example only. + +# --- globals (set during main) ----------------------------------------------- + +REPO_URL="${SUPABASE_REPO_URL:-https://github.com/supabase/supabase}" +STAMP_FILE=".supabase-version" +SELF_NAME=$(basename "$0") +DRY_RUN=0 +ASSUME_YES=0 +TO_REF="" +FROM_REF="" + +TARGET_REF="" +BASE_REF="" +BASE_VER="" +TARGET_VER="" +REPORT_ONLY=0 + +TMP_ROOT="" +TARGET_DIR="" +BASE_DIR="" +REPORT="" +ENV_ADDED="" +ENV_REMOVED="" +GATE_REPORT="" +GATE_REQUIRED=0 +IGNORE_FILE="" +IGNORE_GIT_DIR="" + +# --- logging ----------------------------------------------------------------- + +log() { printf "===> %s\n" "$*"; } +warn() { printf "WARNING: %s\n" "$*" >&2; } +die() { printf "ERROR: %s\n" "$*" >&2; exit 1; } + +print_help() { + awk 'NR==1 {next} /^#/ {sub(/^# ?/,""); print; next} {exit}' "$0" +} + +# --- small helpers ----------------------------------------------------------- + +# load_ignore_file - vendor/user split from the target snapshot's .gitignore. +# Uses a throwaway git dir so check-ignore works on deployment trees that are +# not git repos (and on Apple Git, which rejects --git-dir=/dev/null). +load_ignore_file() { + if [ -f "$TARGET_DIR/.gitignore" ]; then + IGNORE_FILE="$TARGET_DIR/.gitignore" + elif [ -f .gitignore ]; then + IGNORE_FILE="$(pwd)/.gitignore" + else + IGNORE_FILE="" + warn "No .gitignore found; only .git paths are excluded from the merge." + return 0 + fi + case "$IGNORE_FILE" in + /*) ;; + *) IGNORE_FILE="$(cd "$(dirname "$IGNORE_FILE")" && pwd)/$(basename "$IGNORE_FILE")" ;; + esac + IGNORE_GIT_DIR="$TMP_ROOT/ignore-git" + git init -q "$IGNORE_GIT_DIR" +} + +# is_excluded - true when the path is user-owned and must not merge. +is_excluded() { + case "$1" in + .git|.git/*) return 0 ;; + esac + [ -n "$IGNORE_FILE" ] || return 1 + git -C "$IGNORE_GIT_DIR" -c "core.excludesfile=$IGNORE_FILE" \ + check-ignore -q --no-index "$1" 2>/dev/null +} + +read_stamp_ref() { + [ -f "$STAMP_FILE" ] || return 1 + val=$(grep -E '^ref=' "$STAMP_FILE" 2>/dev/null | head -n1 | cut -d= -f2- | tr -d "\r\"' ") + if [ -z "$val" ]; then + val=$(grep -vE '^[[:space:]]*#' "$STAMP_FILE" 2>/dev/null \ + | grep -vE '^[[:space:]]*$' | head -n1 | tr -d "\r\"' ") + fi + [ -n "$val" ] && printf '%s' "$val" +} + +# env_has_key - true if the key appears as KEY=, even commented +# (e.g. "#GOOGLE_ENABLED="), so we never re-add a key the user disabled on purpose. +env_has_key() { + grep -qE "^[[:space:]]*#?[[:space:]]*$1=" "$2" 2>/dev/null +} + +record() { printf '%s:%s\n' "$1" "$2" >> "$REPORT"; } + +count_status() { grep -cE "^$1:" "$REPORT" 2>/dev/null || true; } + +list_status() { + grep -E "^$1:" "$REPORT" 2>/dev/null | cut -d: -f2- | sed 's/^/ /' +} + +# --- upstream snapshots ------------------------------------------------------ + +_sparse_init() { + git -C "$1" init -q + git -C "$1" remote add origin "$REPO_URL" + git -C "$1" config core.sparseCheckout true + git -C "$1" sparse-checkout init --cone >/dev/null 2>&1 + git -C "$1" sparse-checkout set docker >/dev/null 2>&1 +} + +# fetch_snapshot +# Materializes ./docker at into via a shallow fetch. Also the +# seam an artifact source (tarball + sha256) would slot into later. +fetch_snapshot() { + _ref="$1" + _dest="$2" + _work=$(mktemp -d "$TMP_ROOT/fetch.XXXXXX") + if _sparse_init "$_work" \ + && git -C "$_work" fetch --depth=1 --filter=blob:none -q origin "$_ref" 2>/dev/null \ + && git -C "$_work" checkout -q FETCH_HEAD 2>/dev/null \ + && [ -d "$_work/docker" ]; then + mkdir -p "$_dest" + cp -rf "$_work/docker/." "$_dest/" + rm -rf "$_work" + else + rm -rf "$_work" + return 1 + fi +} + +latest_release_tag() { + git ls-remote --tags --refs "$REPO_URL" 2>/dev/null \ + | sed 's#^.*refs/tags/##' \ + | grep -E '^self-hosted/v[0-9]' \ + | sort -V | tail -n1 +} + +list_files() { + ( cd "$1" && find . -type f | sed 's|^\./||' | grep -vE '^\.git(/|$)' | sort ) +} + +# normalize_version - reduce a ref to bare semver (0.7.0) for comparison, +# or empty when it is not a self-hosted release tag (commit SHA, "master", …). +normalize_version() { + _v="${1#refs/tags/}" + _v="${_v#self-hosted/}" + _v="${_v#v}" + case "$_v" in + [0-9]*.[0-9]*) printf '%s' "$_v" ;; + *) ;; + esac +} + +# ver_gt A B - true when version A is strictly greater than B (sort -V order). +ver_gt() { + if [ "$1" = "$2" ]; then return 1; fi + [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n1)" = "$1" ] +} + +# ver_in_window VER - true when BASE_VER < VER <= TARGET_VER. An empty bound +# leaves that side open (over-reports a gate rather than hiding one). +ver_in_window() { + if [ -n "$TARGET_VER" ] && ver_gt "$1" "$TARGET_VER"; then return 1; fi + if [ -n "$BASE_VER" ] && ! ver_gt "$1" "$BASE_VER"; then return 1; fi + return 0 +} + +# --- ref resolution ---------------------------------------------------------- + +resolve_target_ref() { + if [ -n "$TO_REF" ]; then + TARGET_REF="$TO_REF" + return + fi + TARGET_REF=$(latest_release_tag) + if [ -n "$TARGET_REF" ]; then + log "Latest release tag: $TARGET_REF" + else + TARGET_REF="master" + warn "No self-hosted/v* release tags found; targeting 'master'. Pin a version with --to." + fi +} + +resolve_base_ref() { + REPORT_ONLY=0 + if [ -n "$FROM_REF" ]; then + BASE_REF="$FROM_REF" + elif BASE_REF=$(read_stamp_ref) && [ -n "$BASE_REF" ]; then + : + else + REPORT_ONLY=1 + BASE_REF="" + fi +} + +print_report_only_guidance() { + warn "No $STAMP_FILE found and no --from given; cannot determine the version you started from." + cat >&2 < $STAMP_FILE + 3. Re-run: sh update.sh + +Or supply it inline for this run: sh update.sh --from + +Continuing in REPORT-ONLY mode. NOTE: this is NOT the full set of changes - +without a base version it can only list files and .env keys that are entirely +NEW to you; it CANNOT show which existing files would change or conflict. Record +a base and re-run for the real preview. Nothing will be written. +EOF +} + +fetch_snapshots() { + TARGET_DIR="$TMP_ROOT/target" + log "Fetching target snapshot ($TARGET_REF)" + fetch_snapshot "$TARGET_REF" "$TARGET_DIR" \ + || die "Could not fetch target snapshot '$TARGET_REF' from $REPO_URL" + + if [ "$REPORT_ONLY" = "1" ]; then + return + fi + + BASE_DIR="$TMP_ROOT/base" + log "Fetching base snapshot ($BASE_REF)" + fetch_snapshot "$BASE_REF" "$BASE_DIR" \ + || die "Could not fetch base snapshot '$BASE_REF' from $REPO_URL" +} + +run_report_only() { + log "Files in '$TARGET_REF' you do NOT have yet (brand-new only; existing files that changed are NOT shown here):" + while IFS= read -r f; do + is_excluded "$f" && continue + [ -f "$f" ] || echo " + $f" + done </dev/null 2>&1; then + die "$_manifest is present but is not a valid JSON object; refusing to update without a working breaking-change gate. Please report this to the maintainers." + fi + + if [ -z "$BASE_VER" ] || [ -z "$TARGET_VER" ]; then + warn "Base or target is not a self-hosted/vX.Y.Z tag; cannot compute an exact update window." + warn "Showing all applicable manual-action releases." + warn "Review which ones apply to your deployment." + fi + + for k in $(jq -r 'keys[]' "$_manifest" 2>/dev/null); do + case "$k" in [0-9]*.[0-9]*) ;; *) continue ;; esac + ver_in_window "$k" || continue + + breaking=$(jq -r --arg k "$k" '.[$k].breaking // false' "$_manifest") + gate=$(jq -r --arg k "$k" '.[$k].gate // empty' "$_manifest") + url=$(jq -r --arg k "$k" '.[$k].migration_guide_url // empty' "$_manifest") + reqs=$(jq -r --arg k "$k" '.[$k].requires[]? // empty' "$_manifest") + + if [ "$breaking" = "true" ] || [ -n "$gate" ]; then + GATE_REQUIRED=1 + fi + + { + [ "$breaking" = "true" ] && echo "[$k] BREAKING" || echo "[$k]" + [ -n "$gate" ] && echo " gate: '$gate' must be run first (see the steps below for the exact command)" + [ -n "$url" ] && echo " guide: $url" + [ -n "$reqs" ] && printf '%s\n' "$reqs" | sed 's/^/ - /' + } >> "$GATE_REPORT" + done + # Explicit success: the loop's last iteration can end on a false test (e.g. + # an entry with no 'requires'), which would otherwise make this function + # return non-zero and abort the whole script under 'set -e'. + return 0 +} + +confirm_gate() { + [ "$GATE_REQUIRED" = "1" ] || return 0 + [ "$DRY_RUN" != "1" ] || return 0 + [ "$ASSUME_YES" != "1" ] || return 0 + + echo "" >&2 + warn "This update requires manual action - review before continuing:" + sed 's/^/ /' "$GATE_REPORT" >&2 + echo "" >&2 + + if { : > /dev/tty; } 2>/dev/null; then + printf "Have you completed the required steps and want to continue? [y/N]: " > /dev/tty + read -r reply < /dev/tty + case "$reply" in + y|Y|yes|YES) return 0 ;; + *) die "Aborted by user. Nothing was modified." ;; + esac + fi + die "Breaking/gated changes present and no controlling terminal to confirm. Re-run with --yes to proceed." +} + +# --- backup ------------------------------------------------------------------ + +take_backup() { + if [ "$DRY_RUN" = "1" ]; then + log "Dry run: no backup taken, nothing will be written." + return 0 + fi + mkdir -p backups + _backup="backups/pre-update-$(date +%Y%m%d-%H%M%S).tgz" + log "Backing up current configuration to $_backup (excluding data directories)" + tar czf "$_backup" \ + --exclude='./backups' \ + --exclude='./volumes/db/data' \ + --exclude='./volumes/storage' \ + . 2>/dev/null || warn "Backup archive reported errors; review $_backup before relying on it." + warn "This does NOT back up your database. Back it up separately before updating." +} + +# --- vendor file merge ------------------------------------------------------- + +apply_file() { + [ "$DRY_RUN" = "1" ] && return 0 + _dir=$(dirname "$1") + [ "$_dir" = "." ] || mkdir -p "$_dir" + cp -f "$2" "$1" +} + +# Per-file 3-way merge (u=yours, b=base snapshot, t=target snapshot): +# no t → keep u; report removed-upstream +# no u → copy t; report new +# u == t → report unchanged +# u == b → copy t; report updated (user never edited) +# else → git merge-file u b t; report merged-clean or CONFLICT +# If b had no file, an empty file stands in for b. +merge_one_file() { + f="$1" + empty="$2" + merged="$TMP_ROOT/merged.out" + + b="$BASE_DIR/$f" + t="$TARGET_DIR/$f" + u="$f" + + if [ ! -f "$t" ]; then + [ -f "$u" ] && record "removed-upstream" "$f" + return 0 + fi + + if [ ! -f "$u" ]; then + apply_file "$f" "$t" + record "new" "$f" + return 0 + fi + + if cmp -s "$u" "$t"; then + record "unchanged" "$f" + return 0 + fi + + base_for_merge="$b" + [ -f "$base_for_merge" ] || base_for_merge="$empty" + + if cmp -s "$u" "$base_for_merge"; then + apply_file "$f" "$t" + record "updated" "$f" + return 0 + fi + + if git merge-file -p -q \ + -L "yours ($f)" -L "base" -L "new ($TARGET_REF)" \ + "$u" "$base_for_merge" "$t" > "$merged" 2>/dev/null; then + [ "$DRY_RUN" != "1" ] && cp -f "$merged" "$u" + record "merged-clean" "$f" + elif [ -s "$merged" ]; then + # Non-zero exit with output = a normal conflict (markers written). + [ "$DRY_RUN" != "1" ] && cp -f "$merged" "$u" + record "CONFLICT" "$f" + else + # git merge-file errored and produced no output; keep the user's file + # intact rather than truncating it to empty. + record "merge-failed" "$f" + fi +} + +# The running script is a vendor file too, but overwriting it in place would +# corrupt this process (the shell reads $0 as it runs). Never write it directly: +# if the target ships a different version, stage it as .dist to review. +stage_self_update() { + _t="$TARGET_DIR/$SELF_NAME" + if [ ! -f "$_t" ] || cmp -s "$SELF_NAME" "$_t"; then + record "unchanged" "$SELF_NAME" + return 0 + fi + [ "$DRY_RUN" = "1" ] || cp -f "$_t" "$SELF_NAME.dist" + record "self-staged" "$SELF_NAME" +} + +merge_vendor_files() { + _empty="$TMP_ROOT/empty" + : > "$_empty" + : > "$REPORT" + + while IFS= read -r f; do + [ -n "$f" ] || continue + is_excluded "$f" && continue + if [ "$f" = "$SELF_NAME" ]; then + stage_self_update + continue + fi + merge_one_file "$f" "$_empty" + done < "$ENV_ADDED" + : > "$ENV_REMOVED" + [ -f "$_example" ] || return 0 + + while IFS= read -r k; do + env_has_key "$k" .env || echo "$k" >> "$ENV_ADDED" + done <>>>>>> markers):" + list_status CONFLICT + fi + if [ "$(count_status merge-failed)" != "0" ]; then + echo "" + warn "Files git could not merge (left unchanged - update these manually):" + list_status merge-failed + fi + if [ "$(count_status merged-clean)" != "0" ]; then + echo "" + log "Files merged cleanly (review recommended):" + list_status merged-clean + fi + if [ "$(count_status removed-upstream)" != "0" ]; then + echo "" + log "Removed upstream but kept in place (you may no longer need these):" + list_status removed-upstream + fi + if [ "$(count_status self-staged)" != "0" ]; then + echo "" + if [ "$DRY_RUN" = "1" ]; then + log "$SELF_NAME differs from the version in '$TARGET_REF'; a real run would stage that version as $SELF_NAME.dist (the running script is never overwritten in place)." + else + log "$SELF_NAME differs from the version in '$TARGET_REF', staged as $SELF_NAME.dist (the running script was not modified)." + log "Review it, then swap it in if needed: mv $SELF_NAME.dist $SELF_NAME" + fi + fi + if [ -s "$ENV_ADDED" ]; then + echo "" + log ".env keys added (review values):" + sed 's/^/ + /' "$ENV_ADDED" + fi + if [ -s "$ENV_REMOVED" ]; then + echo "" + log ".env keys you have that are gone from the new .env.example (review/remove manually):" + sed 's/^/ - /' "$ENV_REMOVED" + fi + if [ -s "$GATE_REPORT" ]; then + echo "" + log "Required manual steps for this update (from upgrades.json):" + sed 's/^/ /' "$GATE_REPORT" + fi + echo "" + log "For what changed in this update, see CHANGELOG.md (from $BASE_REF to $TARGET_REF)." +} + +write_stamp() { + { + echo "# Supabase self-hosted version stamp. Managed by setup.sh / update.sh." + echo "# Do not commit or edit by hand. Records the ref this deployment was based on." + echo "ref=$TARGET_REF" + } > "$STAMP_FILE" +} + +print_next_steps() { + echo "" + log "Next steps:" + echo " 1. Review the changes above (compare against the latest backup in backups/ if needed)." + echo " 2. sh run.sh pull" + echo " 3. sh run.sh recreate" +} + +# --- main -------------------------------------------------------------------- + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --yes|-y) ASSUME_YES=1; shift ;; + --to) [ $# -ge 2 ] || die "--to requires a argument."; TO_REF="$2"; shift 2 ;; + --from) [ $# -ge 2 ] || die "--from requires a argument."; FROM_REF="$2"; shift 2 ;; + -h|--help) print_help; exit 0 ;; + *) echo "Unknown option: $1" >&2; print_help; exit 1 ;; + esac +done + +[ -f docker-compose.yml ] || die "docker-compose.yml not found in $(pwd). Run this from your deployment directory." +[ -f .env ] || die ".env not found in $(pwd). This does not look like a configured deployment." +command -v git >/dev/null 2>&1 || die "git is required but was not found on PATH." +command -v jq >/dev/null 2>&1 || die "jq is required but was not found on PATH." + +resolve_target_ref +resolve_base_ref +TARGET_VER=$(normalize_version "$TARGET_REF") +BASE_VER=$(normalize_version "$BASE_REF") +[ "$REPORT_ONLY" = "1" ] && print_report_only_guidance + +TMP_ROOT=$(mktemp -d) +REPORT="$TMP_ROOT/report" +ENV_ADDED="$TMP_ROOT/env_added" +ENV_REMOVED="$TMP_ROOT/env_removed" +GATE_REPORT="$TMP_ROOT/gate_report" +trap 'rm -rf "$TMP_ROOT"' EXIT INT TERM + +fetch_snapshots +load_ignore_file + +if [ "$REPORT_ONLY" = "1" ]; then + run_report_only + build_gate_report + if [ -s "$GATE_REPORT" ]; then + echo "" + log "Breaking changes / required manual steps in this range (from upgrades.json):" + sed 's/^/ /' "$GATE_REPORT" + warn "Record a base version (see above) and re-run to apply with the gate enforced." + fi + exit 0 +fi + +build_gate_report +confirm_gate + +take_backup +merge_vendor_files +merge_env_file +print_summary + +if [ "$DRY_RUN" = "1" ]; then + echo "" + log "Dry run complete. Re-run without --dry-run to apply." + exit 0 +fi + +if [ "$(count_status CONFLICT)" != "0" ] || [ "$(count_status merge-failed)" != "0" ]; then + echo "" + warn "Update applied WITH CONFLICTS. Resolve the files listed above (remove the" + warn "<<<<<<< ======= >>>>>>> markers, or fix the files git could not merge)" + warn "before starting the stack." + warn "The version stamp was NOT advanced; it will update on your next clean run." + warn "Exiting with status 2." + exit 2 +fi + +write_stamp +print_next_steps +log "Update applied cleanly." diff --git a/supabase/upgrades.json b/supabase/upgrades.json new file mode 100644 index 0000000000..21704448b0 --- /dev/null +++ b/supabase/upgrades.json @@ -0,0 +1,39 @@ +{ + "_schema": { + "description": "Manual-action manifest for self-hosted upgrades, read by update.sh with jq. Hand-edit this file directly; it is the source of truth (no generation step). Keys are self-hosted release versions as bare semver (e.g. \"0.7.0\"), matching the self-hosted/vX.Y.Z tag and the \"## [0.7.0]\" CHANGELOG heading. Only add an entry for a release that needs an action a file diff cannot encode: a data migration, a run-this-first script, or a breaking default. Routine config changes are applied automatically by update.sh's 3-way merge and must NOT be listed here. update.sh gates on entries in (your version, target version], ordered with sort -V.", + "fields": { + "breaking": "bool - requires explicit user confirmation before applying", + "gate": "string|null - script to run before upgrading past this release (e.g. utils/upgrade-pg17.sh)", + "migration_guide_url": "string|null - link to the per-release migration guide", + "requires": "string[] - free-text manual steps shown to the user" + } + }, + "0.6.0": { + "breaking": true, + "gate": "utils/upgrade-pg17.sh", + "migration_guide_url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "requires": [ + "Postgres 17 is now the default. Do NOT start Postgres 17 against an existing Postgres 15 data directory - back up your database first.", + "Run 'sudo bash utils/upgrade-pg17.sh' (needs bash + root) to migrate Postgres 15 -> 17, then recreate containers. To defer, pin Postgres 15 with the docker-compose.pg15.yml override.", + "Includes a security fix for the API gateway (Realtime /api/tenants and /api/openapi routes) - strongly recommended for any instance running Realtime." + ] + }, + "0.7.0": { + "breaking": true, + "gate": null, + "migration_guide_url": "https://github.com/orgs/supabase/discussions/47093", + "requires": [ + "API_EXTERNAL_URL now includes the /auth/v1 path prefix, and SAML SSO endpoints moved to /auth/v1/sso/saml/*. Update custom OAuth provider callback URLs and any SAML configuration accordingly.", + "Anon (publishable) key access to the OpenAPI spec at /rest/v1/ has been removed. Use the service role or secret API key if you relied on it; normal data access via /rest/v1/ is unaffected." + ] + }, + "0.8.0": { + "breaking": true, + "gate": null, + "migration_guide_url": "https://github.com/orgs/supabase/discussions/48048", + "requires": [ + "Envoy is now the default API gateway, replacing Kong. The gateway service is renamed from 'kong' to 'api-gw' (container 'supabase-envoy'); the 'kong' network alias still resolves, so internal service references keep working.", + "If you customized volumes/api/kong.yml or the gateway service, enable the Kong override with 'sh run.sh config add kong' to keep running Kong; otherwise the merge switches you to Envoy." + ] + } +} diff --git a/supabase/utils/add-new-auth-keys.sh b/supabase/utils/add-new-auth-keys.sh new file mode 100644 index 0000000000..f921b6e876 --- /dev/null +++ b/supabase/utils/add-new-auth-keys.sh @@ -0,0 +1,226 @@ +#!/bin/sh +# +# Add asymmetric key pair and opaque API keys to a self-hosted Supabase installation. +# +# Reads JWT_SECRET from .env and generates: +# - EC P-256 key pair (JWT_KEYS, JWT_JWKS) +# - Opaque API keys (SUPABASE_PUBLISHABLE_KEY, SUPABASE_SECRET_KEY) +# - Internal: ES256 JWT API keys (ANON_KEY_ASYMMETRIC, SERVICE_ROLE_KEY_ASYMMETRIC) +# +# Usage: +# sh add-new-auth-keys.sh # Interactive: prints keys, prompts to update .env +# sh add-new-auth-keys.sh --update-env # Prints keys and writes them to .env +# sh add-new-auth-keys.sh | tee keys # Non-interactive: prints keys only +# +# Prerequisites: +# - .env file with JWT_SECRET set (run generate-keys.sh first) +# - node (>= 16) or docker +# + +set -e + +node_ok() { + command -v node >/dev/null 2>&1 || return 1 + major=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) + [ -n "$major" ] && [ "$major" -ge 16 ] 2>/dev/null +} + +# Resolve how to run node: local install (>= 16) preferred, docker fallback. +if node_ok; then + node_runner="node" +else + if command -v node >/dev/null 2>&1; then + echo "Local node $(node -v) is too old (need >= 16), falling back to docker." + fi + + if ! command -v docker >/dev/null 2>&1; then + echo "Error: requires either node (>= 16) or docker." + exit 1 + fi + + if ! ${DOCKER:-docker} info >/dev/null; then + echo "Error: cannot connect to the Docker daemon." + exit 1 + fi + + if ! ${DOCKER:-docker} image inspect node:22-alpine >/dev/null 2>&1; then + echo "Pulling node:22-alpine (first-run only)..." + ${DOCKER:-docker} pull node:22-alpine + fi + + node_runner="${DOCKER:-docker} run --rm node:22-alpine node" +fi + +# Read JWT_SECRET from .env +if [ ! -f .env ]; then + echo "Error: .env file not found. Run generate-keys.sh first." + exit 1 +fi + +jwt_secret=$(grep '^JWT_SECRET=' .env | cut -d= -f2- | tr -d '\r') +if [ -z "$jwt_secret" ]; then + echo "Error: JWT_SECRET not found in .env. Run generate-keys.sh first." + exit 1 +fi + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT + +# Node.js does the crypto-heavy work: +# - EC P-256 keypair generation +# - JWKS construction (with symmetric key included) +# - ES256 JWT signing +# - Opaque API key generation with checksum +$node_runner -e ' +const crypto = require("crypto"); + +const jwtSecret = process.argv[1]; + +// Generate EC P-256 keypair and export as JWK +const { privateKey } = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); +const jwkPrivate = privateKey.export({ format: "jwk" }); + +const kid = crypto.randomUUID(); + +// Symmetric key as JWK (base64url-encoded) +const octKey = { + kty: "oct", + k: Buffer.from(jwtSecret).toString("base64url"), + alg: "HS256" +}; + +// JWKS with private key (for Auth to sign tokens) +const jwksKeypair = { keys: [ + { kty: "EC", kid, use: "sig", key_ops: ["sign", "verify"], alg: "ES256", ext: true, + crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y, d: jwkPrivate.d }, + octKey +]}; + +// JWKS with public key only (for PostgREST, Realtime, Storage to verify) +const jwksPublic = { keys: [ + { kty: "EC", kid, use: "sig", key_ops: ["verify"], alg: "ES256", ext: true, + crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y }, + octKey +]}; + +// Sign ES256 JWT +function signES256(payload) { + const header = { alg: "ES256", typ: "JWT", kid }; + const b64Header = Buffer.from(JSON.stringify(header)).toString("base64url"); + const b64Payload = Buffer.from(JSON.stringify(payload)).toString("base64url"); + const data = b64Header + "." + b64Payload; + const sig = crypto.sign("SHA256", Buffer.from(data), { + key: privateKey, + dsaEncoding: "ieee-p1363" + }).toString("base64url"); + return data + "." + sig; +} + +const iat = Math.floor(Date.now() / 1000); +const exp = iat + 5 * 365 * 24 * 3600; // 5 years + +const anonJwt = signES256({ role: "anon", iss: "supabase", iat, exp }); +const serviceJwt = signES256({ role: "service_role", iss: "supabase", iat, exp }); + +// Generate opaque API keys with checksum +const PROJECT_REF = "supabase-self-hosted"; + +function generateOpaqueKey(prefix) { + const random = crypto.randomBytes(17).toString("base64url").slice(0, 22); + const intermediate = prefix + random; + const checksum = crypto.createHash("sha256") + .update(PROJECT_REF + "|" + intermediate) + .digest("base64url") + .slice(0, 8); + return intermediate + "_" + checksum; +} + +const publishableKey = generateOpaqueKey("sb_publishable_"); +const secretKey = generateOpaqueKey("sb_secret_"); + +// Output as KEY=value lines for shell to parse +console.log("SUPABASE_PUBLISHABLE_KEY=" + publishableKey); +console.log("SUPABASE_SECRET_KEY=" + secretKey); +console.log("ANON_KEY_ASYMMETRIC=" + anonJwt); +console.log("SERVICE_ROLE_KEY_ASYMMETRIC=" + serviceJwt); +console.log("JWT_KEYS=" + JSON.stringify(jwksKeypair.keys)); +console.log("JWT_JWKS=" + JSON.stringify(jwksPublic)); +' "$jwt_secret" > "$tmpdir/output" + +# Read generated values +SUPABASE_PUBLISHABLE_KEY=$(grep '^SUPABASE_PUBLISHABLE_KEY=' "$tmpdir/output" | cut -d= -f2-) +SUPABASE_SECRET_KEY=$(grep '^SUPABASE_SECRET_KEY=' "$tmpdir/output" | cut -d= -f2-) +ANON_KEY_ASYMMETRIC=$(grep '^ANON_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) +SERVICE_ROLE_KEY_ASYMMETRIC=$(grep '^SERVICE_ROLE_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) +JWT_KEYS=$(grep '^JWT_KEYS=' "$tmpdir/output" | cut -d= -f2-) +JWT_JWKS=$(grep '^JWT_JWKS=' "$tmpdir/output" | cut -d= -f2-) + +echo "" +echo "SUPABASE_PUBLISHABLE_KEY=${SUPABASE_PUBLISHABLE_KEY}" +echo "SUPABASE_SECRET_KEY=${SUPABASE_SECRET_KEY}" +echo "" +echo "JWT_KEYS=${JWT_KEYS}" +echo "" +echo "JWT_JWKS=${JWT_JWKS}" +echo "" +echo "Ensure the following configuration is uncommented in docker-compose.yml for the asymmetric key pair to work:" +echo "" +echo " Auth: GOTRUE_JWT_KEYS: \${JWT_KEYS:-[]}" +echo " Realtime: API_JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" +echo " Storage: JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" +echo " Functions: SUPABASE_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" +echo "" + +if [ "$1" = "--update-env" ]; then + update_env=true +elif test -t 0; then + printf "Update .env file? (y/N) " + read -r REPLY + case "$REPLY" in + [Yy]) update_env=true ;; + *) update_env=false ;; + esac +else + echo "Running non-interactively. Pass --update-env to write to .env." + update_env=false +fi + +if [ "$update_env" != "true" ]; then + exit 0 +fi + +echo "Updating .env..." + +# Append new variables if they don't exist, or update them if they do +for var in SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY ANON_KEY_ASYMMETRIC SERVICE_ROLE_KEY_ASYMMETRIC JWT_KEYS JWT_JWKS; do + eval "val=\$$var" + if grep -q "^${var}=" .env; then + sed -i.old -e "s|^${var}=.*$|${var}=${val}|" .env + else + echo "${var}=${val}" >> .env + fi +done + +# Uncomment new auth configuration in docker-compose.yml +echo "Updating docker-compose.yml..." +if [ ! -f docker-compose.yml ]; then + echo "Error: docker-compose.yml not found in $(pwd)" + exit 1 +fi + +# Always fall through to the grep check +sed -i.old \ + -e '/^[ ]*#GOTRUE_JWT_KEYS:/ s/#//' \ + -e '/^[ ]*#API_JWT_JWKS:/ s/#//' \ + -e '/^[ ]*#JWT_JWKS:/ s/#//' \ + -e '/^[ ]*#SUPABASE_JWKS:/ s/#//' \ + docker-compose.yml || true + +if grep -q '^[ ]*GOTRUE_JWT_KEYS:' docker-compose.yml && \ + grep -q '^[ ]*API_JWT_JWKS:' docker-compose.yml && \ + grep -q '^[ ]*JWT_JWKS:' docker-compose.yml && \ + grep -q '^[ ]*SUPABASE_JWKS:' docker-compose.yml; then + echo "Done." +else + echo "Warning: could not edit docker-compose.yml. Uncomment auth configuration manually." +fi diff --git a/supabase/utils/generate-keys.sh b/supabase/utils/generate-keys.sh new file mode 100644 index 0000000000..da1fc19218 --- /dev/null +++ b/supabase/utils/generate-keys.sh @@ -0,0 +1,138 @@ +#!/bin/sh +# +# Generate secrets and legacy symmetric JWT API keys for self-hosted Supabase. +# +# Generates: JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, and other secrets +# needed for a fresh installation. +# +# Usage: +# sh generate-keys.sh # Interactive: prints keys, prompts to update .env +# sh generate-keys.sh --update-env # Prints keys and writes them to .env +# sh generate-keys.sh | tee keys # Non-interactive: prints keys only +# +# Portions of this code are derived from Inder Singh's setup.sh shell script. +# Copyright 2025 Inder Singh. Licensed under Apache License 2.0. +# Original source: https://github.com/singh-inder/supabase-automated-self-host/blob/main/setup.sh +# + +set -e + +gen_hex() { + openssl rand -hex "$1" +} + +gen_base64() { + openssl rand -base64 "$1" +} + +base64_url_encode() { + openssl enc -base64 -A | tr '+/' '-_' | tr -d '=' +} + +gen_token() { + payload=$1 + payload_base64=$(printf %s "$payload" | base64_url_encode) + header_base64=$(printf %s "$header" | base64_url_encode) + signed_content="${header_base64}.${payload_base64}" + signature=$(printf %s "$signed_content" | openssl dgst -binary -sha256 -hmac "$jwt_secret" | base64_url_encode) + printf '%s' "${signed_content}.${signature}" +} + +if ! command -v openssl >/dev/null 2>&1; then + echo "Error: openssl is required but not found." + exit 1 +fi + +jwt_secret="$(gen_base64 30)" + +# Used in gen_token() +header='{"alg":"HS256","typ":"JWT"}' +iat=$(date +%s) +exp=$((iat + 5 * 3600 * 24 * 365)) # 5 years + +# Normalizes JSON formatting so that the token matches https://www.jwt.io/ results +anon_payload="{\"role\":\"anon\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" +service_role_payload="{\"role\":\"service_role\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" + +#echo "anon_payload=$anon_payload" +#echo "service_role_payload=$service_role_payload" + +anon_key=$(gen_token "$anon_payload") +service_role_key=$(gen_token "$service_role_payload") + +secret_key_base=$(gen_base64 48) +realtime_db_enc_key=$(gen_hex 8) +vault_enc_key=$(gen_hex 16) +pg_meta_crypto_key=$(gen_base64 24) + +logflare_public_access_token=$(gen_base64 24) +logflare_private_access_token=$(gen_base64 24) + +s3_protocol_access_key_id=$(gen_hex 16) +s3_protocol_access_key_secret=$(gen_hex 32) + +minio_root_password=$(gen_hex 16) + +echo "" +echo "JWT_SECRET=${jwt_secret}" +echo "" +#echo "Issued at: $iat" +#echo "Expire: $exp" +echo "ANON_KEY=${anon_key}" +echo "SERVICE_ROLE_KEY=${service_role_key}" +echo "" +echo "SECRET_KEY_BASE=${secret_key_base}" +echo "REALTIME_DB_ENC_KEY=${realtime_db_enc_key}" +echo "VAULT_ENC_KEY=${vault_enc_key}" +echo "PG_META_CRYPTO_KEY=${pg_meta_crypto_key}" +echo "LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}" +echo "LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}" +echo "S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}" +echo "S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}" +echo "MINIO_ROOT_PASSWORD=${minio_root_password}" +echo "" + +postgres_password=$(gen_hex 16) +dashboard_password=$(gen_hex 16) + +echo "POSTGRES_PASSWORD=${postgres_password}" +echo "DASHBOARD_PASSWORD=${dashboard_password}" +echo "" + +if [ "$1" = "--update-env" ]; then + update_env=true +elif test -t 0; then + printf "Update .env file? (y/N) " + read -r REPLY + case "$REPLY" in + [Yy]) update_env=true ;; + *) update_env=false ;; + esac +else + echo "Running non-interactively. Pass --update-env to write to .env." + update_env=false +fi + +if [ "$update_env" != "true" ]; then + exit 0 +fi + +echo "Updating .env..." + +sed \ + -i.old \ + -e "s|^JWT_SECRET=.*$|JWT_SECRET=${jwt_secret}|" \ + -e "s|^ANON_KEY=.*$|ANON_KEY=${anon_key}|" \ + -e "s|^SERVICE_ROLE_KEY=.*$|SERVICE_ROLE_KEY=${service_role_key}|" \ + -e "s|^SECRET_KEY_BASE=.*$|SECRET_KEY_BASE=${secret_key_base}|" \ + -e "s|^REALTIME_DB_ENC_KEY=.*$|REALTIME_DB_ENC_KEY=${realtime_db_enc_key}|" \ + -e "s|^VAULT_ENC_KEY=.*$|VAULT_ENC_KEY=${vault_enc_key}|" \ + -e "s|^PG_META_CRYPTO_KEY=.*$|PG_META_CRYPTO_KEY=${pg_meta_crypto_key}|" \ + -e "s|^LOGFLARE_PUBLIC_ACCESS_TOKEN=.*$|LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}|" \ + -e "s|^LOGFLARE_PRIVATE_ACCESS_TOKEN=.*$|LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}|" \ + -e "s|^S3_PROTOCOL_ACCESS_KEY_ID=.*$|S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}|" \ + -e "s|^S3_PROTOCOL_ACCESS_KEY_SECRET=.*$|S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}|" \ + -e "s|^MINIO_ROOT_PASSWORD=.*$|MINIO_ROOT_PASSWORD=${minio_root_password}|" \ + -e "s|^POSTGRES_PASSWORD=.*$|POSTGRES_PASSWORD=${postgres_password}|" \ + -e "s|^DASHBOARD_PASSWORD=.*$|DASHBOARD_PASSWORD=${dashboard_password}|" \ + .env diff --git a/supabase/utils/upgrade-pg17.sh b/supabase/utils/upgrade-pg17.sh new file mode 100644 index 0000000000..c05a7b2571 --- /dev/null +++ b/supabase/utils/upgrade-pg17.sh @@ -0,0 +1,822 @@ +#!/usr/bin/env bash +# +# Requires bash (not sh) for pipefail, which ensures failures in piped +# commands are caught during the upgrade. +# +# Upgrade self-hosted Supabase Postgres from 15 to 17. +# +# Uses Supabase's pg_upgrade scripts (initiate.sh + complete.sh) inside a +# temporary PG 15 container, then swaps data directories and starts Postgres 17. +# +# Usage (must be run as root or with sudo): +# cd docker/ +# sudo bash utils/upgrade-pg17.sh # Interactive (prompts for confirmation) +# sudo bash utils/upgrade-pg17.sh --yes # Non-interactive (skip all prompts) +# +# Requirements: +# - Docker with Docker Compose (docker compose, not docker-compose) +# - Running Supabase self-hosted setup with Postgres 15 +# - At least 2x current database size + 5 GB free disk space +# +# Backup: +# The original Postgres 15 data directory is preserved as +# ./volumes/db/data.bak.pg15 during the upgrade. +# DO NOT DELETE it until you have verified the upgrade was successful. +# +# Rollback (if the upgrade fails or you want to revert): +# 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down +# 2. rm -rf ./volumes/db/data +# 3. mv ./volumes/db/data.bak.pg15 ./volumes/db/data +# 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/ +# 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d +# + +# Ensure we're running under bash (not sh/zsh/dash). +# Check that $BASH ends with /bash (not /sh, /zsh, etc.). +case "${BASH:-}" in + */bash) ;; + *) echo "Error: This script requires bash. Run it with: sudo bash $0" >&2; exit 1 ;; +esac + +set -euo pipefail + +AUTO_CONFIRM=false +for arg in "$@"; do + case "$arg" in + --yes|-y) AUTO_CONFIRM=true ;; + esac +done + +# --- Configuration ---------------------------------------------------------- + +# Image used for the upgrade tarball + complete.sh container. +# Must share glibc with PG 15 (the extracted ELF binaries run inside PG15). +# Pinned to .063: later images bumped glibc, which breaks the ELF extraction. +PG17_UPGRADE_IMAGE="supabase/postgres:17.6.1.063" +# Tag in supabase/postgres repo matching the upgrade image (for downloading scripts) +PG17_SCRIPTS_REF="17.6.1.063" + +# Final Postgres 17 image that runs after the upgrade. Pinned here (not read from +# the compose files). Keep in sync with the db image in docker-compose.yml (and +# docker-compose.pg17.yml). Used for pulling, chowning data/db-config to the +# target's postgres UID, and running the post-upgrade migrations. +PG17_TARGET_IMAGE="supabase/postgres:17.6.1.136" + +DB_CONTAINER="supabase-db" +UPGRADE_CONTAINER="supabase-pg-upgrade" +COMPLETE_CONTAINER="supabase-pg-complete" + +DATA_DIR="./volumes/db/data" +BACKUP_DIR="./volumes/db/data.bak.pg15" +# Include image tag in cache filename so changing PG17_UPGRADE_IMAGE invalidates it +PG17_TAG="${PG17_UPGRADE_IMAGE##*:}" +TARBALL_CACHE="./volumes/db/pg17_upgrade_bin_${PG17_TAG}.tar.gz" +# initiate.sh writes pg_upgrade output here: pgdata/, conf/, sql/ +MIGRATION_DIR="./volumes/db/data_migration" + +# --- Helpers ---------------------------------------------------------------- + +die() { printf 'Error: %s\n' "$*" >&2; exit 1; } +info() { printf '\n==> %s\n' "$*"; } +warn() { printf 'Warning: %s\n' "$*" >&2; } + +# Temp dir on host for tarball + scripts (mounted into containers) +staging_dir="" +pg_password="" +current_image="" +drop_extensions="" +db_config_vol="" + +# Remove leftover containers and staging dir on exit. +# Uses an alpine container for rm because the tarball build runs as root +# inside Docker - the resulting files are root-owned and can't be deleted +# by the host user on macOS. +cleanup() { + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true + if [ -n "$staging_dir" ] && [ -d "$staging_dir" ]; then + docker run --rm -v "$staging_dir:/cleanup" alpine rm -rf /cleanup 2>/dev/null || true + rm -rf "$staging_dir" 2>/dev/null || true + fi +} +trap cleanup EXIT + +on_interrupt() { + echo "" + warn "Interrupted. Cleaning up..." + # If db-config was chowned to PG17, restore for PG 15 rollback + if [ -n "$db_config_vol" ] && [ -n "$current_image" ]; then + docker run --rm -v "${db_config_vol}:/vol" "$current_image" \ + chown -R postgres:postgres /vol/ 2>/dev/null || true + fi + die "Interrupted." +} +trap on_interrupt INT + +confirm() { + if [ "$AUTO_CONFIRM" = true ]; then return 0; fi + if ! test -t 0; then + die "This script must be run interactively, or use --yes to skip prompts." + fi + printf '%s (y/N) ' "$1" + read -r reply + case "$reply" in + [Yy]*) return 0 ;; + *) echo "Aborted."; exit 0 ;; + esac +} + +run_sql_on() { + local container=$1; shift + docker exec -i \ + -e PGPASSWORD="$pg_password" \ + "$container" \ + psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 "$@" +} + +wait_for_healthy() { + local container=$1 retries=30 + while [ $retries -gt 0 ]; do + if docker exec "$container" pg_isready -U postgres -h localhost >/dev/null 2>&1; then + return 0 + fi + retries=$((retries - 1)) + sleep 1 + done + die "Postgres in '$container' did not become ready in 30 seconds." +} + +# --- Pre-flight checks ----------------------------------------------------- + +preflight() { + info "Running pre-flight checks" + + if [ "$(id -u)" -ne 0 ]; then + die "This script must be run as root (e.g. sudo bash $0)." + fi + + docker compose version >/dev/null 2>&1 || die "Docker Compose not found." + command -v curl >/dev/null 2>&1 || die "curl is required (for downloading upgrade scripts)." + [ -f docker-compose.yml ] || die "Run this script from the docker/ directory." + [ -f docker-compose.pg17.yml ] || die "Missing docker-compose.pg17.yml." + [ -f .env ] || die "Missing .env file." + + # Resolve db-config volume (exact match on _db-config suffix or bare db-config) + db_config_vol=$(docker volume ls --filter "name=db-config" --format '{{.Name}}' \ + | grep -E '^db-config$|_db-config$' | head -n 1) + [ -n "$db_config_vol" ] || die "Could not find db-config volume. Is Supabase running?" + + pg_password=$(grep '^POSTGRES_PASSWORD=' .env | cut -d '=' -f 2- | sed "s/^['\"]//;s/['\"]$//" | head -n 1) + [ -n "$pg_password" ] || die "POSTGRES_PASSWORD not set in .env." + + docker inspect "$DB_CONTAINER" >/dev/null 2>&1 \ + || die "Container '$DB_CONTAINER' not found. Is Supabase running?" + + current_image=$(docker inspect "$DB_CONTAINER" --format '{{.Config.Image}}') + case "$current_image" in + supabase/postgres:15.*|supabase.postgres:15.*) ;; + supabase/postgres:17.*|supabase.postgres:17.*) die "Already running Postgres 17 ($current_image)." ;; + *) die "Unexpected database image: $current_image" ;; + esac + + local status + status=$(docker inspect "$DB_CONTAINER" --format '{{.State.Status}}') + [ "$status" = "running" ] || die "'$DB_CONTAINER' is not running (status: $status)." + [ -d "$DATA_DIR" ] || die "Data directory not found: $DATA_DIR" + + if [ -d "$BACKUP_DIR" ]; then + warn "Backup directory already exists: $BACKUP_DIR" + warn "This is likely from a previous upgrade attempt." + warn "If you haven't verified that previous upgrade, roll back first:" + warn " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" + warn " 2. rm -rf $DATA_DIR" + warn " 3. mv $BACKUP_DIR $DATA_DIR" + warn " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + warn " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" + echo "" + warn "Continuing will DELETE the existing backup permanently." + confirm "Delete $BACKUP_DIR and start a fresh upgrade?" + rm -rf "$BACKUP_DIR" + fi + if [ -d "$MIGRATION_DIR" ]; then + rm -rf "$MIGRATION_DIR" + fi + + # Disk space + local data_size_kb data_size_mb avail_kb avail_mb needed_mb + data_size_kb=$(du -sk "$DATA_DIR" 2>/dev/null | cut -f1) + [ -n "$data_size_kb" ] || die "Could not calculate data size for $DATA_DIR" + data_size_mb=$((data_size_kb / 1024)) + avail_kb=$(df -k "$(dirname "$DATA_DIR")" | awk 'NR==2 { print $4 }') + [ -n "$avail_kb" ] || die "Could not calculate available disk space for $(dirname "$DATA_DIR")" + avail_mb=$((avail_kb / 1024)) + needed_mb=$((data_size_mb * 2 + 5000)) + echo " Data size: ${data_size_mb} MB" + echo " Available space: ${avail_mb} MB" + echo " Estimated need: ${needed_mb} MB" + if [ "$avail_mb" -lt "$needed_mb" ]; then + warn "Disk space may be insufficient." + warn "pg_upgrade copies data; need ~2x data size + ~5 GB for the upgrade tarball." + confirm "Continue anyway?" + fi + + # Incompatible extensions + info "Checking for incompatible extensions" + local incompatible + incompatible=$(run_sql_on "$DB_CONTAINER" -A -t -c " + SELECT string_agg(extname, ', ') + FROM pg_extension + WHERE extname IN ('timescaledb', 'plv8', 'plcoffee', 'plls'); + " 2>/dev/null | tr -d '[:space:]') || true + + if [ -n "$incompatible" ]; then + warn "Incompatible extensions found: $incompatible" + warn "These do not exist in Postgres 17 and must be dropped before upgrading." + warn "If you proceed, they will be dropped automatically." + warn "The original data is preserved as a backup so you can roll back." + confirm "Drop these extensions and continue with the upgrade?" + drop_extensions="$incompatible" + fi + + echo "" + echo "This script will:" + echo " 1. Pull the Postgres 17 image" + echo " 2. Build an upgrade tarball from the image (~1.2 GB compressed, temporary)" + echo " 3. Stop all Supabase services" + echo " 4. Run pg_upgrade (Postgres 15 -> 17)" + echo " 5. Apply post-upgrade patches" + echo " 6. Start Supabase with Postgres 17" + echo " 7. Apply additional migrations" + echo "" + echo " Current image: $current_image" + echo " Target image: $PG17_TARGET_IMAGE" + echo " Upgrade image: $PG17_UPGRADE_IMAGE" + echo " Data directory: $DATA_DIR" + echo " Backup location: $BACKUP_DIR" + echo "" + confirm "Proceed with the upgrade?" +} + +# --- Step 1: Pull Postgres 17 image ---------------------------------------- + +pull_image() { + info "Pulling Postgres 17 images" + docker pull "$PG17_UPGRADE_IMAGE" + if [ "$PG17_TARGET_IMAGE" != "$PG17_UPGRADE_IMAGE" ]; then + docker pull "$PG17_TARGET_IMAGE" + fi +} + +# --- Step 2: Build upgrade tarball ----------------------------------------- +# +# Extracts PG 17 binaries, libraries, share data, and upgrade scripts from +# the PG 17 Docker image into a tarball that initiate.sh can consume. +# +# The tarball uses the "non-nix" layout (17/bin, 17/lib, 17/share - no +# nix_flake_version file), so initiate.sh sets LD_LIBRARY_PATH to find +# the bundled libraries. + +build_tarball() { + local tmpbase="${TMPDIR:-/tmp}" + staging_dir=$(mktemp -d "${tmpbase%/}/supabase-pg17-upgrade.XXXXXX") + # World-writable so Docker containers can write to bind mounts on macOS, + # where the VM's root user has no special access to host directories. + chmod 777 "$staging_dir" + echo " Staging directory: $staging_dir" + + # Download upgrade scripts from the supabase/postgres repo (pinned to PG17_SCRIPTS_REF). + # These are no longer bundled in the latest PG 17 Docker images. + info "Downloading upgrade scripts (ref: $PG17_SCRIPTS_REF)" + local scripts_base="https://raw.githubusercontent.com/supabase/postgres/${PG17_SCRIPTS_REF}/ansible/files/admin_api_scripts/pg_upgrade_scripts" + mkdir -p "$staging_dir/scripts" + for script in initiate.sh complete.sh common.sh pgsodium_getkey.sh check.sh prepare.sh; do + curl -fsSL "$scripts_base/$script" -o "$staging_dir/scripts/$script" \ + || die "Failed to download $script from GitHub" + done + + if [ -f "$TARBALL_CACHE" ]; then + info "Using cached upgrade tarball: $TARBALL_CACHE" + cp "$TARBALL_CACHE" "$staging_dir/pg_upgrade_bin.tar.gz" + return + fi + + info "Building upgrade tarball from Postgres 17 image (first run)" + docker run --rm --user root --entrypoint bash \ + -v "$staging_dir:/export" \ + "$PG17_UPGRADE_IMAGE" \ + -c ' + set -euo pipefail + mkdir -p /export/17/bin /export/17/lib /export/17/share + + echo " Copying binaries..." + # Binaries in the nix profile are either ELF binaries or shell + # wrappers that exec a .xxx-wrapped ELF from the nix store. + # Extract the actual ELF binaries so they work outside nix. + BIN_DIR=$(dirname $(readlink -f /usr/lib/postgresql/bin/postgres)) + for f in "$BIN_DIR"/*; do + name=$(basename "$f") + + # Skip nix wrapper-internal files + case "$name" in .*-wrapped) continue ;; esac + + # Check for ELF + if [ -x "$f" ] && file -b "$f" | grep -q "ELF .* executable"; then + cp "$f" /export/17/bin/"$name" + else + # Shell wrapper - extract the real .xxx-wrapped ELF path + wrapped=$(grep -o "/nix/store/[^ \"]*-wrapped" "$f" 2>/dev/null | head -n 1 || true) + if [ -n "$wrapped" ] && [ -f "$wrapped" ]; then + cp "$wrapped" /export/17/bin/"$name" + else + cp "$f" /export/17/bin/"$name" + fi + fi + done + + echo " Copying libraries..." + PKGLIBDIR=$(pg_config --pkglibdir) + LIBDIR=$(pg_config --libdir) + + # These paths may overlap (PKGLIBDIR and LIBDIR often point to the + # same nix store path). Use cp -Lf to handle overwrites from + # read-only nix store source files. + cp -Lf "$PKGLIBDIR"/*.so /export/17/lib/ || echo " Warning: cp from $PKGLIBDIR failed" >&2 + cp -Lf "$LIBDIR"/*.so* /export/17/lib/ || echo " Warning: cp from $LIBDIR failed" >&2 + cp -Lf /nix/var/nix/profiles/default/lib/*.so* /export/17/lib/ || echo " Warning: cp from nix profile lib failed" >&2 + + echo " Copying share data..." + + # Nix-built binaries resolve share dir relative to their location: + # /../share/postgresql/ + # so we need share/postgresql/ not just share/ + mkdir -p /export/17/share/postgresql + + # Remove cyclic symlink (timezonesets/timezonesets -> timezonesets). + rm -f /usr/share/postgresql/timezonesets/timezonesets 2>/dev/null || true + + # Pre-create subdirectories so cp -rL does not need to mkdir them. + # On macOS with Docker Desktop bind mount rejects mkdir with the nix store + # read-only (dr-xr-xr-x) permissions; pre-creating with default + # writable permissions fixed this + mkdir -p /export/17/share/postgresql/{extension,timezonesets,tsearch_data} + mkdir -p /export/17/share/postgresql/extension/{functions,procedures,tables,types} + + cp -rL /usr/share/postgresql/* /export/17/share/postgresql/ || echo " Warning: cp share data had errors" >&2 + + # initiate.sh copies .control/.sql from PGLIBNEW to PGSHARENEW/extension/ + echo " Copying extension definitions to lib..." + SHAREDIR=$(pg_config --sharedir) + cp "$SHAREDIR"/extension/*.control /export/17/lib/ || echo " Warning: cp .control from $SHAREDIR/extension failed" >&2 + cp "$SHAREDIR"/extension/*.sql /export/17/lib/ || echo " Warning: cp .sql from $SHAREDIR/extension failed" >&2 + + # Verify critical files before creating tarball + echo " Checking for key files..." + [ -f /export/17/bin/postgres ] || { echo "Error: bin/postgres missing"; exit 1; } + [ -f /export/17/share/postgresql/timezonesets/Default ] || { echo "Error: timezonesets/Default missing"; exit 1; } + ls /export/17/share/postgresql/extension/*.control >/dev/null 2>&1 || { echo "Error: no .control files in extension/"; exit 1; } + ls /export/17/lib/*.so >/dev/null 2>&1 || { echo "Error: no .so files in lib/"; exit 1; } + + echo " Creating tarball (this may take several minutes)..." + cd /export && tar czf pg_upgrade_bin.tar.gz 17/ + + echo " Tarball: $(du -sh /export/pg_upgrade_bin.tar.gz | cut -f1)" + ' + + # Cache for next run + cp "$staging_dir/pg_upgrade_bin.tar.gz" "$TARBALL_CACHE" + info "Tarball cached at $TARBALL_CACHE" +} + +# --- Step 3: Drop incompatible extensions ---------------------------------- + +drop_incompatible_extensions() { + if [ -z "$drop_extensions" ]; then + return + fi + info "Dropping incompatible extensions" + + local ext + echo "$drop_extensions" | tr ',' '\n' | while read -r ext; do + ext=$(echo "$ext" | tr -d '[:space:]') + [ -z "$ext" ] && continue + echo " DROP EXTENSION $ext CASCADE" + run_sql_on "$DB_CONTAINER" -c "DROP EXTENSION IF EXISTS \"$ext\" CASCADE;" + done +} + +# --- Step 4: Stop services and back up ------------------------------------- + +stop_and_backup() { + info "Backing up pgsodium root key" + local key_backup="./volumes/db/pgsodium_root.key.bak.pg15" + docker run --rm -v "${db_config_vol}:/src:ro" -v "$(pwd)/volumes/db:/dst" \ + alpine cp /src/pgsodium_root.key /dst/pgsodium_root.key.bak.pg15 \ + || die "Failed to back up pgsodium root key from db-config volume." + echo " Saved to: $key_backup" + + info "Stopping all Supabase services" + docker compose down + + echo " Original data will be preserved as: $BACKUP_DIR" +} + +# --- Step 5: Run pg_upgrade via initiate.sh + complete.sh ------------------ +# +# Host directories are mounted at non-standard paths (/mnt/host-*) with +# symlinks at the paths the upgrade scripts expect. This lets complete.sh's +# CI wrapper (which does rm/mv/ln on /var/lib/postgresql/data and +# /data_migration) operate on symlinks rather than bind mounts. + +run_upgrade() { + local abs_data_dir abs_migration_dir + + mkdir -p "$MIGRATION_DIR" + # World-writable for macOS Docker bind mount compatibility (see build_tarball) + chmod 777 "$MIGRATION_DIR" + abs_data_dir=$(cd "$DATA_DIR" && pwd) + abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) + + info "Starting upgrade container" + docker run -d --name "$UPGRADE_CONTAINER" \ + --entrypoint sleep \ + -v "${abs_data_dir}:/mnt/host-pgdata" \ + -v "${abs_migration_dir}:/mnt/host-migration" \ + -v "${db_config_vol}:/etc/postgresql-custom" \ + -v "${staging_dir}:/tmp/staging:ro" \ + -e PGPASSWORD="$pg_password" \ + "$current_image" \ + infinity + + info "Preparing upgrade environment" + docker exec "$UPGRADE_CONTAINER" bash -c ' + # Symlink bind mounts to the paths the upgrade scripts expect + rm -rf /var/lib/postgresql/data + ln -s /mnt/host-pgdata /var/lib/postgresql/data + ln -s /mnt/host-migration /data_migration + + mkdir -p /tmp/persistent /tmp/upgrade /tmp/pg_upgrade + cp /tmp/staging/pg_upgrade_bin.tar.gz /tmp/persistent/ + cp /tmp/staging/scripts/*.sh /tmp/upgrade/ + chmod +x /tmp/upgrade/*.sh + + # Patch CI_start_postgres to use "restart" instead of "start" so it + # is idempotent (initiate.sh starts postgres for top-level queries, + # then handle_extensions calls CI_start_postgres again) + sed -i "s/pg_ctl start -o/pg_ctl restart -o/g" /tmp/upgrade/common.sh + + # Patch PGSHARENEW to match nix binary expectations (share/postgresql/) + sed -i "s|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share\"|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share/postgresql\"|" /tmp/upgrade/initiate.sh + ' + + info "Starting Postgres 15 in upgrade container" + docker exec "$UPGRADE_CONTAINER" bash -c ' + su postgres -c "pg_ctl start -o \"-c config_file=/etc/postgresql/postgresql.conf\" -l /tmp/postgres.log" + ' + wait_for_healthy "$UPGRADE_CONTAINER" + + # initiate.sh expects the PG 17 binaries tarball at /tmp/persistent/pg_upgrade_bin.tar.gz + # (hardcoded path - copied there during container setup above). + # + # Env vars for the unwrapped nix ELF binaries in the tarball: + # LD_LIBRARY_PATH - find libpq, libssl, etc. (RUNPATH points to absent nix store paths) + # NIX_PGLIBDIR - postgres uses this to find extension .so files + + info "Running initiate.sh (pg_upgrade: Postgres 15 -> 17)" + echo " This may take several minutes depending on database size..." + echo "" + if ! docker exec \ + -e IS_CI=true \ + -e PG_MAJOR_VERSION=17 \ + -e PGPASSWORD="$pg_password" \ + -e LD_LIBRARY_PATH=/tmp/pg_upgrade_bin/17/lib \ + -e NIX_PGLIBDIR=/tmp/pg_upgrade_bin/17/lib \ + "$UPGRADE_CONTAINER" \ + /tmp/upgrade/initiate.sh 17; then + echo "" + warn "initiate.sh failed. Its cleanup may have restored the original state" + warn "(re-enabled extensions, revoked superuser). Your data directory is" + warn "unchanged - no data was moved or deleted." + warn "" + warn "Check the output above for the root cause, fix it, and re-run." + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true + die "initiate.sh failed" + fi + + info "initiate.sh completed successfully" + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true +} + +# --- Step 6: Run complete.sh in a native PG 17 container ------------------- +# +# complete.sh applies post-upgrade patches (pg_net grants, vault re-encryption, +# pg_cron, predefined roles, vacuumdb, etc.). We run it in a PG17 container +# where the binaries are native - no nix extraction or LD_LIBRARY_PATH needed. + +run_complete() { + local abs_migration_dir + + abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) + + info "Starting PG17 container for complete.sh" + docker run -d --name "$COMPLETE_CONTAINER" \ + --entrypoint sleep \ + -v "${abs_migration_dir}:/mnt/host-migration" \ + -v "${db_config_vol}:/etc/postgresql-custom" \ + -v "${staging_dir}:/tmp/staging:ro" \ + -e PGPASSWORD="$pg_password" \ + "$PG17_UPGRADE_IMAGE" \ + infinity + + info "Preparing complete.sh environment" + # Save original db-config ownership so we can restore it if complete.sh fails. + # complete.sh needs PG 17 ownership to start postgres, but if it fails the + # user needs to fall back to PG 15 which uses a different uid. + docker exec "$COMPLETE_CONTAINER" bash -c ' + stat -c "%u:%g" /etc/postgresql-custom/pgsodium_root.key 2>/dev/null > /tmp/dbconfig_owner || true + ' + + docker exec "$COMPLETE_CONTAINER" bash -c ' + # Symlink bind mount so complete.sh CI wrapper can mv/rm/ln + ln -s /mnt/host-migration /data_migration + + # Remove the image default data dir (complete.sh creates a symlink here) + rm -rf /var/lib/postgresql/data + + # Fix ownership on db-config volume (PG15 uid differs from PG17) + chown -R postgres:postgres /etc/postgresql-custom/ + + # PG17 config includes this directory; may not exist from PG15 + mkdir -p /etc/postgresql-custom/conf.d + + mkdir -p /tmp/upgrade + + # Copy upgrade scripts + cp /tmp/staging/scripts/*.sh /tmp/upgrade/ + chmod +x /tmp/upgrade/*.sh + + # Patch --new-bin to use native bindir (we are in a PG17 container, + # no need for /tmp/pg_upgrade_bin/ paths) + sed -i "s|BINDIR=\"/tmp/pg_upgrade_bin/\$PG_MAJOR_VERSION/bin\"|BINDIR=\$(pg_config --bindir)|g" /tmp/upgrade/common.sh + ' + + info "Running complete.sh (post-upgrade patches, vacuum analyze)" + docker exec \ + -e IS_CI=true \ + -e PG_MAJOR_VERSION=17 \ + -e PGPASSWORD="$pg_password" \ + "$COMPLETE_CONTAINER" \ + /tmp/upgrade/complete.sh || true + + # complete.sh's ERR trap exits with 0 in some cases; check status file + local status + status=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/pg-upgrade-status 2>/dev/null || echo "unknown") + if [ "$status" != "complete" ]; then + warn "complete.sh failed. Postgres log:" + docker exec "$COMPLETE_CONTAINER" cat /tmp/postgres.log 2>/dev/null || true + echo "" + # Restore db-config ownership so PG 15 can start for rollback + warn "Restoring db-config ownership for PG15..." + local orig_owner + orig_owner=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/dbconfig_owner 2>/dev/null || true) + if [ -n "$orig_owner" ]; then + docker exec "$COMPLETE_CONTAINER" chown -R "$orig_owner" /etc/postgresql-custom/ 2>/dev/null || true + fi + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true + echo "" + echo " Your Postgres 15 data is unchanged (data swap has not happened yet)." + echo " To restart Postgres 15:" + echo " rm -rf $MIGRATION_DIR" + echo " docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" + echo "" + die "complete.sh failed (status: $status)" + fi + + info "complete.sh finished successfully" + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true +} + +# --- Step 7: Swap data directories ----------------------------------------- + +swap_data() { + info "Swapping data directories" + + echo " $DATA_DIR -> $BACKUP_DIR" + mv "$DATA_DIR" "$BACKUP_DIR" + + echo " $MIGRATION_DIR/pgdata -> $DATA_DIR" + mv "$MIGRATION_DIR/pgdata" "$DATA_DIR" + rm -rf "$MIGRATION_DIR" +} + +# --- Step 8: Start Postgres 17 --------------------------------------------- + +start_pg17() { + info "Starting Supabase with Postgres 17" + + # Ensure db-config volume has correct ownership and structure for PG 17. + # complete.sh does this too, but just in case of partial + # failures from previous runs. + docker run --rm -v "${db_config_vol}:/vol" "$PG17_TARGET_IMAGE" sh -c ' + mkdir -p /vol/conf.d + chown -R postgres:postgres /vol/ + ' + + docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d + + echo " Waiting for Postgres 17 to be ready..." + local retries=60 + while [ $retries -gt 0 ]; do + if docker exec "$DB_CONTAINER" pg_isready -U postgres -h localhost >/dev/null 2>&1; then + break + fi + retries=$((retries - 1)) + sleep 2 + done + [ $retries -gt 0 ] || die "Postgres 17 did not start within 120 seconds." + + local new_version + new_version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SHOW server_version;" 2>/dev/null | head -n 1) + echo " Postgres version: $new_version" + case "$new_version" in + 17.*) ;; + *) die "Expected Postgres 17.x, got: $new_version" ;; + esac +} + +# --- Step 9: Apply migrations not covered by complete.sh ------------------- +# +# These PG 17 migrations run on fresh installs via initdb but not after +# pg_upgrade (init scripts don't rerun when PG_VERSION already exists). +# complete.sh doesn't cover them either. On the platform a tracked migration +# runner (dbmate) applies them; self-hosted has no such tracking, so we apply +# the idempotent, safe-on-an-existing-DB ones directly here. +# +# Source: postgres/migrations/db/migrations/ +# - 20250710151649_supabase_read_only_user_default_transaction_read_only.sql +# - 20251001204436_predefined_role_grants.sql (supabase_etl_admin + pg_monitor) +# - 20251105172723_grant_pg_reload_conf_to_postgres.sql +# - 20251121132723_correct_search_path_pgbouncer.sql +# - 20260211120934_supabase_privileged_role.sql (target image's supautils.conf +# references this role; it must exist or supautils config is broken) +# - 20260413000000_fix-authenticator-session-preload-libraries.sql +# - 20260421000001_rescope_pg_graphql_access_trigger.sql +# +# NOT applied: +# - 20260421000000_pg_graphql-off-by-default.sql: drops pg_graphql. Safe on a +# fresh install, but destructive on an existing DB that uses pg_graphql. +# +# This step also reconciles extension versions (complete.sh ran with the .063 +# binaries) and the pg_cron 1.6 -> 1.6.4 version label. See below. + +apply_role_migrations() { + info "Applying Postgres 17 migrations" + + # Fix collation version mismatch first (upgrade used glibc 2.39, target + # image may use glibc 2.40). Do this before any other SQL to suppress + # the noisy warnings on every subsequent command. + for db in postgres template1 _supabase; do + docker exec -i -e PGPASSWORD="$pg_password" "$DB_CONTAINER" \ + psql -h localhost -U supabase_admin -d "$db" \ + -c "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION;" || true + done + + # Create supabase_etl_admin role (doesn't exist in PG 15 images). + # Must be created before running predefined_role_grants.sql which + # assumes it exists. + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'supabase_etl_admin') THEN + CREATE USER supabase_etl_admin WITH LOGIN REPLICATION; + GRANT pg_read_all_data TO supabase_etl_admin; + GRANT CREATE ON DATABASE postgres TO supabase_etl_admin; + END IF; + END + \$\$;" || true + + # Run the migration files directly from the PG 17 container image. + # They're idempotent (IF EXISTS / IF NOT EXISTS guards). + local migration_dir="/docker-entrypoint-initdb.d/migrations" + local migrations=" + 20250710151649_supabase_read_only_user_default_transaction_read_only.sql + 20251001204436_predefined_role_grants.sql + 20251105172723_grant_pg_reload_conf_to_postgres.sql + 20251121132723_correct_search_path_pgbouncer.sql + 20260211120934_supabase_privileged_role.sql + 20260413000000_fix-authenticator-session-preload-libraries.sql + 20260421000001_rescope_pg_graphql_access_trigger.sql + " + + for m in $migrations; do + echo " Running: $m" + docker exec -i \ + -e PGPASSWORD="$pg_password" \ + "$DB_CONTAINER" \ + psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 \ + -f "${migration_dir}/${m}" || warn " $m failed (non-fatal)" + done + + # Reconcile extension versions to the target image. pg_upgrade generates + # update_extensions.sql, but complete.sh runs it inside the .063 container, so + # extensions only get updated to .063's versions. The final image ships newer + # .so files, leaving the catalog version behind what's loaded. + # + # Mirror what update_extensions.sql does, but against the running target image + # and for every installed extension: ALTER EXTENSION ... UPDATE brings each one + # up to the image's default version. This is generic on purpose - it tracks the + # image (matching platform behavior) and stays correct across future image + # bumps without maintaining a hardcoded list. ALTER ... UPDATE is a no-op when + # already at the default. Per-extension exceptions are caught so one extension + # with no update path (e.g. pg_cron 1.6 -> 1.6.4, handled below) does not abort + # the rest. + info "Reconciling extension versions to the target image" + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + DECLARE r record; + BEGIN + FOR r IN SELECT extname FROM pg_extension LOOP + BEGIN + EXECUTE format('ALTER EXTENSION %I UPDATE', r.extname); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'skipped extension %: %', r.extname, SQLERRM; + END; + END LOOP; + END + \$\$;" || warn "extension version reconcile had errors (non-fatal)" + + # pg_cron: PG 15 registers the extension as '1.6'; PG 17 images package it as + # '1.6.4' with no '1.6' -> '1.6.4' update path, so ALTER EXTENSION ... UPDATE + # has no path to follow. complete.sh only fixes this when pg_cron is owned by + # 'postgres' (it drops + recreates). Manually-enabled instances are often + # owned by supabase_admin, where that path is skipped. Reconcile the catalog + # label directly: the loaded .so and the SQL objects are already + # 1.6.4-equivalent (no SQL delta between 1.6 and 1.6.4). No DROP, so any + # scheduled jobs are untouched. + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + DECLARE want text; + BEGIN + SELECT default_version INTO want FROM pg_available_extensions WHERE name = 'pg_cron'; + IF want IS NOT NULL + AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron' AND extversion <> want) THEN + UPDATE pg_extension SET extversion = want WHERE extname = 'pg_cron'; + END IF; + END + \$\$;" || warn "pg_cron version reconcile had errors (non-fatal)" +} + +# --- Step 10: Verify ------------------------------------------------------ + +verify() { + info "Verification" + + local version + version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SELECT version();" 2>/dev/null | head -n 1) + echo " $version" + + echo "" + echo " Extensions:" + run_sql_on "$DB_CONTAINER" -c \ + "SELECT extname, extversion FROM pg_extension ORDER BY extname;" + + echo "" + info "Upgrade complete!" + echo "" + echo " To use Postgres 17 going forward, always include the override:" + echo " docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d" + echo "" + echo " Postgres 15 backup: $BACKUP_DIR" + echo " pgsodium key backup: ./volumes/db/pgsodium_root.key.bak.pg15" + echo " Once satisfied, you can reclaim space:" + echo " rm -rf $BACKUP_DIR ./volumes/db/pg17_upgrade_bin_*.tar.gz" + echo "" + echo " Rollback (if needed):" + echo " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" + echo " 2. rm -rf $DATA_DIR" + echo " 3. mv $BACKUP_DIR $DATA_DIR" + echo " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + echo " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" + echo "" +} + +# --- Main ------------------------------------------------------------------- + +main() { + echo "" + echo "Supabase Self-Hosted: Postgres 15 -> 17 Upgrade" + echo "================================================" + + preflight + pull_image + build_tarball + drop_incompatible_extensions + stop_and_backup + run_upgrade + run_complete + swap_data + start_pg17 + apply_role_migrations + verify +} + +main "$@" diff --git a/supabase/volumes/api/envoy/cds.yaml b/supabase/volumes/api/envoy/cds.yaml new file mode 100644 index 0000000000..2d47842d6b --- /dev/null +++ b/supabase/volumes/api/envoy/cds.yaml @@ -0,0 +1,223 @@ +resources: + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: auth + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: auth + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: auth + port_value: 9999 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /health + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: rest + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: rest + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: rest + port_value: 3000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: / + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: realtime + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: realtime + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: realtime-dev.supabase-realtime + port_value: 4000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: / + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: storage + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: storage + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: storage + port_value: 5000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /status + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: functions + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: functions + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: functions + port_value: 9000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + tcp_health_check: {} + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: meta + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: meta + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: meta + port_value: 8080 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /health + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: studio + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: studio + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: studio + port_value: 3000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /project/default + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 diff --git a/supabase/volumes/api/envoy/docker-entrypoint.sh b/supabase/volumes/api/envoy/docker-entrypoint.sh new file mode 100755 index 0000000000..2c67584a30 --- /dev/null +++ b/supabase/volumes/api/envoy/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -e + +# Generate SHA1 base64 hash for Envoy basic auth user list +PASSWORD_HASH=$(printf '%s' "${DASHBOARD_PASSWORD}" | openssl sha1 -binary | openssl base64) +DASHBOARD_BASIC_AUTH="${DASHBOARD_USERNAME}:{SHA}${PASSWORD_HASH}" + +echo "Generating Envoy configuration..." + +# Process the lds.yaml template with environment variables using sed +# Using | as delimiter since JWT tokens contain / +sed -e "s|\${ANON_KEY}|${ANON_KEY}|g" \ + -e "s|\${ANON_KEY_ASYMMETRIC}|${ANON_KEY_ASYMMETRIC}|g" \ + -e "s|\${SERVICE_ROLE_KEY}|${SERVICE_ROLE_KEY}|g" \ + -e "s|\${SERVICE_ROLE_KEY_ASYMMETRIC}|${SERVICE_ROLE_KEY_ASYMMETRIC}|g" \ + -e "s|\${SUPABASE_PUBLISHABLE_KEY}|${SUPABASE_PUBLISHABLE_KEY}|g" \ + -e "s|\${SUPABASE_SECRET_KEY}|${SUPABASE_SECRET_KEY}|g" \ + -e "s|\${SUPABASE_PUBLIC_URL}|${SUPABASE_PUBLIC_URL}|g" \ + -e "s|\${DASHBOARD_BASIC_AUTH}|${DASHBOARD_BASIC_AUTH}|g" \ + /etc/envoy/lds.template.yaml > /etc/envoy/lds.yaml + +if [ -n "$SUPABASE_SECRET_KEY" ] && \ + [ -n "$SUPABASE_PUBLISHABLE_KEY" ] && \ + [ -n "$SERVICE_ROLE_KEY_ASYMMETRIC" ] && \ + [ -n "$ANON_KEY_ASYMMETRIC" ]; then + echo "Envoy sb_ key translation enabled" +else + echo "Envoy running in legacy API key mode (sb_ keys disabled)" +fi + +echo "Envoy configuration generated successfully" +echo "Starting Envoy..." + +# Start Envoy +exec envoy -c /etc/envoy/envoy.yaml "$@" diff --git a/supabase/volumes/api/envoy/envoy.yaml b/supabase/volumes/api/envoy/envoy.yaml new file mode 100644 index 0000000000..def443b688 --- /dev/null +++ b/supabase/volumes/api/envoy/envoy.yaml @@ -0,0 +1,27 @@ +dynamic_resources: + cds_config: + path_config_source: + path: /etc/envoy/cds.yaml + resource_api_version: V3 + lds_config: + path_config_source: + path: /etc/envoy/lds.yaml + resource_api_version: V3 + +node: + cluster: supabase_cluster + id: supabase_node + +overload_manager: + resource_monitors: + - name: envoy.resource_monitors.global_downstream_max_connections + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 30000 + +admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 diff --git a/supabase/volumes/api/envoy/lds.template.yaml b/supabase/volumes/api/envoy/lds.template.yaml new file mode 100644 index 0000000000..1a3c345188 --- /dev/null +++ b/supabase/volumes/api/envoy/lds.template.yaml @@ -0,0 +1,1101 @@ +resources: + - '@type': type.googleapis.com/envoy.config.listener.v3.Listener + name: supabase + per_connection_buffer_limit_bytes: 32768 # 32 KiB + + address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + normalize_path: true + merge_slashes: true + path_with_escaped_slashes_action: REJECT_REQUEST + use_remote_address: true + common_http_protocol_options: + headers_with_underscores_action: REJECT_REQUEST + upgrade_configs: + - upgrade_type: websocket + access_log: + - name: envoy.access_loggers.stdout + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format_source: + inline_string: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% - - [%START_TIME(%d/%b/%Y:%H:%M:%S %z)%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %BYTES_SENT% \"%REQ(REFERER)%\" \"%REQ(USER-AGENT)%\"\n" + + route_config: + name: supabase_route + virtual_hosts: + - name: supabase_host + domains: + - '*' + cors: + allow_origin_string_match: + - safe_regex: + regex: ".*" + allow_methods: "GET,POST,PUT,PATCH,DELETE,OPTIONS,HEAD,CONNECT,TRACE" + allow_headers: "*" + expose_headers: "*" + max_age: "3600" + request_headers_to_add: + - header: + key: X-Forwarded-Host + value: "%REQ(:AUTHORITY)%" + append_action: ADD_IF_ABSENT + - header: + key: X-Forwarded-Port + value: "%DOWNSTREAM_LOCAL_PORT%" + append_action: ADD_IF_ABSENT + routes: + - match: + prefix: /auth/v1/verify + route: + cluster: auth + prefix_rewrite: /verify + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/verify + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/callback + route: + cluster: auth + prefix_rewrite: /callback + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/callback + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/authorize + route: + cluster: auth + prefix_rewrite: /authorize + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/authorize + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/.well-known/jwks.json + route: + cluster: auth + prefix_rewrite: /.well-known/jwks.json + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/.well-known/jwks.json + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /.well-known/oauth-authorization-server + route: + cluster: auth + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /.well-known/oauth-authorization-server + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/sso/saml/acs + route: + cluster: auth + prefix_rewrite: /sso/saml/acs + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/sso/saml/acs + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/sso/saml/metadata + route: + cluster: auth + prefix_rewrite: /sso/saml/metadata + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/sso/saml/metadata + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - name: functions-v1-all + match: + prefix: /functions/v1/ + route: + cluster: functions + prefix_rewrite: / + timeout: 150s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /functions/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /storage/v1/ + route: + cluster: storage + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /storage/v1 + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - name: auth-v1-protected + match: + prefix: /auth/v1/ + route: + cluster: auth + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: rest-v1-openapi-protected + match: + path: /rest/v1/ + route: + cluster: rest + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /rest/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + admin: + permissions: + - any: true + principals: + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY}' + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' + + - name: rest-v1-protected + match: + prefix: /rest/v1/ + route: + cluster: rest + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /rest/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: graphql-v1-protected + match: + prefix: /graphql/v1 + route: + cluster: rest + prefix_rewrite: /rpc/graphql + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /graphql/v1 + append_action: ADD_IF_ABSENT + - header: + key: Content-Profile + value: graphql_public + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: realtime-v1-api-openapi-blocked + match: + prefix: /realtime/v1/api/openapi + route: + cluster: realtime + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/api/openapi + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + + - name: realtime-v1-api-tenants-blocked + match: + prefix: /realtime/v1/api/tenants + route: + cluster: realtime + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/api/tenants + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + + - name: realtime-v1-api-protected + match: + prefix: /realtime/v1/api + route: + cluster: realtime + prefix_rewrite: /api + timeout: 30s + host_rewrite_literal: realtime-dev.supabase-realtime + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/api + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: realtime-v1-ws-protected + match: + prefix: /realtime/v1/ + route: + cluster: realtime + prefix_rewrite: /socket/ + timeout: 30s + host_rewrite_literal: realtime-dev.supabase-realtime + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: pg-protected + match: + prefix: /pg/ + route: + cluster: meta + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /pg/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.cors: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy + allow_origin_string_match: + - exact: '${SUPABASE_PUBLIC_URL}' + - safe_regex: + regex: 'https?://(localhost|127\.0\.0\.1)(:[0-9]+)?' + allow_methods: "GET,POST,PUT,PATCH,DELETE,OPTIONS,HEAD" + allow_headers: "*" + expose_headers: "*" + max_age: "3600" + + - match: + prefix: /api/mcp + route: + cluster: studio + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /api/mcp + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /mcp + route: + cluster: studio + prefix_rewrite: /api/mcp + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /mcp + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + # Block access to /mcp by default + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + # Enable local access (danger zone!) + # 1. Comment out the 'rbac' block above. + # 2. Uncomment and adjust the 'rbac' block below. + # 3. Add or adjust your local IPs in 'principals'. + #rbac: + # rules: + # action: ALLOW + # policies: + # allow_local: + # permissions: + # - any: true + # principals: + # - direct_remote_ip: + # address_prefix: 127.0.0.1 + # prefix_len: 32 + # - direct_remote_ip: + # address_prefix: ::1 + # prefix_len: 128 + + - match: + prefix: / + route: + cluster: studio + timeout: 30s + request_headers_to_remove: + - authorization + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: / + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + http_filters: + - name: envoy.filters.http.cors + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + + - name: envoy.filters.http.basic_auth + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth + users: + inline_string: '${DASHBOARD_BASIC_AUTH}' + + # Copies ?apikey=... from the URL into the apikey header when clients omit the header. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + if headers:get("apikey") ~= nil then + return + end + + local path = headers:get(":path") + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return + end + + local query = string.sub(path, query_start + 1) + for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do + if key == "apikey" and value ~= "" then + headers:add("apikey", value) + return + end + end + end + + # Returns 401 for missing/invalid API keys on protected API routes. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local ANON_KEY = "${ANON_KEY}" + local SERVICE_ROLE_KEY = "${SERVICE_ROLE_KEY}" + local SUPABASE_PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SUPABASE_SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local ANON_KEY_ASYMMETRIC = "${ANON_KEY_ASYMMETRIC}" + local SERVICE_ROLE_KEY_ASYMMETRIC = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SUPABASE_SECRET_KEY ~= "" and SUPABASE_PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_KEY_ASYMMETRIC ~= "" and ANON_KEY_ASYMMETRIC ~= "" + + local PROTECTED_ROUTES = { + ["auth-v1-protected"] = true, + ["rest-v1-openapi-protected"] = true, + ["rest-v1-protected"] = true, + ["graphql-v1-protected"] = true, + ["realtime-v1-api-protected"] = true, + ["realtime-v1-ws-protected"] = true, + ["pg-protected"] = true, + } + + local function is_protected_route(route_name) + if route_name == nil or route_name == "" then + return false + end + + return PROTECTED_ROUTES[route_name] == true + end + + local function is_valid_apikey(apikey) + if apikey == nil or apikey == "" then + return false + end + + if SERVICE_ROLE_KEY ~= "" and apikey == SERVICE_ROLE_KEY then + return true + end + + if ANON_KEY ~= "" and apikey == ANON_KEY then + return true + end + + if TRANSLATION_ENABLED and apikey == SUPABASE_SECRET_KEY then + return true + end + + if TRANSLATION_ENABLED and apikey == SUPABASE_PUBLISHABLE_KEY then + return true + end + + return false + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + local route_name = request_handle:streamInfo():routeName() + if not is_protected_route(route_name) then + return + end + + if is_valid_apikey(headers:get("apikey")) then + return + end + + request_handle:respond({ + [":status"] = "401", + ["content-type"] = "text/plain", + }, "Unauthorized") + end + + # Translates the query parameter apikey into the matching internal JWT and rewrites the URL so only JWTs propagate downstream. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function translate_apikey(apikey) + if apikey == nil or apikey == "" then + return nil + end + + if not TRANSLATION_ENABLED then + return nil + end + + if apikey == SECRET_KEY then + return SERVICE_ROLE_JWT + end + + if apikey == PUBLISHABLE_KEY then + return ANON_JWT + end + + return nil + end + + local function extract_query_apikey(path) + if path == nil or path == "" then + return nil + end + + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return nil + end + + local query = string.sub(path, query_start + 1) + for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do + if key == "apikey" and value ~= "" then + return value + end + end + + return nil + end + + local function replace_query_apikey(path, new_value) + if path == nil or path == "" or new_value == nil or new_value == "" then + return nil + end + + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return nil + end + + local base = string.sub(path, 1, query_start) + local query = string.sub(path, query_start + 1) + local updated = {} + local replaced = false + + for part in string.gmatch(query, "([^&]+)") do + local key, value = string.match(part, "([^=]+)=(.*)") + if key == "apikey" then + part = key .. "=" .. new_value + replaced = true + end + table.insert(updated, part) + end + + if not replaced then + return nil + end + + return base .. table.concat(updated, "&") + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + local path = headers:get(":path") + local apikey = extract_query_apikey(path) + local translated = translate_apikey(apikey) + + if translated == nil then + return + end + + headers:replace("apikey", translated) + + local rewritten_path = replace_query_apikey(path, translated) + if rewritten_path ~= nil then + headers:replace(":path", rewritten_path) + end + end + + # Translates an apikey header into the appropriate internal JWT for downstream RBAC checks. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function translate_apikey(apikey) + if apikey == nil or apikey == "" then + return nil + end + + if not TRANSLATION_ENABLED then + return nil + end + + if apikey == SECRET_KEY then + return SERVICE_ROLE_JWT + end + + if apikey == PUBLISHABLE_KEY then + return ANON_JWT + end + + return nil + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + local translated = translate_apikey(headers:get("apikey")) + if translated ~= nil and translated ~= "" then + headers:replace("apikey", translated) + end + end + + # Mirrors apikey into x-api-key for realtime WS compatibility. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" + + function envoy_on_request(request_handle) + local route_name = request_handle:streamInfo():routeName() + if route_name ~= REALTIME_WS_ROUTE then + return + end + + local headers = request_handle:headers() + local apikey = headers:get("apikey") + if apikey == nil or apikey == "" then + return + end + + headers:replace("x-api-key", apikey) + end + + # Synthesizes an Authorization header (Bearer …) from apikey when callers don’t provide a real JWT header. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function has_real_jwt(auth_header) + if auth_header == nil or auth_header == "" then + return false + end + + if string.sub(auth_header, 1, 7) ~= "Bearer " then + return false + end + + return string.sub(auth_header, 1, 10) ~= "Bearer sb_" + end + + local function format_authorization(value) + if value == nil or value == "" then + return nil + end + + if string.sub(value, 1, 7) == "Bearer " then + return value + end + + return "Bearer " .. value + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if request_handle:streamInfo():routeName() == REALTIME_WS_ROUTE then + return + end + + if is_functions_request(request_handle, headers) then + return + end + + if has_real_jwt(headers:get("authorization")) then + return + end + + local apikey = headers:get("apikey") + local authorization_value = format_authorization(apikey) + if authorization_value ~= nil then + headers:replace("authorization", authorization_value) + end + end + + - name: envoy.filters.http.rbac + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC + rules: + action: ALLOW + policies: + admin: + permissions: + - url_path: + path: + prefix: /pg/ + principals: + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY}' + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' + apikey: + permissions: + - url_path: + path: + prefix: /auth/v1/ + - url_path: + path: + prefix: /rest/v1/ + - url_path: + path: + prefix: /realtime/v1/api + - url_path: + path: + prefix: /realtime/v1/ + - url_path: + path: + prefix: /graphql/v1 + principals: + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY}' + - header: + name: apikey + string_match: + exact: '${ANON_KEY}' + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' + - header: + name: apikey + string_match: + exact: '${ANON_KEY_ASYMMETRIC}' + - name: envoy.filters.http.router + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/supabase/volumes/api/kong.yml b/supabase/volumes/api/kong.yml deleted file mode 100644 index b90c111b69..0000000000 --- a/supabase/volumes/api/kong.yml +++ /dev/null @@ -1,241 +0,0 @@ -_format_version: '2.1' -_transform: true - -### -### Consumers / Users -### -consumers: - - username: DASHBOARD - - username: anon - keyauth_credentials: - - key: $SUPABASE_ANON_KEY - - username: service_role - keyauth_credentials: - - key: $SUPABASE_SERVICE_KEY - -### -### Access Control List -### -acls: - - consumer: anon - group: anon - - consumer: service_role - group: admin - -### -### Dashboard credentials -### -basicauth_credentials: - - consumer: DASHBOARD - username: $DASHBOARD_USERNAME - password: $DASHBOARD_PASSWORD - -### -### API Routes -### -services: - ## Open Auth routes - - name: auth-v1-open - url: http://auth:9999/verify - routes: - - name: auth-v1-open - strip_path: true - paths: - - /auth/v1/verify - plugins: - - name: cors - - name: auth-v1-open-callback - url: http://auth:9999/callback - routes: - - name: auth-v1-open-callback - strip_path: true - paths: - - /auth/v1/callback - plugins: - - name: cors - - name: auth-v1-open-authorize - url: http://auth:9999/authorize - routes: - - name: auth-v1-open-authorize - strip_path: true - paths: - - /auth/v1/authorize - plugins: - - name: cors - - ## Secure Auth routes - - name: auth-v1 - _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' - url: http://auth:9999/ - routes: - - name: auth-v1-all - strip_path: true - paths: - - /auth/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure REST routes - - name: rest-v1 - _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' - url: http://rest:3000/ - routes: - - name: rest-v1-all - strip_path: true - paths: - - /rest/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure GraphQL routes - - name: graphql-v1 - _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql' - url: http://rest:3000/rpc/graphql - routes: - - name: graphql-v1-all - strip_path: true - paths: - - /graphql/v1 - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: request-transformer - config: - add: - headers: - - Content-Profile:graphql_public - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure Realtime routes - - name: realtime-v1-ws - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/socket - protocol: ws - routes: - - name: realtime-v1-ws - strip_path: true - paths: - - /realtime/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - name: realtime-v1-rest - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/api - protocol: http - routes: - - name: realtime-v1-rest - strip_path: true - paths: - - /realtime/v1/api - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - ## Storage routes: the storage server manages its own auth - - name: storage-v1 - _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' - url: http://storage:5000/ - routes: - - name: storage-v1-all - strip_path: true - paths: - - /storage/v1/ - plugins: - - name: cors - - ## Edge Functions routes - - name: functions-v1 - _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' - url: http://functions:9000/ - routes: - - name: functions-v1-all - strip_path: true - paths: - - /functions/v1/ - plugins: - - name: cors - - ## Analytics routes - - name: analytics-v1 - _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*' - url: http://analytics:4000/ - routes: - - name: analytics-v1-all - strip_path: true - paths: - - /analytics/v1/ - - ## Secure Database routes - - name: meta - _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*' - url: http://meta:8080/ - routes: - - name: meta-all - strip_path: true - paths: - - /pg/ - plugins: - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - ## Protected Dashboard - catch all remaining routes - - name: dashboard - _comment: 'Studio: /* -> http://studio:3000/*' - url: http://studio:3000/ - routes: - - name: dashboard-all - strip_path: true - paths: - - / - plugins: - - name: cors - - name: basic-auth - config: - hide_credentials: true \ No newline at end of file diff --git a/supabase/volumes/db/init/data.sql b/supabase/volumes/db/init/data.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/supabase/volumes/db/jwt.sql b/supabase/volumes/db/jwt.sql index cfd3b16028..09a7283666 100644 --- a/supabase/volumes/db/jwt.sql +++ b/supabase/volumes/db/jwt.sql @@ -1,5 +1,3 @@ -\set jwt_secret `echo "$JWT_SECRET"` \set jwt_exp `echo "$JWT_EXP"` -ALTER DATABASE postgres SET "app.settings.jwt_secret" TO :'jwt_secret'; ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; diff --git a/supabase/volumes/db/pooler.sql b/supabase/volumes/db/pooler.sql new file mode 100644 index 0000000000..162c5b96aa --- /dev/null +++ b/supabase/volumes/db/pooler.sql @@ -0,0 +1,6 @@ +\set pguser `echo "$POSTGRES_USER"` + +\c _supabase +create schema if not exists _supavisor; +alter schema _supavisor owner to :pguser; +\c postgres diff --git a/supabase/volumes/db/realtime.sql b/supabase/volumes/db/realtime.sql new file mode 100644 index 0000000000..4d4b9ffb91 --- /dev/null +++ b/supabase/volumes/db/realtime.sql @@ -0,0 +1,4 @@ +\set pguser `echo "$POSTGRES_USER"` + +create schema if not exists _realtime; +alter schema _realtime owner to :pguser; diff --git a/supabase/volumes/db/webhooks.sql b/supabase/volumes/db/webhooks.sql index 5837b86188..fe273decb8 100644 --- a/supabase/volumes/db/webhooks.sql +++ b/supabase/volumes/db/webhooks.sql @@ -122,6 +122,10 @@ BEGIN; ALTER table "supabase_functions".hooks OWNER TO supabase_functions_admin; ALTER function "supabase_functions".http_request() OWNER TO supabase_functions_admin; GRANT supabase_functions_admin TO postgres; + -- http_request() is SECURITY DEFINER and runs as supabase_functions_admin. Serializing + -- NEW/OLD can touch objects in the extensions schema (PostGIS geometry reads + -- extensions.spatial_ref_sys), so the role needs USAGE on it. + GRANT USAGE ON SCHEMA extensions TO supabase_functions_admin; -- Remove unused supabase_pg_net_admin role DO $$ diff --git a/supabase/volumes/functions/deno.jsonc b/supabase/volumes/functions/deno.jsonc new file mode 100644 index 0000000000..db206e8f46 --- /dev/null +++ b/supabase/volumes/functions/deno.jsonc @@ -0,0 +1,6 @@ +{ + "imports": { + "@supabase/functions-js": "jsr:@supabase/functions-js@^2", + "@supabase/server": "npm:@supabase/server@^1" + } +} diff --git a/supabase/volumes/functions/hello/index.ts b/supabase/volumes/functions/hello/index.ts new file mode 100644 index 0000000000..50d0e6be8c --- /dev/null +++ b/supabase/volumes/functions/hello/index.ts @@ -0,0 +1,37 @@ +// Follow this setup guide to integrate the Deno language server with your editor: +// https://deno.land/manual/getting_started/setup_your_environment +// This enables autocomplete, go to definition, etc. + +// Setup type definitions for built-in Supabase Runtime APIs +import "@supabase/functions-js/edge-runtime.d.ts" +import { withSupabase } from "@supabase/server" + +// Logs are visible from 'functions' container inspector +console.log("Hello from Functions!"); + +// This endpoint uses 'publishable' | 'secret' access, apiKey is required. +// Use publishable for Client-facing, key-validated endpoints +// Use secret for Server-to-server, internal calls +export default { + fetch: withSupabase({ auth: ["publishable", "secret"] }, async (req, ctx) => { + // Called by another service with a secret key + // ctx.supabaseAdmin bypasses RLS — use for privileged operations + /* + if (ctx.authMode === "secret") { + const { user_id } = await req.json(); + const { data } = await ctx.supabaseAdmin.auth.admin.getUserById(user_id); + + return Response.json({ + email: data?.user?.email, + }); + } + */ + + return Response.json({ message: "Hello from Edge Functions!" }); + }), +}; + +// To invoke: +// curl 'http://localhost:/functions/v1/hello' \ +// --header 'apiKey: ' + diff --git a/supabase/volumes/functions/main/index.ts b/supabase/volumes/functions/main/index.ts new file mode 100644 index 0000000000..6bfad388a8 --- /dev/null +++ b/supabase/volumes/functions/main/index.ts @@ -0,0 +1,232 @@ +import * as jose from 'jsr:@panva/jose@6' + +console.log('main function started') + +const JWT_SECRET = Deno.env.get('JWT_SECRET') +const SUPABASE_JWKS = parseJwks(Deno.env.get('SUPABASE_JWKS')) +const LOCAL_JWKS = SUPABASE_JWKS ? jose.createLocalJWKSet(SUPABASE_JWKS) : null +const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' + +type AuthFailure = { + code: RequestErrors + message?: string +} + +export enum RequestErrors { + InvalidLegacyJWT = 'UNAUTHORIZED_LEGACY_JWT', + InvalidAsymmetricJWT = 'UNAUTHORIZED_ASYMMETRIC_JWT', + InvalidTokenFormat = 'UNAUTHORIZED_INVALID_JWT_FORMAT', + UnsupportedTokenAlgorithm = 'UNAUTHORIZED_UNSUPPORTED_TOKEN_ALGORITHM', + MissingAuthHeader = 'UNAUTHORIZED_NO_AUTH_HEADER', +} + +// NOTE:(kallebysantos) We don't check for valid keys but just the bare array parsing, +// let this for 'jose' lib verification +export function parseJwks(raw: string | undefined): jose.JSONWebKeySet | null { + if (!raw) return null + try { + const parsed = JSON.parse(raw) + if (parsed?.keys && Array.isArray(parsed.keys)) { + return parsed as jose.JSONWebKeySet + } + return null + } catch { + return null + } +} + +/** + * Extract JWT token from Authorization header + * + * Parses the Authorization header to extract the Bearer token. + * Expects format: "Bearer " + * + * @param req - The HTTP request object + * @returns The JWT token string or an authentication failure + */ +function getAuthToken(req: Request): string | AuthFailure { + const authHeader = req.headers.get('authorization') + if (!authHeader) { + return { + code: RequestErrors.MissingAuthHeader, + message: 'Missing authorization header', + } + } + const tokenParts = authHeader.trim().split(/\s+/) + const [bearer, token] = tokenParts + if (bearer.toLowerCase() !== 'bearer' || tokenParts.length !== 2 || !token) { + return { + code: RequestErrors.InvalidTokenFormat, + message: 'Invalid JWT format', + } + } + return token +} + +function getAuthErrorResponse({ code, message = 'Invalid JWT' }: AuthFailure) { + return Response.json( + { + code, + message, + // DEPRECATED: Retained for backward compatibility. + msg: message, + }, + { + status: 401, + headers: { + 'sb-error-code': code, + 'Access-Control-Expose-Headers': 'sb-error-code', + }, + } + ) +} + +async function isValidLegacyJWT(jwt: string): Promise { + if (!JWT_SECRET) { + console.error('JWT_SECRET not available for HS256 token verification') + return { code: RequestErrors.InvalidLegacyJWT } + } + + const encoder = new TextEncoder(); + const secretKey = encoder.encode(JWT_SECRET); + + try { + await jose.jwtVerify(jwt, secretKey); + } catch (e) { + console.error('Symmetric Legacy JWT verification error', e); + return { code: RequestErrors.InvalidLegacyJWT } + } + return null +} + +async function isValidJWT(jwt: string): Promise { + if (!LOCAL_JWKS) { + console.error('JWKS not available for ES256/RS256 token verification') + return { code: RequestErrors.InvalidAsymmetricJWT } + } + + try { + await jose.jwtVerify(jwt, LOCAL_JWKS); + } catch (e) { + console.error('Asymmetric JWT verification error', e); + return { code: RequestErrors.InvalidAsymmetricJWT } + } + + return null +} + +/** + * Verify JWT token, handling both legacy (HS256) and newer (ES256/RS256) algorithms + * + * This function automatically detects the algorithm used in the token and applies + * the appropriate verification method: + * - HS256: Uses JWT_SECRET (symmetric key) + * - ES256/RS256: Uses JWKS endpoint (asymmetric public keys) + * + * This fix ensures compatibility with both legacy tokens and newer asymmetric tokens, + * resolving the "Key for the ES256 algorithm must be of type CryptoKey" error. + * + * @param jwt - The JWT token string to verify + * @returns Authentication failure details, or null when verification succeeds + */ +async function isValidHybridJWT(jwt: string): Promise { + let jwtAlgorithm: string | undefined + try { + jwtAlgorithm = jose.decodeProtectedHeader(jwt).alg + } catch (e) { + console.error('JWT format error', e) + return { + code: RequestErrors.InvalidTokenFormat, + message: 'Invalid JWT format', + } + } + + if (!jwtAlgorithm) { + return { + code: RequestErrors.InvalidTokenFormat, + message: 'Invalid JWT format', + } + } + + if (jwtAlgorithm === 'HS256') { + console.log(`Legacy token type detected, attempting ${jwtAlgorithm} verification.`) + + return await isValidLegacyJWT(jwt) + } + + if (jwtAlgorithm === 'ES256' || jwtAlgorithm === 'RS256') { + return await isValidJWT(jwt) + } + + return { + code: RequestErrors.UnsupportedTokenAlgorithm, + message: `Unsupported JWT algorithm ${jwtAlgorithm}`, + } +} + +Deno.serve(async (req: Request) => { + if (req.method !== 'OPTIONS' && VERIFY_JWT) { + try { + const token = getAuthToken(req) + if (typeof token !== 'string') { + return getAuthErrorResponse(token) + } + const authFailure = await isValidHybridJWT(token) + if (authFailure) { + return getAuthErrorResponse(authFailure) + } + } catch (e) { + console.error(e) + return getAuthErrorResponse({ + code: RequestErrors.InvalidTokenFormat, + message: 'Invalid JWT format', + }) + } + } + + const url = new URL(req.url) + const { pathname } = url + const path_parts = pathname.split('/') + const service_name = path_parts[1] + + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' } + return new Response(JSON.stringify(error), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + const servicePath = `/home/deno/functions/${service_name}` + console.error(`serving the request with ${servicePath}`) + + const memoryLimitMb = 150 + const workerTimeoutMs = 1 * 60 * 1000 + const noModuleCache = false + // Using a common Import Map for all functions + // to use a scope 'deno.json' it must be dinamically resolved base on the 'service_name' + const importMapPath = `/home/deno/functions/deno.jsonc` + // SUPABASE_FUNCTION_SLUG is listed after the container env snapshot so + // nothing in it can shadow the value, and it is per-request because only this + // worker knows which function the request resolved to. + const envVarsObj = { ...Deno.env.toObject(), SUPABASE_FUNCTION_SLUG: service_name } + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + }) + return await worker.fetch(req) + } catch (e) { + const error = { msg: e.toString() } + return new Response(JSON.stringify(error), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } +}) diff --git a/supabase/volumes/pooler/pooler.exs b/supabase/volumes/pooler/pooler.exs new file mode 100644 index 0000000000..06f05e1b4a --- /dev/null +++ b/supabase/volumes/pooler/pooler.exs @@ -0,0 +1,30 @@ +{:ok, _} = Application.ensure_all_started(:supavisor) + +{:ok, version} = + case Supavisor.Repo.query!("select version()") do + %{rows: [[ver]]} -> Supavisor.Helpers.parse_pg_version(ver) + _ -> nil + end + +params = %{ + "external_id" => System.get_env("POOLER_TENANT_ID"), + "db_host" => System.get_env("POSTGRES_HOST") || "db", + "db_port" => System.get_env("POSTGRES_PORT"), + "db_database" => System.get_env("POSTGRES_DB"), + "require_user" => false, + "auth_query" => "SELECT * FROM pgbouncer.get_auth($1)", + "default_max_clients" => System.get_env("POOLER_MAX_CLIENT_CONN"), + "default_pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), + "default_parameter_status" => %{"server_version" => version}, + "users" => [%{ + "db_user" => "pgbouncer", + "db_password" => System.get_env("POSTGRES_PASSWORD"), + "mode_type" => System.get_env("POOLER_POOL_MODE"), + "pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), + "is_manager" => true + }] +} + +if !Supavisor.Tenants.get_tenant_by_external_id(params["external_id"]) do + {:ok, _} = Supavisor.Tenants.create_tenant(params) +end diff --git a/tests/supabase-live.spec.ts b/tests/supabase-live.spec.ts new file mode 100644 index 0000000000..5e90f6c407 --- /dev/null +++ b/tests/supabase-live.spec.ts @@ -0,0 +1,113 @@ +/* eslint-disable no-await-in-loop */ +import { randomUUID } from 'node:crypto'; +import { createClient } from '@supabase/supabase-js'; +import { expect, Page, test } from '@playwright/test'; +import { ParticipantData } from '../src/storage/types'; +import { nextClick, waitForStudyEndMessage } from './utils'; + +// Run against an isolated local stack, with Vite configured to use the same Supabase URL. +// Keep the participants so this suite can verify them again after a migration/restart. +const enabled = process.env.PW_SUPABASE_LIVE === '1'; +const studyId = 'dev-demo-condition'; + +test.describe('Live self-hosted Supabase', () => { + test.skip(!enabled, 'Set PW_SUPABASE_LIVE=1 for the local Supabase migration rehearsal'); + + function storageClient() { + const url = process.env.VITE_SUPABASE_URL; + const key = process.env.PW_SUPABASE_SERVICE_ROLE_KEY; + if (!url || !key) throw new Error('Live Supabase requires VITE_SUPABASE_URL and PW_SUPABASE_SERVICE_ROLE_KEY'); + return createClient(url, key, { auth: { persistSession: false, autoRefreshToken: false } }); + } + + async function readParticipant(participantId: string) { + const { data, error } = await storageClient().storage.from('revisit') + .download(`${studyId}/participants/${participantId}_participantData`); + if (error || !data) return null; + return JSON.parse(await data.text()) as ParticipantData; + } + + function colorAnswers(participant: ParticipantData | null) { + return Object.values(participant?.answers ?? {}) + .filter((answer) => answer.componentName.startsWith('color-trial-')) + .map((answer) => answer.answer); + } + + async function selectParticipant(page: Page, participantId: string) { + await page.goto('/'); + await page.evaluate(({ id, study }) => new Promise((resolve, reject) => { + const request = indexedDB.open('revisit-supabase'); + request.onupgradeneeded = () => request.result.createObjectStore('keyvaluepairs'); + request.onerror = () => reject(request.error); + request.onsuccess = () => { + const database = request.result; + const transaction = database.transaction('keyvaluepairs', 'readwrite'); + transaction.objectStore('keyvaluepairs').put(id, `${study}/currentParticipantId`); + transaction.oncomplete = () => { database.close(); resolve(); }; + transaction.onerror = () => { database.close(); reject(transaction.error); }; + }; + }), { id: participantId, study: studyId }); + } + + test('persists real participant answers and resumes from Supabase in a fresh browser context', async ({ page, browser }) => { + const startedAt = Date.now(); + const participantId = process.env.PW_SUPABASE_PARTICIPANT_ID || randomUUID(); + const studyPath = '/demo-condition?condition=color'; + await selectParticipant(page, participantId); + await page.goto(studyPath); + await expect(page.getByText(/url query parameter.*condition/i)).toBeVisible(); + await nextClick(page); + + for (let trial = 0; trial < 2; trial += 1) { + await expect(page.getByText(/Which color is the (lightest|darkest)\?/)).toBeVisible(); + await page.getByPlaceholder('Select an option').click(); + await page.getByRole('option', { name: 'A', exact: true }).click(); + await nextClick(page); + } + await waitForStudyEndMessage(page); + + await expect.poll(async () => colorAnswers(await readParticipant(participantId)), { timeout: 15000 }) + .toEqual(expect.arrayContaining([ + { 'color-trial-1-response': 'A' }, + { 'color-trial-2-response': 'A' }, + ])); + const participant = await readParticipant(participantId); + expect(participant?.participantId).toBe(participantId); + expect(participant?.participantConfigHash).toBeTruthy(); + const { data: assignments, error } = await storageClient().from('revisit').select('data') + .eq('studyId', studyId) + .eq('docId', `sequenceAssignment_${participantId}`); + expect(error).toBeNull(); + expect(assignments?.[0]?.data.completed).toBeGreaterThanOrEqual(startedAt); + + // Force the reload to retrieve the participant from the backend, not localforage. + const completedUrl = page.url(); + const freshContext = await browser.newContext({ baseURL: new URL(completedUrl).origin }); + try { + const freshPage = await freshContext.newPage(); + await selectParticipant(freshPage, participantId); + await freshPage.goto(completedUrl); + await waitForStudyEndMessage(freshPage); + } finally { + await freshContext.close(); + } + expect(colorAnswers(await readParticipant(participantId))).toEqual(colorAnswers(participant)); + }); + + test('reads a participant retained from the old stack after migration', async ({ page }) => { + const participantId = process.env.PW_SUPABASE_EXISTING_PARTICIPANT; + test.skip(!participantId, 'Set PW_SUPABASE_EXISTING_PARTICIPANT to the pre-migration participant ID'); + const participant = await readParticipant(participantId!); + expect(participant?.participantId).toBe(participantId); + expect(colorAnswers(participant)).toEqual(expect.arrayContaining([ + { 'color-trial-1-response': 'A' }, + { 'color-trial-2-response': 'A' }, + ])); + await selectParticipant(page, participantId!); + await page.goto('/demo-condition?condition=color'); + await nextClick(page); + await expect(page.getByText(/Which color is the (lightest|darkest)\?/)).toBeVisible(); + await expect(page.getByPlaceholder('Select an option')).toHaveValue('A'); + expect(colorAnswers(await readParticipant(participantId!))).toEqual(colorAnswers(participant)); + }); +}); From 39abb1d481392b50d16cddc47d85888390e1eb10 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 21:52:57 -0600 Subject: [PATCH 02/14] Ignore private Supabase migration backups --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 04ccce548e..d1e3884e9d 100644 --- a/.gitignore +++ b/.gitignore @@ -38,5 +38,7 @@ supabase/volumes/db/data_migration/ supabase/volumes/db/pg17_upgrade_bin_*.tar.gz supabase/volumes/db/pgsodium_root.key.bak.* supabase/.env.old +/supabase-migration-backup/ +/supabase/volumes/storage.minio-backup/ coverage/ From e0e80e16e43284643c27767afc1ad3082fa524a6 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:03:48 -0600 Subject: [PATCH 03/14] Use the pinned upstream Supabase deployment directly --- .gitignore | 7 +- README.md | 2 +- eslint.config.js | 2 +- supabase/.env.example | 389 ------ supabase/.gitignore | 17 +- supabase/LICENSE | 201 --- supabase/MIGRATION.md | 67 +- supabase/README.md | 61 +- supabase/UPSTREAM | 6 +- supabase/docker-compose.pg15.yml | 18 - supabase/docker-compose.pg17.yml | 19 - supabase/docker-compose.yml | 585 --------- supabase/run.sh | 297 ----- supabase/update.sh | 692 ----------- supabase/upgrades.json | 39 - supabase/utils/add-new-auth-keys.sh | 226 ---- supabase/utils/generate-keys.sh | 138 --- supabase/utils/upgrade-pg17.sh | 822 ------------ supabase/volumes/api/envoy/cds.yaml | 223 ---- .../volumes/api/envoy/docker-entrypoint.sh | 35 - supabase/volumes/api/envoy/envoy.yaml | 27 - supabase/volumes/api/envoy/lds.template.yaml | 1101 ----------------- supabase/volumes/db/_supabase.sql | 3 - supabase/volumes/db/jwt.sql | 3 - supabase/volumes/db/logs.sql | 6 - supabase/volumes/db/pooler.sql | 6 - supabase/volumes/db/realtime.sql | 4 - supabase/volumes/db/roles.sql | 8 - supabase/volumes/db/webhooks.sql | 212 ---- supabase/volumes/functions/deno.jsonc | 6 - supabase/volumes/functions/hello/index.ts | 37 - supabase/volumes/functions/main/index.ts | 232 ---- supabase/volumes/pooler/pooler.exs | 30 - 33 files changed, 99 insertions(+), 5422 deletions(-) delete mode 100644 supabase/.env.example delete mode 100644 supabase/LICENSE delete mode 100644 supabase/docker-compose.pg15.yml delete mode 100644 supabase/docker-compose.pg17.yml delete mode 100644 supabase/docker-compose.yml delete mode 100755 supabase/run.sh delete mode 100755 supabase/update.sh delete mode 100644 supabase/upgrades.json delete mode 100644 supabase/utils/add-new-auth-keys.sh delete mode 100644 supabase/utils/generate-keys.sh delete mode 100644 supabase/utils/upgrade-pg17.sh delete mode 100644 supabase/volumes/api/envoy/cds.yaml delete mode 100755 supabase/volumes/api/envoy/docker-entrypoint.sh delete mode 100644 supabase/volumes/api/envoy/envoy.yaml delete mode 100644 supabase/volumes/api/envoy/lds.template.yaml delete mode 100644 supabase/volumes/db/_supabase.sql delete mode 100644 supabase/volumes/db/jwt.sql delete mode 100644 supabase/volumes/db/logs.sql delete mode 100644 supabase/volumes/db/pooler.sql delete mode 100644 supabase/volumes/db/realtime.sql delete mode 100644 supabase/volumes/db/roles.sql delete mode 100644 supabase/volumes/db/webhooks.sql delete mode 100644 supabase/volumes/functions/deno.jsonc delete mode 100644 supabase/volumes/functions/hello/index.ts delete mode 100644 supabase/volumes/functions/main/index.ts delete mode 100644 supabase/volumes/pooler/pooler.exs diff --git a/.gitignore b/.gitignore index d1e3884e9d..859f88a055 100644 --- a/.gitignore +++ b/.gitignore @@ -33,12 +33,7 @@ docsLibraries/ public/__revisit-widget src/public/__revisit-widget -supabase/volumes/db/data.bak.* -supabase/volumes/db/data_migration/ -supabase/volumes/db/pg17_upgrade_bin_*.tar.gz -supabase/volumes/db/pgsodium_root.key.bak.* -supabase/.env.old +supabase/volumes/ /supabase-migration-backup/ -/supabase/volumes/storage.minio-backup/ coverage/ diff --git a/README.md b/README.md index c24f6f4a14..808b1adbeb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ To run this demo experiment locally, you will need to install Node.js on your co ## Self-hosted Supabase -See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for the bundled, version-pinned Supabase stack. Existing MinIO installations must migrate before starting the new configuration. +See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for a version-pinned upstream Supabase deployment. Existing MinIO installations must migrate before starting the new configuration. ## Adding Tests diff --git a/eslint.config.js b/eslint.config.js index 805e90af95..24866873b8 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -18,7 +18,7 @@ const compat = new FlatCompat({ }); export default [{ - ignores: ['public/**', 'src/public/**', 'supabase/volumes/functions/**'], + ignores: ['public/**', 'src/public/**'], }, ...fixupConfigRules(compat.extends( 'airbnb-base', 'airbnb/rules/react', diff --git a/supabase/.env.example b/supabase/.env.example deleted file mode 100644 index eb130e1cfe..0000000000 --- a/supabase/.env.example +++ /dev/null @@ -1,389 +0,0 @@ -############ -# Docker compose override files to layer on top of docker-compose.yml. -# Native docker compose COMPOSE_FILE: colon-separated list, base file first. -# Manage with: ./run.sh config add|remove -# -# Examples: -# COMPOSE_FILE=docker-compose.yml -# COMPOSE_FILE=docker-compose.yml:docker-compose.pg17.yml -# -############ -COMPOSE_FILE=docker-compose.yml - - -############ -# Secrets -# -# YOU MUST CHANGE ALL THE DEFAULT VALUES BELOW BEFORE STARTING -# THE CONTAINERS FOR THE FIRST TIME! -# -# Documentation: -# https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase -# -# To generate secrets and API keys: -# 1. sh utils/generate-keys.sh -# 2. sh utils/add-new-auth-keys.sh -# -############ - -# Postgres -POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password - -# Legacy symmetric HS256 key -JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long -# Legacy API keys (HS256-signed JWTs) -ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE -SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q - -# Asymmetric key pair (ES256) and opaque API keys -# -# Documentation: -# https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys -# -# To generate: -# sh ./utils/add-new-auth-keys.sh -# -# Opaque API key for client-side use (anon role). -SUPABASE_PUBLISHABLE_KEY= -# Opaque API key for server-side use (service_role). Never expose in client code. -SUPABASE_SECRET_KEY= -# JSON array of signing JWKs (EC private + legacy symmetric). -# Used by Auth. -JWT_KEYS= -# JWKS for token verification (EC public + legacy symmetric). -# Used by PostgREST, Realtime, Storage to verify tokens. -JWT_JWKS= - -# Access to Dashboard -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated - -# Encryption key for securing Realtime and Supavisor communications. -# (Must be at least 64 characters; generate with: openssl rand -base64 48) -SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq - -# Encryption key used by Realtime for sensitive fields in the `_realtime` schema. -# (Must be exactly 16 characters; generate with: `openssl rand -hex 8`) -REALTIME_DB_ENC_KEY=supabaserealtime - -# Encryption key used by Supavisor for storing encrypted configuration. -# (Must be exactly 32 characters; generate with: openssl rand -hex 16) -VAULT_ENC_KEY=your-32-character-encryption-key - -# Encryption key for securing connection strings used by Studio against postgres-meta. -# (Must be at least 32 characters; generate with openssl rand -base64 24) -PG_META_CRYPTO_KEY=your-encryption-key-32-chars-min - -# API token for log ingestion used by Logflare and Vector. -# (Must be at least 32 characters; generate with openssl rand -base64 24) -LOGFLARE_PUBLIC_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-public -# API token used for Logflare management operations. Never expose client-side. -# (Must be at least 32 characters; generate with openssl rand -base64 24) -LOGFLARE_PRIVATE_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-private - -# Access key ID (username-like) for accessing the S3 protocol endpoint in Storage. -# (Generate with: openssl rand -hex 16) -S3_PROTOCOL_ACCESS_KEY_ID=625729a08b95bf1b7ff351a663f3a23c -# Secret key (password-like) used with S3_PROTOCOL_ACCESS_KEY_ID. -# (Generate with: openssl rand -hex 32) -S3_PROTOCOL_ACCESS_KEY_SECRET=850181e4652dd023b7a98c58ae0d2d34bd487ee0cc3254aed6eda37307425907 - - -############ -# URLs - Configure hostnames below to reflect your actual domain name -############ - -# Access to Dashboard and REST API -SUPABASE_PUBLIC_URL=http://localhost:8000 - -# Full external URL of the Auth service, used to construct OAuth callbacks, -# SAML endpoints, and email links -API_EXTERNAL_URL=http://localhost:8000/auth/v1 - -# See also the Auth section below for Site URL and Redirect URLs configuration - - -############ -# Database - Postgres configuration -############ - -# Using default user (postgres) -POSTGRES_HOST=db -POSTGRES_DB=postgres - -# Default configuration includes Supavisor exposing POSTGRES_PORT -# Postgres uses POSTGRES_PORT inside the container -# Documentation: -# https://supabase.com/docs/guides/self-hosting/accessing-postgres -POSTGRES_PORT=5432 - - -############ -# Database pooler -############ - -# Self-hosted Supabase uses Supavisor as the default database pooler. -# If you use the PgBouncer docker-compose override, Supavisor is disabled -# and the pooler settings below are used to configure PgBouncer instead. -# -# Supavisor exposes POSTGRES_PORT and POOLER_PROXY_PORT_TRANSACTION, -# POSTGRES_PORT is used for session mode pooling -# PgBouncer only exposes POOLER_PROXY_PORT_TRANSACTION. -# -# Port to use for transaction mode pooling connections -POOLER_PROXY_PORT_TRANSACTION=6543 - -# Maximum number of PostgreSQL connections Supavisor or PgBouncer opens per pool -POOLER_DEFAULT_POOL_SIZE=20 - -# Maximum number of client connections Supavisor or PgBouncer accepts per pool -POOLER_MAX_CLIENT_CONN=100 - -# Unique Supavisor tenant identifier -# Documentation: -# https://supabase.com/docs/guides/self-hosting/accessing-postgres -POOLER_TENANT_ID=your-tenant-id - -# Pool size for internal metadata storage used by Supavisor -# This is separate from client connections and used only by Supavisor itself -POOLER_DB_POOL_SIZE=5 - - -############ -# Studio - Configuration for the Dashboard -############ - -STUDIO_DEFAULT_ORGANIZATION=Default Organization -STUDIO_DEFAULT_PROJECT=Default Project - -# Add your OpenAI API key to enable AI Assistant -OPENAI_API_KEY=sk-proj-xxxxxxxx - - -############ -# Auth - Configuration for the authentication server -############ - -## General settings - -# Equivalent to "Site URL" and "Redirect URLs" platform configuration options -# Documentation: https://supabase.com/docs/guides/auth/redirect-urls -SITE_URL=http://localhost:3000 -ADDITIONAL_REDIRECT_URLS= - -JWT_EXPIRY=3600 -DISABLE_SIGNUP=false - -## Mailer Config -MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" -MAILER_URLPATHS_INVITE="/auth/v1/verify" -MAILER_URLPATHS_RECOVERY="/auth/v1/verify" -MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" - -## Email auth -ENABLE_EMAIL_SIGNUP=true -ENABLE_EMAIL_AUTOCONFIRM=false -SMTP_ADMIN_EMAIL=admin@example.com -SMTP_HOST=supabase-mail -SMTP_PORT=2500 -SMTP_USER=fake_mail_user -SMTP_PASS=fake_mail_password -SMTP_SENDER_NAME=fake_sender -ENABLE_ANONYMOUS_USERS=false - -## Phone auth -ENABLE_PHONE_SIGNUP=true -ENABLE_PHONE_AUTOCONFIRM=true - -## OAuth / Social login providers - -# Uncomment and fill in the providers you want to enable. -# You must ALSO uncomment the matching GOTRUE_EXTERNAL_* lines in docker-compose.yml -# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-oauth -# GOOGLE_ENABLED=false -# GOOGLE_CLIENT_ID= -# GOOGLE_SECRET= - -# GITHUB_ENABLED=false -# GITHUB_CLIENT_ID= -# GITHUB_SECRET= - -# AZURE_ENABLED=false -# AZURE_CLIENT_ID= -# AZURE_SECRET= - -# Phone / SMS provider configuration -# Uncomment to configure SMS delivery for phone auth and phone MFA. -# You must ALSO uncomment the matching GOTRUE_SMS_* lines in docker-compose.yml -# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa -# SMS_PROVIDER=twilio -# SMS_OTP_EXP=60 -# SMS_OTP_LENGTH=6 -# SMS_MAX_FREQUENCY=60s -# SMS_TEMPLATE=Your code is {{ .Code }} - -# SMS_TWILIO_ACCOUNT_SID= -# SMS_TWILIO_AUTH_TOKEN= -# SMS_TWILIO_MESSAGE_SERVICE_SID= - -# Test OTP: map phone numbers to fixed OTP codes for development -# Format: phone1:code1,phone2:code2 -# SMS_TEST_OTP= - -# Multi-factor authentication (MFA) -# Uncomment to change MFA defaults. -# You must ALSO uncomment the matching GOTRUE_MFA_* lines in docker-compose.yml - -# App Authenticator (TOTP) - enabled by default -# MFA_TOTP_ENROLL_ENABLED=true -# MFA_TOTP_VERIFY_ENABLED=true - -# Phone MFA - disabled by default (opt-in) -# MFA_PHONE_ENROLL_ENABLED=false -# MFA_PHONE_VERIFY_ENABLED=false - -# Maximum MFA factors a user can enroll -# MFA_MAX_ENROLLED_FACTORS=10 - -## SAML SSO - -# You must ALSO uncomment the matching GOTRUE_* lines in docker-compose.yml -# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso - -# SAML_ENABLED=true -# SAML_PRIVATE_KEY= - -# Optional: accept encrypted SAML assertions from IdPs (default: false) -# SAML_ALLOW_ENCRYPTED_ASSERTIONS=false - -# Optional: how long relay state tokens remain valid (default: 2m0s) -# SAML_RELAY_STATE_VALIDITY_PERIOD=2m0s - -# Optional: override the SAML entity ID / ACS base URL -# Defaults to API_EXTERNAL_URL if not set -# SAML_EXTERNAL_URL=https://supabase.example.com:8000/auth/v1 - -# Optional: rate limit on the ACS endpoint (requests per second, default: 15) -# SAML_RATE_LIMIT_ASSERTION=15 - - -############ -# Storage - Configuration for Storage -############ - -# Check the S3_PROTOCOL_ACCESS_KEY_ID/SECRET above, and -# refer to the documentation at: -# https://supabase.com/docs/guides/self-hosting/self-hosted-s3 -# to learn how to configure the S3 protocol endpoint - -# S3 bucket when using S3 backend, directory name when using 'file' -GLOBAL_S3_BUCKET=stub - -# Used for S3 protocol endpoint configuration -REGION=stub - -# Used by MinIO when added via: -# docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d -MINIO_ROOT_USER=supa-storage -# Root administrator password for the RustFS or MinIO server. -# (Must be 8+ characters; generate with: openssl rand -hex 16) -MINIO_ROOT_PASSWORD=secret1234 - -# Equivalent to project_ref as described here: -# https://supabase.com/docs/guides/storage/s3/authentication#session-token -STORAGE_TENANT_ID=stub - - -############ -# Functions - Configuration for Edge functions -############ - -# Documentation: -# https://supabase.com/docs/guides/self-hosting/self-hosted-functions - -# NOTE: VERIFY_JWT applies to all functions -FUNCTIONS_VERIFY_JWT=false - - -############ -# API - Configuration for PostgREST -############ - -# Postgres schemas exposed via the REST API -PGRST_DB_SCHEMAS=public,graphql_public - -# Max number of rows returned by a request -PGRST_DB_MAX_ROWS=1000 - -# Extra schemas added to the search_path of every request -PGRST_DB_EXTRA_SEARCH_PATH=public - - -############ -# Logs and Analytics -############ - -## Vector log collection and routing - -# Docker socket location - required for proper Vector operation -DOCKER_SOCKET_LOCATION=/var/run/docker.sock -# For Podman use the following: -# DOCKER_SOCKET_LOCATION=/run/podman/podman.sock - -## Analytics (Logflare) - -# Check the LOGFLARE_* access token configuration _above_. -# If Logflare has to be externally exposed - configure securely! - -# Google Cloud Project details -# Documentation: -# https://supabase.com/docs/reference/self-hosting-analytics/introduction -GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID -GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER - - -############ -# API gateway -############ - -# Host port the API gateway (Envoy by default) listens on. -API_GW_HTTP_PORT=8000 - -# Kong gateway override only (sh run.sh config add kong). KONG_HTTPS_PORT is -# Kong's built-in HTTPS listener; KONG_HTTP_PORT is kept as a fallback for -# API_GW_HTTP_PORT so existing .env files continue to work. -KONG_HTTP_PORT=8000 -KONG_HTTPS_PORT=8443 - -# Used internally by the API gateway - DO NOT use in any client or server code. -# Pre-signed ES256 JWT "API key" for anon role. -ANON_KEY_ASYMMETRIC= -# Pre-signed ES256 JWT "API key" for service_role. -SERVICE_ROLE_KEY_ASYMMETRIC= - - -############ -# imgproxy -############ - -# Enable webp support -IMGPROXY_AUTO_WEBP=true - - -############ -# TLS Proxy - Optional Caddy or Nginx reverse proxy with Let's Encrypt -############ - -# Documentation: -# https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https - -# Usage: -# docker compose -f docker-compose.yml -f docker-compose.caddy.yml up -d -# docker compose -f docker-compose.yml -f docker-compose.nginx.yml up -d - -# Domain name for the proxy (must point to your server) -PROXY_DOMAIN=your-domain.example.com - -# Email for Let's Encrypt certificate notifications (nginx only, Caddy uses PROXY_DOMAIN). -# This should be a valid email, not a placeholder (otherwise Certbot may fail to start). -CERTBOT_EMAIL=admin@example.com diff --git a/supabase/.gitignore b/supabase/.gitignore index 1f9c343619..fb726cc7ad 100644 --- a/supabase/.gitignore +++ b/supabase/.gitignore @@ -1,16 +1,5 @@ -volumes/db/data -volumes/storage -volumes/snippets -volumes/functions/** -!volumes/functions/deno.json* -!volumes/functions/main/ -volumes/functions/main/** -!volumes/functions/main/index.ts -!volumes/functions/hello/ -volumes/functions/hello/** -!volumes/functions/hello/index.ts .env -test.http -docker-compose.override.yml +.env.old +volumes/ .supabase-version -backups +backups/ diff --git a/supabase/LICENSE b/supabase/LICENSE deleted file mode 100644 index 49ea124025..0000000000 --- a/supabase/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2024 Supabase - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 3e25f34dac..e0ac0a54c4 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -12,7 +12,7 @@ and policies remain in Postgres. `migrate-storage.py` exports object bytes throu the old API and installs their versioned files with HTTP metadata in the new backend. It never writes database records or re-uploads objects through the API. -Requirements: a Linux server, Docker Compose, Bash, curl, OpenSSL, Python 3.11+, +Requirements: a Linux server, Git, Docker Compose, Bash, curl, OpenSSL, Python 3.11+, root access for the upstream database upgrade, and enough space for the database backup/upgrade and two copies of object data. Pre-pull the target images before starting the upgrade; allow at least 15 GB additional temporary space for the @@ -32,23 +32,27 @@ Restrict application traffic and pause participant/designer/background writes for the remainder of the procedure. Keep Supabase accessible to the migration operator until export finishes. -Copy `migrate-storage.py` from the new checkout to the **old deployment directory**. +Obtain the new reVISit checkout separately; do not update the old deployment +in place before taking backups, because this change removes its bundled +configuration files. Copy `migrate-storage.py` from the new checkout to the +**old deployment directory**. From that old directory, create a private backup outside the deployment: ```sh umask 077 -mkdir ../supabase-migration-backup -cp .env docker-compose.yml ../supabase-migration-backup/ -cp -a volumes/api ../supabase-migration-backup/api +migration_backup="$(cd .. && pwd)/supabase-migration-backup" +mkdir "$migration_backup" +cp .env docker-compose.yml "$migration_backup/" +cp -a volumes/api "$migration_backup/api" docker image save minio/minio minio/mc \ - -o ../supabase-migration-backup/minio-images.tar + -o "$migration_backup/minio-images.tar" docker exec supabase-db pg_dumpall -U supabase_admin \ - > ../supabase-migration-backup/database.sql + > "$migration_backup/database.sql" db_config_volume=$(docker inspect supabase-db --format \ '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}') test -n "$db_config_volume" docker run --rm -v "$db_config_volume:/source:ro" \ - -v "$(cd ../supabase-migration-backup && pwd):/backup" \ + -v "$migration_backup:/backup" \ alpine:3.22 tar -cf /backup/db-config.tar -C /source . ``` @@ -58,7 +62,7 @@ SUPABASE_SERVICE_ROLE_KEY`; paste the key when prompted). Do not put the key in shell history or in the application's `VITE_*` variables. ```sh -python3 migrate-storage.py export ../supabase-migration-backup/objects \ +python3 migrate-storage.py export "$migration_backup/objects" \ --url http://localhost:8000 ``` @@ -72,8 +76,8 @@ Stop the old stack **using its old Compose file**, without deleting volumes: ```sh docker compose down -sudo cp -a volumes/db/data ../supabase-migration-backup/postgres15-data -sudo cp -a volumes/storage ../supabase-migration-backup/minio-data +sudo cp -a volumes/db/data "$migration_backup/postgres15-data" +sudo cp -a volumes/storage "$migration_backup/minio-data" ``` Save the other deployment-specific configuration files as needed. Verify backup @@ -82,18 +86,41 @@ backups must belong to the same quiesced deployment. ## 2. Install the new configuration; preserve existing secrets -Copy the new checkout's `supabase/` configuration, scripts, and supporting files -into the deployment directory, excluding runtime data and `.env`. For example, -from the old deployment directory, with rsync installed: +If this deployment is inside the old reVISit checkout, move the stopped directory +outside it after verifying the backups. Choose a destination that does not exist: ```sh +deployment=/absolute/path/to/supabase-project +test ! -e "$deployment" +sudo mv "$PWD" "$deployment" +cd "$deployment" +``` + +Skip the move if the deployment is already separate. Keep the same shell session +so `migration_backup` continues to refer to the original absolute backup path. +Application checkout updates must not replace deployment configuration or data. + +Fetch the pinned official bundle into a separate source directory. From the +old deployment directory, replace the support path with your new reVISit checkout: + +```sh +revisit_support=/absolute/path/to/new-checkout/supabase +git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ + https://github.com/supabase/supabase.git ../supabase-upstream +git -C ../supabase-upstream sparse-checkout set docker +test "$(git -C ../supabase-upstream rev-parse HEAD)" = \ + 564eab8ad7840b13324f68b1bfac074ef8d51c21 rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ --exclude='volumes/db/data*' --exclude='volumes/storage*' \ - /path/to/new-checkout/supabase/ ./ + ../supabase-upstream/docker/ ./ +cp "$revisit_support/docker-compose.github.yml" . ``` -Use the actual path to the new checkout. Do not add `--delete`. Preserve the -existing Compose project name and `db-config` named volume. +Do not add `--delete`. The command replaces files supplied by upstream; review +any local edits to those files before copying and carry them over deliberately. +Preserve the existing Compose project name and `db-config` named volume. +The upstream scripts now live in this deployment; +reVISit does not maintain copies of them. Merge new `.env.example` settings into your existing private `.env`. Keep the existing database password, `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY`, and @@ -149,7 +176,7 @@ directory so its relative paths still refer to the original data: ```sh docker compose --project-directory "$PWD" \ - -f ../supabase-migration-backup/docker-compose.yml up -d --wait db + -f "$migration_backup/docker-compose.yml" up -d --wait db docker exec supabase-db pg_isready -U postgres # Our previous Compose did not initialize the upstream Realtime schema. docker exec -i supabase-db psql -U supabase_admin -d postgres \ @@ -186,7 +213,7 @@ Keep application access blocked. Stop Storage and install exported files: ```sh docker compose stop storage -sudo python3 migrate-storage.py install ../supabase-migration-backup/objects \ +sudo python3 migrate-storage.py install "$migration_backup/objects" \ --destination volumes/storage --bucket stub --tenant stub docker compose up -d storage sh run.sh start @@ -200,7 +227,7 @@ It can be rerun with the same verified export after fixing an installation error ## 4. Verify before reopening writes ```sh -python3 migrate-storage.py verify ../supabase-migration-backup/objects \ +python3 migrate-storage.py verify "$migration_backup/objects" \ --url http://localhost:8000 sh run.sh status docker exec supabase-db psql -U supabase_admin -d postgres \ diff --git a/supabase/README.md b/supabase/README.md index f9eb89c043..08736b3f5c 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -1,14 +1,15 @@ # Self-hosted Supabase for reVISit -This directory bundles the default stack from **Supabase self-hosted/v0.8.2** -(see [UPSTREAM](UPSTREAM)). The Compose file and upstream configuration/scripts -are unchanged. Supabase Storage uses persistent local files; MinIO is not required. +This directory contains the reVISit-specific setup and migration files. Obtain +the deployment bundle directly from **Supabase self-hosted/v0.8.2** (see +[UPSTREAM](UPSTREAM)), following the upstream manual installation below. Its +default Storage backend uses persistent local files; MinIO is not required. For an existing deployment, follow [MIGRATION.md](MIGRATION.md) **before starting this configuration**. Hosted Supabase deployments do not need this migration. ## First-time setup -Requirements: a Linux host with Docker Engine and Docker Compose v2 or newer, OpenSSL, +Requirements: a Linux host with Git, Docker Engine, Docker Compose v2 or newer, OpenSSL, and Node.js 16+ (the upstream key helper can use Docker instead). For server resources and operating-system details, follow the [upstream installation guide](https://supabase.com/docs/guides/self-hosting/docker). @@ -19,10 +20,29 @@ Studio and postgres-meta images also crashed on our Apple Silicon Docker Desktop host; their amd64 variants worked under emulation. Verify image startup on your actual host before a migration. -Run the commands below from this directory. The upstream stack uses fixed -container names: run only one copy per Docker host, or deliberately isolate it. +Keep the deployment outside the reVISit checkout so application updates do not +replace its configuration or data. The upstream stack uses fixed container +names: run only one copy per Docker host, or deliberately isolate it. -1. Create your private configuration and generate fresh secrets: +1. From the parent directory where you want your new deployment, obtain the + pinned upstream bundle. Replace `revisit_support` with the absolute path to + this directory in your reVISit checkout: + + ```sh + revisit_support=/absolute/path/to/revisit-study/supabase + git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ + https://github.com/supabase/supabase.git supabase-upstream + git -C supabase-upstream sparse-checkout set docker + test "$(git -C supabase-upstream rev-parse HEAD)" = \ + 564eab8ad7840b13324f68b1bfac074ef8d51c21 + mkdir supabase-project + cp -a supabase-upstream/docker/. supabase-project/ + cd supabase-project + ``` + + These are upstream's manual installation steps with a sparse checkout and + commit check. Subsequent deployment commands run from `supabase-project`. + Create your private configuration and generate fresh secrets: ```sh cp .env.example .env @@ -44,8 +64,14 @@ container names: run only one copy per Docker host, or deliberately isolate it. - Review email confirmation and SMTP settings for Study Designer accounts. For GitHub OAuth, append `ENABLE_GITHUB_OAUTH=true`, `GITHUB_OAUTH_CLIENT_ID`, `GITHUB_OAUTH_SECRET`, and - `GITHUB_OAUTH_REDIRECT_URI` to `.env`, then run - `sh run.sh config add github`. The redirect URI is + `GITHUB_OAUTH_REDIRECT_URI` to `.env`, then copy and enable our override: + + ```sh + cp "$revisit_support/docker-compose.github.yml" . + sh run.sh config add github + ``` + + The redirect URI is `/auth/v1/callback`; register it in the GitHub OAuth app. See [upstream OAuth guidance](https://supabase.com/docs/guides/self-hosting/auth). - Keep `STORAGE_TENANT_ID=stub`, `GLOBAL_S3_BUCKET=stub`, and `REGION=stub` @@ -69,7 +95,7 @@ container names: run only one copy per Docker host, or deliberately isolate it. ```sh docker exec -i supabase-db psql -U supabase_admin -d postgres \ - -v ON_ERROR_STOP=1 < revisit.sql + -v ON_ERROR_STOP=1 < "$revisit_support/revisit.sql" ``` This applies the same schema and access model as the existing @@ -111,20 +137,21 @@ For deployments requiring object storage, use Use `sh run.sh stop`, `start`, `status`, and `logs` to operate the stack. Review future tagged updates using `sh update.sh --dry-run --to ` and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). -This repository bundles the default stack and database upgrade support; -upstream updates can introduce additional files or changes to setup instructions. -Review the plan and reVISit-specific files before applying an update. +The update tool belongs to the downloaded upstream bundle. Review its plan and +your GitHub override before applying an update; this guide and migration helper +target the pinned release above. ## Local regression checks -The migration helper requires Python 3.11+ and no third-party Python packages: +Run these checks from your reVISit checkout. The migration helper requires +Python 3.11+ and no third-party Python packages: ```sh -python3 tests/migrate-storage.spec.py +python3 supabase/tests/migrate-storage.spec.py ``` -The opt-in Chromium test is `../tests/supabase-live.spec.ts`. It needs an isolated -running stack, initialized `revisit.sql`, and the application's Supabase +The opt-in Chromium test is `tests/supabase-live.spec.ts`. It needs an isolated +running stack, initialized `supabase/revisit.sql`, and the application's Supabase environment variables. Set `PW_SUPABASE_LIVE=1` and `PW_SUPABASE_SERVICE_ROLE_KEY` privately, then run the focused suite from the repository root with `yarn exec playwright test tests/supabase-live.spec.ts --project chromium`. diff --git a/supabase/UPSTREAM b/supabase/UPSTREAM index 9e43450ea2..9f3ba99505 100644 --- a/supabase/UPSTREAM +++ b/supabase/UPSTREAM @@ -2,7 +2,5 @@ Supabase self-hosted/v0.8.2 Commit: 564eab8ad7840b13324f68b1bfac074ef8d51c21 Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.2/docker -Compose, configuration, and upstream scripts are copied without edits. -Only the default stack and PostgreSQL migration support are bundled. -README.md, MIGRATION.md, revisit.sql, migrate-storage.py, tests/, and -docker-compose.github.yml are reVISit additions. +The deployment configuration and scripts are obtained directly from upstream. +This directory contains only reVISit setup, migration, and integration files. diff --git a/supabase/docker-compose.pg15.yml b/supabase/docker-compose.pg15.yml deleted file mode 100644 index f87fe8c482..0000000000 --- a/supabase/docker-compose.pg15.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Postgres 15 override for self-hosted Supabase. -# -# Postgres 17 is now the default in docker-compose.yml. Use this override to pin -# Postgres 15, for two cases: -# 1. An existing Postgres 15 deployment that has not been upgraded yet. -# 2. As the rollback target for utils/upgrade-pg17.sh (PG 15 and PG 17 use -# different postgres UIDs, so rollback must start with the PG 15 image). -# -# Usage: -# docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d -# -# To upgrade an existing Postgres 15 database to Postgres 17 in place, see: -# - https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17 -# - utils/upgrade-pg17.sh - -services: - db: - image: supabase/postgres:15.8.1.085 diff --git a/supabase/docker-compose.pg17.yml b/supabase/docker-compose.pg17.yml deleted file mode 100644 index a8e37117e1..0000000000 --- a/supabase/docker-compose.pg17.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Postgres 17 override for self-hosted Supabase. -# -# NOTE: Postgres 17 is now the default in docker-compose.yml, so this override -# is redundant for new installs (kept for explicitness / backwards compatibility). -# Keep the image tag here in sync with the db image in docker-compose.yml. -# -# Usage (new install or after upgrade): -# docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d -# -# For upgrading an existing Postgres 15 database, run utils/upgrade-pg17.sh first. -# -# Note: PG 15 and PG 17 use different postgres UIDs. If starting fresh with a -# leftover db-config volume from PG 15, you may see -# "FATAL: invalid secret key". Either remove the old volume or fix ownership. -# See: docs/guides/self-hosting/postgres-upgrade-17 - -services: - db: - image: supabase/postgres:17.6.1.136 diff --git a/supabase/docker-compose.yml b/supabase/docker-compose.yml deleted file mode 100644 index a37d39dc7c..0000000000 --- a/supabase/docker-compose.yml +++ /dev/null @@ -1,585 +0,0 @@ -# Usage -# Start: docker compose up -d -# Stop: docker compose down -# Dev mode: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml up -d -# Reset everything: sh reset.sh -# -# Notes: -# - Nested variable interpolation (${A:-${B}}) requires podman-compose >= 1.6.0 -# - -name: supabase - -services: - - studio: - container_name: supabase-studio - image: supabase/studio:2026.09.07-sha-7996410 - restart: unless-stopped - healthcheck: - test: - [ - "CMD-SHELL", - "node -e \"fetch('http://localhost:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})\"" - ] - timeout: 10s - interval: 5s - retries: 3 - start_period: 20s - environment: - # Listen on all IPv4 interfaces - HOSTNAME: "0.0.0.0" - - STUDIO_PG_META_URL: http://meta:8080 - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - - # See: https://supabase.com/docs/guides/self-hosting/remove-superuser-access - POSTGRES_USER_READ_WRITE: postgres - - PG_META_CRYPTO_KEY: ${PG_META_CRYPTO_KEY} - PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} - PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} - PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} - - DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} - DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} - OPENAI_API_KEY: ${OPENAI_API_KEY} - - SUPABASE_URL: http://api-gw:8000 - SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} - AUTH_JWT_SECRET: ${JWT_SECRET} - SUPABASE_PUBLISHABLE_KEY: ${SUPABASE_PUBLISHABLE_KEY} - SUPABASE_SECRET_KEY: ${SUPABASE_SECRET_KEY} - - # See: docker-compose.logs.yml - ENABLED_FEATURES_LOGS_ALL: "false" - - SNIPPETS_MANAGEMENT_FOLDER: /app/snippets - EDGE_FUNCTIONS_MANAGEMENT_FOLDER: /app/edge-functions - volumes: - - ./volumes/snippets:/app/snippets:z - - ./volumes/functions:/app/edge-functions:ro,z - - # Envoy is the default API gateway - # See: https://github.com/orgs/supabase/discussions/48048 - api-gw: - container_name: supabase-envoy - image: envoyproxy/envoy:v1.39.1 - restart: unless-stopped - networks: - default: - # Expose `envoy` and `kong` as network aliases, so internal configs - # that reference either hostname resolve to whichever gateway is active. - aliases: - - envoy - - kong - healthcheck: - # Using a TCP port check because this image does not include curl or wget. - test: ["CMD-SHELL", "timeout 1 bash -c '" - - # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_ENABLED: "true" - # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/mfa_verification_attempt" - - # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_ENABLED: "true" - # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/password_verification_attempt" - - # GOTRUE_HOOK_SEND_SMS_ENABLED: "false" - # GOTRUE_HOOK_SEND_SMS_URI: "pg-functions://postgres/public/custom_access_token_hook" - # GOTRUE_HOOK_SEND_SMS_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" - - # GOTRUE_HOOK_SEND_EMAIL_ENABLED: "false" - # GOTRUE_HOOK_SEND_EMAIL_URI: "http://host.docker.internal:54321/functions/v1/email_sender" - # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" - - rest: - container_name: supabase-rest - image: postgrest/postgrest:v14.17 - restart: unless-stopped - depends_on: - db: - # Disable this if you are using an external Postgres database - condition: service_healthy - healthcheck: - test: [ "CMD", "postgrest", "--ready" ] - interval: 5s - timeout: 5s - retries: 3 - environment: - PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} - PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} - PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} - PGRST_DB_ANON_ROLE: anon - - PGRST_ADMIN_SERVER_PORT: 3001 - PGRST_ADMIN_SERVER_HOST: localhost - - # PostgREST accepts a plain-text symmetric secret, a single JWK, or a JWKS. - # For Podman, use either PGRST_JWT_SECRET: ${JWT_SECRET} or - # PGRST_JWT_SECRET: ${JWT_JWKS} - PGRST_JWT_SECRET: ${JWT_JWKS:-${JWT_SECRET}} - - PGRST_DB_USE_LEGACY_GUCS: "false" - PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} - command: - [ - "postgrest" - ] - - realtime: - # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain - container_name: realtime-dev.supabase-realtime - image: supabase/realtime:v2.134.10 - restart: unless-stopped - depends_on: - db: - # Disable this if you are using an external Postgres database - condition: service_healthy - healthcheck: - test: - [ - "CMD-SHELL", - "curl -sSfL --head -o /dev/null -H \"Authorization: Bearer ${ANON_KEY}\" http://localhost:4000/api/tenants/realtime-dev/health" - ] - timeout: 5s - interval: 30s - retries: 3 - start_period: 10s - environment: - PORT: 4000 - DB_HOST: ${POSTGRES_HOST} - DB_PORT: ${POSTGRES_PORT} - DB_USER: supabase_admin - DB_PASSWORD: ${POSTGRES_PASSWORD} - DB_NAME: ${POSTGRES_DB} - DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' - DB_ENC_KEY: ${REALTIME_DB_ENC_KEY:-supabaserealtime} - - # Legacy symmetric HS256 key - API_JWT_SECRET: ${JWT_SECRET} - - # JWKS for token verification (EC public + legacy symmetric). - # For Podman, use: API_JWT_JWKS: ${JWT_JWKS} - #API_JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} - - SECRET_KEY_BASE: ${SECRET_KEY_BASE} - METRICS_JWT_SECRET: ${JWT_SECRET} - ERL_AFLAGS: -proto_dist inet_tcp - DNS_NODES: "''" - RLIMIT_NOFILE: "10000" - APP_NAME: realtime - SEED_SELF_HOST: "true" - RUN_JANITOR: "true" - DISABLE_HEALTHCHECK_LOGGING: "true" - - # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up - storage: - container_name: supabase-storage - image: supabase/storage-api:v1.74.0 - restart: unless-stopped - depends_on: - db: - # Disable this if you are using an external Postgres database - condition: service_healthy - rest: - condition: service_started - imgproxy: - condition: service_started - healthcheck: - test: - [ - "CMD", - "wget", - "--no-verbose", - "--tries=1", - "--spider", - "http://storage:5000/status" - ] - timeout: 5s - interval: 5s - retries: 3 - start_period: 10s - environment: - ANON_KEY: ${ANON_KEY} - SERVICE_KEY: ${SERVICE_ROLE_KEY} - POSTGREST_URL: http://rest:3000 - - # Legacy symmetric HS256 key - AUTH_JWT_SECRET: ${JWT_SECRET} - - # JWKS for token verification (EC public + legacy symmetric). - # For Podman, use: JWT_JWKS: ${JWT_JWKS} - #JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} - - DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - STORAGE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} - REQUEST_ALLOW_X_FORWARDED_PATH: "true" - FILE_SIZE_LIMIT: 52428800 - STORAGE_BACKEND: file - # S3 bucket when using S3 backend, directory name when using 'file' - GLOBAL_S3_BUCKET: ${GLOBAL_S3_BUCKET} - # S3 Backend configuration - #GLOBAL_S3_ENDPOINT: https://your-s3-endpoint - #GLOBAL_S3_PROTOCOL: https - #GLOBAL_S3_FORCE_PATH_STYLE: "true" - #AWS_ACCESS_KEY_ID: your-access-key-id - #AWS_SECRET_ACCESS_KEY: your-secret-access-key - FILE_STORAGE_BACKEND_PATH: /var/lib/storage - TENANT_ID: ${STORAGE_TENANT_ID} - # TODO: https://github.com/supabase/storage-api/issues/55 - REGION: ${REGION} - ENABLE_IMAGE_TRANSFORMATION: "true" - IMGPROXY_URL: http://imgproxy:5001 - # S3 protocol endpoint configuration - S3_PROTOCOL_ACCESS_KEY_ID: ${S3_PROTOCOL_ACCESS_KEY_ID} - S3_PROTOCOL_ACCESS_KEY_SECRET: ${S3_PROTOCOL_ACCESS_KEY_SECRET} - volumes: - - ./volumes/storage:/var/lib/storage:z - - imgproxy: - container_name: supabase-imgproxy - image: darthsim/imgproxy:v3.31.4 - restart: unless-stopped - volumes: - - ./volumes/storage:/var/lib/storage:z - healthcheck: - test: - [ - "CMD", - "imgproxy", - "health" - ] - timeout: 5s - interval: 5s - retries: 3 - environment: - IMGPROXY_BIND: ":5001" - IMGPROXY_LOCAL_FILESYSTEM_ROOT: / - IMGPROXY_USE_ETAG: "true" - IMGPROXY_AUTO_WEBP: ${IMGPROXY_AUTO_WEBP} - IMGPROXY_MAX_SRC_RESOLUTION: 16.8 - - meta: - container_name: supabase-meta - image: supabase/postgres-meta:v0.99.0 - restart: unless-stopped - depends_on: - db: - # Disable this if you are using an external Postgres database - condition: service_healthy - environment: - PG_META_PORT: 8080 - PG_META_DB_HOST: ${POSTGRES_HOST} - PG_META_DB_PORT: ${POSTGRES_PORT} - PG_META_DB_NAME: ${POSTGRES_DB} - PG_META_DB_USER: postgres - PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} - CRYPTO_KEY: ${PG_META_CRYPTO_KEY} - - functions: - container_name: supabase-edge-functions - image: supabase/edge-runtime:v1.76.2 - restart: unless-stopped - volumes: - - ./volumes/functions:/home/deno/functions:z - - deno-cache:/root/.cache/deno - depends_on: - api-gw: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "timeout 1 bash -c ' | config remove -# (accepts either a short name like 'pg17' or 'docker-compose.pg17.yml') -# -# Usage: -# sh run.sh start # docker compose up -d --wait -# sh run.sh stop # docker compose down -# sh run.sh restart [service] # restart the stack (or named services) -# sh run.sh restart --except ... # restart all services except the named ones -# sh run.sh recreate [service] # stop then start (or force-recreate one service) -# sh run.sh recreate --except ... # force-recreate all services except the named ones -# sh run.sh status # docker compose ps -# sh run.sh logs [service] # follow logs (all or one service) -# sh run.sh inspect # docker inspect on a service's container -# sh run.sh printenv # print a service's environment variables -# sh run.sh pull # pull images -# sh run.sh config # show the active COMPOSE_FILE list -# sh run.sh config add # add an override to COMPOSE_FILE in .env -# sh run.sh config remove # remove an override from COMPOSE_FILE in .env -# sh run.sh compose-config # dump fully-resolved docker compose config -# sh run.sh secrets # print key passwords and API keys from .env -# - -set -e - -cd "$(dirname "$0")" - -if [ ! -f docker-compose.yml ]; then - echo "ERROR: docker-compose.yml not found in $(pwd)" >&2 - exit 1 -fi - -# Normalize an override argument: -# pg17 -> docker-compose.pg17.yml -# docker-compose.pg17.yml -> docker-compose.pg17.yml -# ./docker-compose.pg17.yml -> docker-compose.pg17.yml -# docker-compose.yml -> error (base file, always implicit) -normalize_override() { - arg="${1#./}" - case "$arg" in - docker-compose.yml) - echo "ERROR: docker-compose.yml is the base file, always included" >&2 - return 1 - ;; - docker-compose.*.yml) - echo "$arg" - ;; - *) - echo "docker-compose.${arg}.yml" - ;; - esac -} - -# Read COMPOSE_FILE from .env (stripping quotes and CR). -read_compose_file() { - [ -f .env ] || return 0 - grep '^COMPOSE_FILE=' .env | head -n1 | cut -d= -f2- | tr -d "\r\"'" -} - -# Pretty-print the effective compose file list. -print_config() { - val="$1" - [ -z "$val" ] && val="docker-compose.yml" - echo "COMPOSE_FILE=$val" - echo "compose files:" - OLD_IFS=$IFS - IFS=: - for f in $val; do - echo " $f" - done - IFS=$OLD_IFS - echo "" -} - -# Update or append COMPOSE_FILE in .env. -write_compose_file() { - new_value="$1" - if [ ! -f .env ]; then - echo "ERROR: .env not found in $(pwd)" >&2 - exit 1 - fi - new_line="COMPOSE_FILE=$new_value" - if grep -q '^COMPOSE_FILE=' .env; then - sed -i.old -e "s|^COMPOSE_FILE=.*$|$new_line|" .env - rm -f .env.old - else - cat >> .env <. -# -# Examples: -# COMPOSE_FILE=docker-compose.yml -# COMPOSE_FILE=docker-compose.yml:docker-compose.pg17.yml -############ -$new_line -EOF - fi -} - -# Echoes the list of services (one per line) minus those passed as args. -# Warns on unknown names; returns 1 if no services remain. -services_except() { - all_services=$(docker compose config --services) - filtered="$all_services" - for ex in "$@"; do - echo "$all_services" | grep -qFx "$ex" \ - || echo "Warning: '$ex' is not a service in this project" >&2 - filtered=$(echo "$filtered" | grep -vFx "$ex" || true) - done - if [ -z "$filtered" ]; then - echo "No services left after applying --except" >&2 - return 1 - fi - printf '%s\n' "$filtered" -} - -CMD="${1:-help}" -[ "$#" -gt 0 ] && shift - -case "$CMD" in - start|up) - exec docker compose up -d --wait "$@" - ;; - stop|down) - exec docker compose down "$@" - ;; - restart) - if [ "${1:-}" = "--except" ]; then - shift - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") restart --except ..." >&2; exit 1; } - services=$(services_except "$@") || exit 1 - # shellcheck disable=SC2086 - exec docker compose restart $services - fi - exec docker compose restart "$@" - ;; - recreate) - if [ "${1:-}" = "--except" ]; then - shift - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") recreate --except ..." >&2; exit 1; } - services=$(services_except "$@") || exit 1 - # shellcheck disable=SC2086 - exec docker compose up -d --wait --force-recreate --no-deps $services - fi - if [ $# -eq 0 ]; then - docker compose down - exec docker compose up -d --wait - fi - # Single-service recreate: force-recreate the named services only, - # leave their dependencies running. - exec docker compose up -d --wait --force-recreate --no-deps "$@" - ;; - status|ps) - exec docker compose ps "$@" - ;; - logs) - exec docker compose logs -f "$@" - ;; - inspect) - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") inspect [docker-inspect-args]" >&2; exit 1; } - svc="$1"; shift - cid=$(docker compose ps -q "$svc") - [ -z "$cid" ] && { echo "Service '$svc' is not running" >&2; exit 1; } - exec docker inspect "$cid" "$@" - ;; - printenv) - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") printenv " >&2; exit 1; } - svc="$1" - cid=$(docker compose ps -q "$svc") - [ -z "$cid" ] && { echo "Service '$svc' is not running" >&2; exit 1; } - exec docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$cid" - ;; - pull) - exec docker compose pull "$@" - ;; - compose-config) - exec docker compose config "$@" - ;; - config) - sub="${1:-show}" - [ "$#" -gt 0 ] && shift - current=$(read_compose_file) - case "$sub" in - show) - print_config "$current" - ;; - add) - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") config add ..." >&2; exit 1; } - new_value="${current:-docker-compose.yml}" - changed=false - for arg in "$@"; do - file=$(normalize_override "$arg") || exit 1 - if [ ! -f "$file" ]; then - echo "ERROR: $file not found" >&2 - exit 1 - fi - case ":$new_value:" in - *":$file:"*) echo "Already present: $file" ;; - *) new_value="$new_value:$file"; changed=true ;; - esac - done - [ "$changed" = true ] && write_compose_file "$new_value" - print_config "$new_value" - ;; - remove|rm) - [ $# -eq 0 ] && { echo "Usage: $(basename "$0") config remove ..." >&2; exit 1; } - new_value="${current:-docker-compose.yml}" - changed=false - for arg in "$@"; do - file=$(normalize_override "$arg") || exit 1 - case ":$new_value:" in - *":$file:"*) - # Drop $file by rebuilding the colon list - tmp="" - OLD_IFS=$IFS - IFS=: - for tok in $new_value; do - [ "$tok" = "$file" ] || tmp="${tmp:+$tmp:}$tok" - done - IFS=$OLD_IFS - new_value="$tmp" - changed=true - ;; - *) echo "Not present: $file" ;; - esac - done - [ "$changed" = true ] && write_compose_file "$new_value" - print_config "$new_value" - ;; - *) - echo "Unknown config subcommand: $sub" >&2 - echo "Use: config | config add ... | config remove ..." >&2 - exit 1 - ;; - esac - ;; - secrets) - if [ ! -f .env ]; then - echo "ERROR: .env not found in $(pwd)" >&2 - exit 1 - fi - for var in POSTGRES_PASSWORD DASHBOARD_PASSWORD \ - SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY \ - S3_PROTOCOL_ACCESS_KEY_ID S3_PROTOCOL_ACCESS_KEY_SECRET; do - line=$(grep "^${var}=" .env | head -n1) - if [ -n "$line" ]; then - echo "$line" - else - echo "${var}=" - fi - done - echo "" - ;; - help|-h|--help) - cat < - -Commands: - start Start the stack (docker compose up -d --wait) - stop Stop the stack (docker compose down) - restart [service] Restart the stack (or named services) - restart --except ... - Restart all services except the named ones - recreate [service] Stop then start, or force-recreate one service (--no-deps) - recreate --except ... - Force-recreate all services except the named ones (--no-deps) - status Show service status - logs [service] Follow logs (optionally for a single service) - inspect Inspect a service's container (forwards extra args to docker inspect) - printenv Print a service's environment variables (one per line) - pull Pull all images - config Show the active COMPOSE_FILE list - config add Add an override to COMPOSE_FILE in .env (short name or full filename) - config remove Remove an override from COMPOSE_FILE in .env - compose-config Dump the fully-resolved docker compose config - secrets Show key passwords and API keys from .env - -EOF - ;; - *) - echo "Unknown command: $CMD" >&2 - echo "Run '$0 help' for usage." >&2 - exit 1 - ;; -esac diff --git a/supabase/update.sh b/supabase/update.sh deleted file mode 100755 index cf3313b639..0000000000 --- a/supabase/update.sh +++ /dev/null @@ -1,692 +0,0 @@ -#!/bin/sh -# -# Update an existing self-hosted Supabase deployment in place. -# -# The deployment directory mixes vendor-owned files (docker-compose.yml, the -# override files, volumes/*, scripts, .env.example) with user-owned state -# (.env, docker-compose.override.yml, volumes/db/data, volumes/storage, etc.). -# This script pulls a newer version of the Supabase files on top of yours -# using a 3-way merge against the version you started from, so local edits -# survive and genuine conflicts are surfaced rather than silently overwritten. -# -# The version you started from is recorded in .supabase-version (written by -# setup.sh). If it is missing, pass --from or follow the printed guidance. -# -# What it never touches: .env values you set, docker-compose.override.yml, and -# the data directories (volumes/db/data, volumes/storage, etc.). New keys from -# .env.example are appended to your .env; existing values are kept as-is. -# -# By default it updates to the latest self-hosted/v* release tag (or 'master' -# until the first tag exists). Pass --to to pin a specific tag/branch. -# -# Usage: -# sh update.sh # update to the latest release tag -# sh update.sh --dry-run # show what would change, write nothing -# sh update.sh --to # update to a specific tag/branch -# sh update.sh --from # base to merge from (if no .supabase-version) -# sh update.sh --yes # don't prompt, even on breaking changes -# -# Env: -# SUPABASE_REPO_URL Override the upstream repo (default: github supabase/supabase) -# -# Documentation: https://supabase.com/docs/guides/self-hosting/updating -# - -set -e - -cd "$(dirname "$0")" - -# Pipeline (see main at the bottom): -# resolve refs -> fetch base+target snapshots -> [report-only exit] -# -> build manifest gate -> confirm_gate (before any writes) -# -> backup → merge vendor files + .env keys -> summary → stamp -# -# Three trees for every vendor file path: -# - base - upstream at BASE_REF (.supabase-version ref=, or --from) -# - target - upstream at TARGET_REF (--to, or latest self-hosted/v* tag) -# - user's - the deployment directory (cwd); _not_ a git checkout -# -# Git is only used to fetch snapshots (fetch_snapshot) and to run git merge-file. -# -# Breaking-change gate (upgrades.json on the target snapshot): -# - Keyed by version (e.g. "0.7.0"); window = entries in (BASE_VER, TARGET_VER], -# where the bounds are the base/target refs reduced to bare semver -# (self-hosted/vX.Y.Z -> X.Y.Z) and compared with sort -V. -# - If a bound is not a release tag (a commit SHA, or "master"), that side of -# the window is left open and all applicable entries are shown with a warning. -# - Prompt when an entry has breaking:true or a gate script; runs before -# backup/merge so abort leaves the deployment untouched. -# - CHANGELOG.md is never parsed (update.sh only points users at it); routine -# "requires compose update" items are applied by the merge, not listed here. -# -# User-owned paths skipped during merge are defined in .gitignore (loaded from the -# target snapshot). git check-ignore --no-index applies negation rules (e.g. -# volumes/functions/** ignored except volumes/functions/main/index.ts). -# .git/ is excluded here only - never listed in .gitignore. -# -# .env is never 3-way merged: append missing keys from .env.example only. - -# --- globals (set during main) ----------------------------------------------- - -REPO_URL="${SUPABASE_REPO_URL:-https://github.com/supabase/supabase}" -STAMP_FILE=".supabase-version" -SELF_NAME=$(basename "$0") -DRY_RUN=0 -ASSUME_YES=0 -TO_REF="" -FROM_REF="" - -TARGET_REF="" -BASE_REF="" -BASE_VER="" -TARGET_VER="" -REPORT_ONLY=0 - -TMP_ROOT="" -TARGET_DIR="" -BASE_DIR="" -REPORT="" -ENV_ADDED="" -ENV_REMOVED="" -GATE_REPORT="" -GATE_REQUIRED=0 -IGNORE_FILE="" -IGNORE_GIT_DIR="" - -# --- logging ----------------------------------------------------------------- - -log() { printf "===> %s\n" "$*"; } -warn() { printf "WARNING: %s\n" "$*" >&2; } -die() { printf "ERROR: %s\n" "$*" >&2; exit 1; } - -print_help() { - awk 'NR==1 {next} /^#/ {sub(/^# ?/,""); print; next} {exit}' "$0" -} - -# --- small helpers ----------------------------------------------------------- - -# load_ignore_file - vendor/user split from the target snapshot's .gitignore. -# Uses a throwaway git dir so check-ignore works on deployment trees that are -# not git repos (and on Apple Git, which rejects --git-dir=/dev/null). -load_ignore_file() { - if [ -f "$TARGET_DIR/.gitignore" ]; then - IGNORE_FILE="$TARGET_DIR/.gitignore" - elif [ -f .gitignore ]; then - IGNORE_FILE="$(pwd)/.gitignore" - else - IGNORE_FILE="" - warn "No .gitignore found; only .git paths are excluded from the merge." - return 0 - fi - case "$IGNORE_FILE" in - /*) ;; - *) IGNORE_FILE="$(cd "$(dirname "$IGNORE_FILE")" && pwd)/$(basename "$IGNORE_FILE")" ;; - esac - IGNORE_GIT_DIR="$TMP_ROOT/ignore-git" - git init -q "$IGNORE_GIT_DIR" -} - -# is_excluded - true when the path is user-owned and must not merge. -is_excluded() { - case "$1" in - .git|.git/*) return 0 ;; - esac - [ -n "$IGNORE_FILE" ] || return 1 - git -C "$IGNORE_GIT_DIR" -c "core.excludesfile=$IGNORE_FILE" \ - check-ignore -q --no-index "$1" 2>/dev/null -} - -read_stamp_ref() { - [ -f "$STAMP_FILE" ] || return 1 - val=$(grep -E '^ref=' "$STAMP_FILE" 2>/dev/null | head -n1 | cut -d= -f2- | tr -d "\r\"' ") - if [ -z "$val" ]; then - val=$(grep -vE '^[[:space:]]*#' "$STAMP_FILE" 2>/dev/null \ - | grep -vE '^[[:space:]]*$' | head -n1 | tr -d "\r\"' ") - fi - [ -n "$val" ] && printf '%s' "$val" -} - -# env_has_key - true if the key appears as KEY=, even commented -# (e.g. "#GOOGLE_ENABLED="), so we never re-add a key the user disabled on purpose. -env_has_key() { - grep -qE "^[[:space:]]*#?[[:space:]]*$1=" "$2" 2>/dev/null -} - -record() { printf '%s:%s\n' "$1" "$2" >> "$REPORT"; } - -count_status() { grep -cE "^$1:" "$REPORT" 2>/dev/null || true; } - -list_status() { - grep -E "^$1:" "$REPORT" 2>/dev/null | cut -d: -f2- | sed 's/^/ /' -} - -# --- upstream snapshots ------------------------------------------------------ - -_sparse_init() { - git -C "$1" init -q - git -C "$1" remote add origin "$REPO_URL" - git -C "$1" config core.sparseCheckout true - git -C "$1" sparse-checkout init --cone >/dev/null 2>&1 - git -C "$1" sparse-checkout set docker >/dev/null 2>&1 -} - -# fetch_snapshot -# Materializes ./docker at into via a shallow fetch. Also the -# seam an artifact source (tarball + sha256) would slot into later. -fetch_snapshot() { - _ref="$1" - _dest="$2" - _work=$(mktemp -d "$TMP_ROOT/fetch.XXXXXX") - if _sparse_init "$_work" \ - && git -C "$_work" fetch --depth=1 --filter=blob:none -q origin "$_ref" 2>/dev/null \ - && git -C "$_work" checkout -q FETCH_HEAD 2>/dev/null \ - && [ -d "$_work/docker" ]; then - mkdir -p "$_dest" - cp -rf "$_work/docker/." "$_dest/" - rm -rf "$_work" - else - rm -rf "$_work" - return 1 - fi -} - -latest_release_tag() { - git ls-remote --tags --refs "$REPO_URL" 2>/dev/null \ - | sed 's#^.*refs/tags/##' \ - | grep -E '^self-hosted/v[0-9]' \ - | sort -V | tail -n1 -} - -list_files() { - ( cd "$1" && find . -type f | sed 's|^\./||' | grep -vE '^\.git(/|$)' | sort ) -} - -# normalize_version - reduce a ref to bare semver (0.7.0) for comparison, -# or empty when it is not a self-hosted release tag (commit SHA, "master", …). -normalize_version() { - _v="${1#refs/tags/}" - _v="${_v#self-hosted/}" - _v="${_v#v}" - case "$_v" in - [0-9]*.[0-9]*) printf '%s' "$_v" ;; - *) ;; - esac -} - -# ver_gt A B - true when version A is strictly greater than B (sort -V order). -ver_gt() { - if [ "$1" = "$2" ]; then return 1; fi - [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n1)" = "$1" ] -} - -# ver_in_window VER - true when BASE_VER < VER <= TARGET_VER. An empty bound -# leaves that side open (over-reports a gate rather than hiding one). -ver_in_window() { - if [ -n "$TARGET_VER" ] && ver_gt "$1" "$TARGET_VER"; then return 1; fi - if [ -n "$BASE_VER" ] && ! ver_gt "$1" "$BASE_VER"; then return 1; fi - return 0 -} - -# --- ref resolution ---------------------------------------------------------- - -resolve_target_ref() { - if [ -n "$TO_REF" ]; then - TARGET_REF="$TO_REF" - return - fi - TARGET_REF=$(latest_release_tag) - if [ -n "$TARGET_REF" ]; then - log "Latest release tag: $TARGET_REF" - else - TARGET_REF="master" - warn "No self-hosted/v* release tags found; targeting 'master'. Pin a version with --to." - fi -} - -resolve_base_ref() { - REPORT_ONLY=0 - if [ -n "$FROM_REF" ]; then - BASE_REF="$FROM_REF" - elif BASE_REF=$(read_stamp_ref) && [ -n "$BASE_REF" ]; then - : - else - REPORT_ONLY=1 - BASE_REF="" - fi -} - -print_report_only_guidance() { - warn "No $STAMP_FILE found and no --from given; cannot determine the version you started from." - cat >&2 < $STAMP_FILE - 3. Re-run: sh update.sh - -Or supply it inline for this run: sh update.sh --from - -Continuing in REPORT-ONLY mode. NOTE: this is NOT the full set of changes - -without a base version it can only list files and .env keys that are entirely -NEW to you; it CANNOT show which existing files would change or conflict. Record -a base and re-run for the real preview. Nothing will be written. -EOF -} - -fetch_snapshots() { - TARGET_DIR="$TMP_ROOT/target" - log "Fetching target snapshot ($TARGET_REF)" - fetch_snapshot "$TARGET_REF" "$TARGET_DIR" \ - || die "Could not fetch target snapshot '$TARGET_REF' from $REPO_URL" - - if [ "$REPORT_ONLY" = "1" ]; then - return - fi - - BASE_DIR="$TMP_ROOT/base" - log "Fetching base snapshot ($BASE_REF)" - fetch_snapshot "$BASE_REF" "$BASE_DIR" \ - || die "Could not fetch base snapshot '$BASE_REF' from $REPO_URL" -} - -run_report_only() { - log "Files in '$TARGET_REF' you do NOT have yet (brand-new only; existing files that changed are NOT shown here):" - while IFS= read -r f; do - is_excluded "$f" && continue - [ -f "$f" ] || echo " + $f" - done </dev/null 2>&1; then - die "$_manifest is present but is not a valid JSON object; refusing to update without a working breaking-change gate. Please report this to the maintainers." - fi - - if [ -z "$BASE_VER" ] || [ -z "$TARGET_VER" ]; then - warn "Base or target is not a self-hosted/vX.Y.Z tag; cannot compute an exact update window." - warn "Showing all applicable manual-action releases." - warn "Review which ones apply to your deployment." - fi - - for k in $(jq -r 'keys[]' "$_manifest" 2>/dev/null); do - case "$k" in [0-9]*.[0-9]*) ;; *) continue ;; esac - ver_in_window "$k" || continue - - breaking=$(jq -r --arg k "$k" '.[$k].breaking // false' "$_manifest") - gate=$(jq -r --arg k "$k" '.[$k].gate // empty' "$_manifest") - url=$(jq -r --arg k "$k" '.[$k].migration_guide_url // empty' "$_manifest") - reqs=$(jq -r --arg k "$k" '.[$k].requires[]? // empty' "$_manifest") - - if [ "$breaking" = "true" ] || [ -n "$gate" ]; then - GATE_REQUIRED=1 - fi - - { - [ "$breaking" = "true" ] && echo "[$k] BREAKING" || echo "[$k]" - [ -n "$gate" ] && echo " gate: '$gate' must be run first (see the steps below for the exact command)" - [ -n "$url" ] && echo " guide: $url" - [ -n "$reqs" ] && printf '%s\n' "$reqs" | sed 's/^/ - /' - } >> "$GATE_REPORT" - done - # Explicit success: the loop's last iteration can end on a false test (e.g. - # an entry with no 'requires'), which would otherwise make this function - # return non-zero and abort the whole script under 'set -e'. - return 0 -} - -confirm_gate() { - [ "$GATE_REQUIRED" = "1" ] || return 0 - [ "$DRY_RUN" != "1" ] || return 0 - [ "$ASSUME_YES" != "1" ] || return 0 - - echo "" >&2 - warn "This update requires manual action - review before continuing:" - sed 's/^/ /' "$GATE_REPORT" >&2 - echo "" >&2 - - if { : > /dev/tty; } 2>/dev/null; then - printf "Have you completed the required steps and want to continue? [y/N]: " > /dev/tty - read -r reply < /dev/tty - case "$reply" in - y|Y|yes|YES) return 0 ;; - *) die "Aborted by user. Nothing was modified." ;; - esac - fi - die "Breaking/gated changes present and no controlling terminal to confirm. Re-run with --yes to proceed." -} - -# --- backup ------------------------------------------------------------------ - -take_backup() { - if [ "$DRY_RUN" = "1" ]; then - log "Dry run: no backup taken, nothing will be written." - return 0 - fi - mkdir -p backups - _backup="backups/pre-update-$(date +%Y%m%d-%H%M%S).tgz" - log "Backing up current configuration to $_backup (excluding data directories)" - tar czf "$_backup" \ - --exclude='./backups' \ - --exclude='./volumes/db/data' \ - --exclude='./volumes/storage' \ - . 2>/dev/null || warn "Backup archive reported errors; review $_backup before relying on it." - warn "This does NOT back up your database. Back it up separately before updating." -} - -# --- vendor file merge ------------------------------------------------------- - -apply_file() { - [ "$DRY_RUN" = "1" ] && return 0 - _dir=$(dirname "$1") - [ "$_dir" = "." ] || mkdir -p "$_dir" - cp -f "$2" "$1" -} - -# Per-file 3-way merge (u=yours, b=base snapshot, t=target snapshot): -# no t → keep u; report removed-upstream -# no u → copy t; report new -# u == t → report unchanged -# u == b → copy t; report updated (user never edited) -# else → git merge-file u b t; report merged-clean or CONFLICT -# If b had no file, an empty file stands in for b. -merge_one_file() { - f="$1" - empty="$2" - merged="$TMP_ROOT/merged.out" - - b="$BASE_DIR/$f" - t="$TARGET_DIR/$f" - u="$f" - - if [ ! -f "$t" ]; then - [ -f "$u" ] && record "removed-upstream" "$f" - return 0 - fi - - if [ ! -f "$u" ]; then - apply_file "$f" "$t" - record "new" "$f" - return 0 - fi - - if cmp -s "$u" "$t"; then - record "unchanged" "$f" - return 0 - fi - - base_for_merge="$b" - [ -f "$base_for_merge" ] || base_for_merge="$empty" - - if cmp -s "$u" "$base_for_merge"; then - apply_file "$f" "$t" - record "updated" "$f" - return 0 - fi - - if git merge-file -p -q \ - -L "yours ($f)" -L "base" -L "new ($TARGET_REF)" \ - "$u" "$base_for_merge" "$t" > "$merged" 2>/dev/null; then - [ "$DRY_RUN" != "1" ] && cp -f "$merged" "$u" - record "merged-clean" "$f" - elif [ -s "$merged" ]; then - # Non-zero exit with output = a normal conflict (markers written). - [ "$DRY_RUN" != "1" ] && cp -f "$merged" "$u" - record "CONFLICT" "$f" - else - # git merge-file errored and produced no output; keep the user's file - # intact rather than truncating it to empty. - record "merge-failed" "$f" - fi -} - -# The running script is a vendor file too, but overwriting it in place would -# corrupt this process (the shell reads $0 as it runs). Never write it directly: -# if the target ships a different version, stage it as .dist to review. -stage_self_update() { - _t="$TARGET_DIR/$SELF_NAME" - if [ ! -f "$_t" ] || cmp -s "$SELF_NAME" "$_t"; then - record "unchanged" "$SELF_NAME" - return 0 - fi - [ "$DRY_RUN" = "1" ] || cp -f "$_t" "$SELF_NAME.dist" - record "self-staged" "$SELF_NAME" -} - -merge_vendor_files() { - _empty="$TMP_ROOT/empty" - : > "$_empty" - : > "$REPORT" - - while IFS= read -r f; do - [ -n "$f" ] || continue - is_excluded "$f" && continue - if [ "$f" = "$SELF_NAME" ]; then - stage_self_update - continue - fi - merge_one_file "$f" "$_empty" - done < "$ENV_ADDED" - : > "$ENV_REMOVED" - [ -f "$_example" ] || return 0 - - while IFS= read -r k; do - env_has_key "$k" .env || echo "$k" >> "$ENV_ADDED" - done <>>>>>> markers):" - list_status CONFLICT - fi - if [ "$(count_status merge-failed)" != "0" ]; then - echo "" - warn "Files git could not merge (left unchanged - update these manually):" - list_status merge-failed - fi - if [ "$(count_status merged-clean)" != "0" ]; then - echo "" - log "Files merged cleanly (review recommended):" - list_status merged-clean - fi - if [ "$(count_status removed-upstream)" != "0" ]; then - echo "" - log "Removed upstream but kept in place (you may no longer need these):" - list_status removed-upstream - fi - if [ "$(count_status self-staged)" != "0" ]; then - echo "" - if [ "$DRY_RUN" = "1" ]; then - log "$SELF_NAME differs from the version in '$TARGET_REF'; a real run would stage that version as $SELF_NAME.dist (the running script is never overwritten in place)." - else - log "$SELF_NAME differs from the version in '$TARGET_REF', staged as $SELF_NAME.dist (the running script was not modified)." - log "Review it, then swap it in if needed: mv $SELF_NAME.dist $SELF_NAME" - fi - fi - if [ -s "$ENV_ADDED" ]; then - echo "" - log ".env keys added (review values):" - sed 's/^/ + /' "$ENV_ADDED" - fi - if [ -s "$ENV_REMOVED" ]; then - echo "" - log ".env keys you have that are gone from the new .env.example (review/remove manually):" - sed 's/^/ - /' "$ENV_REMOVED" - fi - if [ -s "$GATE_REPORT" ]; then - echo "" - log "Required manual steps for this update (from upgrades.json):" - sed 's/^/ /' "$GATE_REPORT" - fi - echo "" - log "For what changed in this update, see CHANGELOG.md (from $BASE_REF to $TARGET_REF)." -} - -write_stamp() { - { - echo "# Supabase self-hosted version stamp. Managed by setup.sh / update.sh." - echo "# Do not commit or edit by hand. Records the ref this deployment was based on." - echo "ref=$TARGET_REF" - } > "$STAMP_FILE" -} - -print_next_steps() { - echo "" - log "Next steps:" - echo " 1. Review the changes above (compare against the latest backup in backups/ if needed)." - echo " 2. sh run.sh pull" - echo " 3. sh run.sh recreate" -} - -# --- main -------------------------------------------------------------------- - -while [ $# -gt 0 ]; do - case "$1" in - --dry-run) DRY_RUN=1; shift ;; - --yes|-y) ASSUME_YES=1; shift ;; - --to) [ $# -ge 2 ] || die "--to requires a argument."; TO_REF="$2"; shift 2 ;; - --from) [ $# -ge 2 ] || die "--from requires a argument."; FROM_REF="$2"; shift 2 ;; - -h|--help) print_help; exit 0 ;; - *) echo "Unknown option: $1" >&2; print_help; exit 1 ;; - esac -done - -[ -f docker-compose.yml ] || die "docker-compose.yml not found in $(pwd). Run this from your deployment directory." -[ -f .env ] || die ".env not found in $(pwd). This does not look like a configured deployment." -command -v git >/dev/null 2>&1 || die "git is required but was not found on PATH." -command -v jq >/dev/null 2>&1 || die "jq is required but was not found on PATH." - -resolve_target_ref -resolve_base_ref -TARGET_VER=$(normalize_version "$TARGET_REF") -BASE_VER=$(normalize_version "$BASE_REF") -[ "$REPORT_ONLY" = "1" ] && print_report_only_guidance - -TMP_ROOT=$(mktemp -d) -REPORT="$TMP_ROOT/report" -ENV_ADDED="$TMP_ROOT/env_added" -ENV_REMOVED="$TMP_ROOT/env_removed" -GATE_REPORT="$TMP_ROOT/gate_report" -trap 'rm -rf "$TMP_ROOT"' EXIT INT TERM - -fetch_snapshots -load_ignore_file - -if [ "$REPORT_ONLY" = "1" ]; then - run_report_only - build_gate_report - if [ -s "$GATE_REPORT" ]; then - echo "" - log "Breaking changes / required manual steps in this range (from upgrades.json):" - sed 's/^/ /' "$GATE_REPORT" - warn "Record a base version (see above) and re-run to apply with the gate enforced." - fi - exit 0 -fi - -build_gate_report -confirm_gate - -take_backup -merge_vendor_files -merge_env_file -print_summary - -if [ "$DRY_RUN" = "1" ]; then - echo "" - log "Dry run complete. Re-run without --dry-run to apply." - exit 0 -fi - -if [ "$(count_status CONFLICT)" != "0" ] || [ "$(count_status merge-failed)" != "0" ]; then - echo "" - warn "Update applied WITH CONFLICTS. Resolve the files listed above (remove the" - warn "<<<<<<< ======= >>>>>>> markers, or fix the files git could not merge)" - warn "before starting the stack." - warn "The version stamp was NOT advanced; it will update on your next clean run." - warn "Exiting with status 2." - exit 2 -fi - -write_stamp -print_next_steps -log "Update applied cleanly." diff --git a/supabase/upgrades.json b/supabase/upgrades.json deleted file mode 100644 index 21704448b0..0000000000 --- a/supabase/upgrades.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "_schema": { - "description": "Manual-action manifest for self-hosted upgrades, read by update.sh with jq. Hand-edit this file directly; it is the source of truth (no generation step). Keys are self-hosted release versions as bare semver (e.g. \"0.7.0\"), matching the self-hosted/vX.Y.Z tag and the \"## [0.7.0]\" CHANGELOG heading. Only add an entry for a release that needs an action a file diff cannot encode: a data migration, a run-this-first script, or a breaking default. Routine config changes are applied automatically by update.sh's 3-way merge and must NOT be listed here. update.sh gates on entries in (your version, target version], ordered with sort -V.", - "fields": { - "breaking": "bool - requires explicit user confirmation before applying", - "gate": "string|null - script to run before upgrading past this release (e.g. utils/upgrade-pg17.sh)", - "migration_guide_url": "string|null - link to the per-release migration guide", - "requires": "string[] - free-text manual steps shown to the user" - } - }, - "0.6.0": { - "breaking": true, - "gate": "utils/upgrade-pg17.sh", - "migration_guide_url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "requires": [ - "Postgres 17 is now the default. Do NOT start Postgres 17 against an existing Postgres 15 data directory - back up your database first.", - "Run 'sudo bash utils/upgrade-pg17.sh' (needs bash + root) to migrate Postgres 15 -> 17, then recreate containers. To defer, pin Postgres 15 with the docker-compose.pg15.yml override.", - "Includes a security fix for the API gateway (Realtime /api/tenants and /api/openapi routes) - strongly recommended for any instance running Realtime." - ] - }, - "0.7.0": { - "breaking": true, - "gate": null, - "migration_guide_url": "https://github.com/orgs/supabase/discussions/47093", - "requires": [ - "API_EXTERNAL_URL now includes the /auth/v1 path prefix, and SAML SSO endpoints moved to /auth/v1/sso/saml/*. Update custom OAuth provider callback URLs and any SAML configuration accordingly.", - "Anon (publishable) key access to the OpenAPI spec at /rest/v1/ has been removed. Use the service role or secret API key if you relied on it; normal data access via /rest/v1/
is unaffected." - ] - }, - "0.8.0": { - "breaking": true, - "gate": null, - "migration_guide_url": "https://github.com/orgs/supabase/discussions/48048", - "requires": [ - "Envoy is now the default API gateway, replacing Kong. The gateway service is renamed from 'kong' to 'api-gw' (container 'supabase-envoy'); the 'kong' network alias still resolves, so internal service references keep working.", - "If you customized volumes/api/kong.yml or the gateway service, enable the Kong override with 'sh run.sh config add kong' to keep running Kong; otherwise the merge switches you to Envoy." - ] - } -} diff --git a/supabase/utils/add-new-auth-keys.sh b/supabase/utils/add-new-auth-keys.sh deleted file mode 100644 index f921b6e876..0000000000 --- a/supabase/utils/add-new-auth-keys.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/sh -# -# Add asymmetric key pair and opaque API keys to a self-hosted Supabase installation. -# -# Reads JWT_SECRET from .env and generates: -# - EC P-256 key pair (JWT_KEYS, JWT_JWKS) -# - Opaque API keys (SUPABASE_PUBLISHABLE_KEY, SUPABASE_SECRET_KEY) -# - Internal: ES256 JWT API keys (ANON_KEY_ASYMMETRIC, SERVICE_ROLE_KEY_ASYMMETRIC) -# -# Usage: -# sh add-new-auth-keys.sh # Interactive: prints keys, prompts to update .env -# sh add-new-auth-keys.sh --update-env # Prints keys and writes them to .env -# sh add-new-auth-keys.sh | tee keys # Non-interactive: prints keys only -# -# Prerequisites: -# - .env file with JWT_SECRET set (run generate-keys.sh first) -# - node (>= 16) or docker -# - -set -e - -node_ok() { - command -v node >/dev/null 2>&1 || return 1 - major=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) - [ -n "$major" ] && [ "$major" -ge 16 ] 2>/dev/null -} - -# Resolve how to run node: local install (>= 16) preferred, docker fallback. -if node_ok; then - node_runner="node" -else - if command -v node >/dev/null 2>&1; then - echo "Local node $(node -v) is too old (need >= 16), falling back to docker." - fi - - if ! command -v docker >/dev/null 2>&1; then - echo "Error: requires either node (>= 16) or docker." - exit 1 - fi - - if ! ${DOCKER:-docker} info >/dev/null; then - echo "Error: cannot connect to the Docker daemon." - exit 1 - fi - - if ! ${DOCKER:-docker} image inspect node:22-alpine >/dev/null 2>&1; then - echo "Pulling node:22-alpine (first-run only)..." - ${DOCKER:-docker} pull node:22-alpine - fi - - node_runner="${DOCKER:-docker} run --rm node:22-alpine node" -fi - -# Read JWT_SECRET from .env -if [ ! -f .env ]; then - echo "Error: .env file not found. Run generate-keys.sh first." - exit 1 -fi - -jwt_secret=$(grep '^JWT_SECRET=' .env | cut -d= -f2- | tr -d '\r') -if [ -z "$jwt_secret" ]; then - echo "Error: JWT_SECRET not found in .env. Run generate-keys.sh first." - exit 1 -fi - -tmpdir=$(mktemp -d) -trap 'rm -rf "$tmpdir"' EXIT - -# Node.js does the crypto-heavy work: -# - EC P-256 keypair generation -# - JWKS construction (with symmetric key included) -# - ES256 JWT signing -# - Opaque API key generation with checksum -$node_runner -e ' -const crypto = require("crypto"); - -const jwtSecret = process.argv[1]; - -// Generate EC P-256 keypair and export as JWK -const { privateKey } = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); -const jwkPrivate = privateKey.export({ format: "jwk" }); - -const kid = crypto.randomUUID(); - -// Symmetric key as JWK (base64url-encoded) -const octKey = { - kty: "oct", - k: Buffer.from(jwtSecret).toString("base64url"), - alg: "HS256" -}; - -// JWKS with private key (for Auth to sign tokens) -const jwksKeypair = { keys: [ - { kty: "EC", kid, use: "sig", key_ops: ["sign", "verify"], alg: "ES256", ext: true, - crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y, d: jwkPrivate.d }, - octKey -]}; - -// JWKS with public key only (for PostgREST, Realtime, Storage to verify) -const jwksPublic = { keys: [ - { kty: "EC", kid, use: "sig", key_ops: ["verify"], alg: "ES256", ext: true, - crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y }, - octKey -]}; - -// Sign ES256 JWT -function signES256(payload) { - const header = { alg: "ES256", typ: "JWT", kid }; - const b64Header = Buffer.from(JSON.stringify(header)).toString("base64url"); - const b64Payload = Buffer.from(JSON.stringify(payload)).toString("base64url"); - const data = b64Header + "." + b64Payload; - const sig = crypto.sign("SHA256", Buffer.from(data), { - key: privateKey, - dsaEncoding: "ieee-p1363" - }).toString("base64url"); - return data + "." + sig; -} - -const iat = Math.floor(Date.now() / 1000); -const exp = iat + 5 * 365 * 24 * 3600; // 5 years - -const anonJwt = signES256({ role: "anon", iss: "supabase", iat, exp }); -const serviceJwt = signES256({ role: "service_role", iss: "supabase", iat, exp }); - -// Generate opaque API keys with checksum -const PROJECT_REF = "supabase-self-hosted"; - -function generateOpaqueKey(prefix) { - const random = crypto.randomBytes(17).toString("base64url").slice(0, 22); - const intermediate = prefix + random; - const checksum = crypto.createHash("sha256") - .update(PROJECT_REF + "|" + intermediate) - .digest("base64url") - .slice(0, 8); - return intermediate + "_" + checksum; -} - -const publishableKey = generateOpaqueKey("sb_publishable_"); -const secretKey = generateOpaqueKey("sb_secret_"); - -// Output as KEY=value lines for shell to parse -console.log("SUPABASE_PUBLISHABLE_KEY=" + publishableKey); -console.log("SUPABASE_SECRET_KEY=" + secretKey); -console.log("ANON_KEY_ASYMMETRIC=" + anonJwt); -console.log("SERVICE_ROLE_KEY_ASYMMETRIC=" + serviceJwt); -console.log("JWT_KEYS=" + JSON.stringify(jwksKeypair.keys)); -console.log("JWT_JWKS=" + JSON.stringify(jwksPublic)); -' "$jwt_secret" > "$tmpdir/output" - -# Read generated values -SUPABASE_PUBLISHABLE_KEY=$(grep '^SUPABASE_PUBLISHABLE_KEY=' "$tmpdir/output" | cut -d= -f2-) -SUPABASE_SECRET_KEY=$(grep '^SUPABASE_SECRET_KEY=' "$tmpdir/output" | cut -d= -f2-) -ANON_KEY_ASYMMETRIC=$(grep '^ANON_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) -SERVICE_ROLE_KEY_ASYMMETRIC=$(grep '^SERVICE_ROLE_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) -JWT_KEYS=$(grep '^JWT_KEYS=' "$tmpdir/output" | cut -d= -f2-) -JWT_JWKS=$(grep '^JWT_JWKS=' "$tmpdir/output" | cut -d= -f2-) - -echo "" -echo "SUPABASE_PUBLISHABLE_KEY=${SUPABASE_PUBLISHABLE_KEY}" -echo "SUPABASE_SECRET_KEY=${SUPABASE_SECRET_KEY}" -echo "" -echo "JWT_KEYS=${JWT_KEYS}" -echo "" -echo "JWT_JWKS=${JWT_JWKS}" -echo "" -echo "Ensure the following configuration is uncommented in docker-compose.yml for the asymmetric key pair to work:" -echo "" -echo " Auth: GOTRUE_JWT_KEYS: \${JWT_KEYS:-[]}" -echo " Realtime: API_JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" -echo " Storage: JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" -echo " Functions: SUPABASE_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" -echo "" - -if [ "$1" = "--update-env" ]; then - update_env=true -elif test -t 0; then - printf "Update .env file? (y/N) " - read -r REPLY - case "$REPLY" in - [Yy]) update_env=true ;; - *) update_env=false ;; - esac -else - echo "Running non-interactively. Pass --update-env to write to .env." - update_env=false -fi - -if [ "$update_env" != "true" ]; then - exit 0 -fi - -echo "Updating .env..." - -# Append new variables if they don't exist, or update them if they do -for var in SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY ANON_KEY_ASYMMETRIC SERVICE_ROLE_KEY_ASYMMETRIC JWT_KEYS JWT_JWKS; do - eval "val=\$$var" - if grep -q "^${var}=" .env; then - sed -i.old -e "s|^${var}=.*$|${var}=${val}|" .env - else - echo "${var}=${val}" >> .env - fi -done - -# Uncomment new auth configuration in docker-compose.yml -echo "Updating docker-compose.yml..." -if [ ! -f docker-compose.yml ]; then - echo "Error: docker-compose.yml not found in $(pwd)" - exit 1 -fi - -# Always fall through to the grep check -sed -i.old \ - -e '/^[ ]*#GOTRUE_JWT_KEYS:/ s/#//' \ - -e '/^[ ]*#API_JWT_JWKS:/ s/#//' \ - -e '/^[ ]*#JWT_JWKS:/ s/#//' \ - -e '/^[ ]*#SUPABASE_JWKS:/ s/#//' \ - docker-compose.yml || true - -if grep -q '^[ ]*GOTRUE_JWT_KEYS:' docker-compose.yml && \ - grep -q '^[ ]*API_JWT_JWKS:' docker-compose.yml && \ - grep -q '^[ ]*JWT_JWKS:' docker-compose.yml && \ - grep -q '^[ ]*SUPABASE_JWKS:' docker-compose.yml; then - echo "Done." -else - echo "Warning: could not edit docker-compose.yml. Uncomment auth configuration manually." -fi diff --git a/supabase/utils/generate-keys.sh b/supabase/utils/generate-keys.sh deleted file mode 100644 index da1fc19218..0000000000 --- a/supabase/utils/generate-keys.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/sh -# -# Generate secrets and legacy symmetric JWT API keys for self-hosted Supabase. -# -# Generates: JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, and other secrets -# needed for a fresh installation. -# -# Usage: -# sh generate-keys.sh # Interactive: prints keys, prompts to update .env -# sh generate-keys.sh --update-env # Prints keys and writes them to .env -# sh generate-keys.sh | tee keys # Non-interactive: prints keys only -# -# Portions of this code are derived from Inder Singh's setup.sh shell script. -# Copyright 2025 Inder Singh. Licensed under Apache License 2.0. -# Original source: https://github.com/singh-inder/supabase-automated-self-host/blob/main/setup.sh -# - -set -e - -gen_hex() { - openssl rand -hex "$1" -} - -gen_base64() { - openssl rand -base64 "$1" -} - -base64_url_encode() { - openssl enc -base64 -A | tr '+/' '-_' | tr -d '=' -} - -gen_token() { - payload=$1 - payload_base64=$(printf %s "$payload" | base64_url_encode) - header_base64=$(printf %s "$header" | base64_url_encode) - signed_content="${header_base64}.${payload_base64}" - signature=$(printf %s "$signed_content" | openssl dgst -binary -sha256 -hmac "$jwt_secret" | base64_url_encode) - printf '%s' "${signed_content}.${signature}" -} - -if ! command -v openssl >/dev/null 2>&1; then - echo "Error: openssl is required but not found." - exit 1 -fi - -jwt_secret="$(gen_base64 30)" - -# Used in gen_token() -header='{"alg":"HS256","typ":"JWT"}' -iat=$(date +%s) -exp=$((iat + 5 * 3600 * 24 * 365)) # 5 years - -# Normalizes JSON formatting so that the token matches https://www.jwt.io/ results -anon_payload="{\"role\":\"anon\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" -service_role_payload="{\"role\":\"service_role\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" - -#echo "anon_payload=$anon_payload" -#echo "service_role_payload=$service_role_payload" - -anon_key=$(gen_token "$anon_payload") -service_role_key=$(gen_token "$service_role_payload") - -secret_key_base=$(gen_base64 48) -realtime_db_enc_key=$(gen_hex 8) -vault_enc_key=$(gen_hex 16) -pg_meta_crypto_key=$(gen_base64 24) - -logflare_public_access_token=$(gen_base64 24) -logflare_private_access_token=$(gen_base64 24) - -s3_protocol_access_key_id=$(gen_hex 16) -s3_protocol_access_key_secret=$(gen_hex 32) - -minio_root_password=$(gen_hex 16) - -echo "" -echo "JWT_SECRET=${jwt_secret}" -echo "" -#echo "Issued at: $iat" -#echo "Expire: $exp" -echo "ANON_KEY=${anon_key}" -echo "SERVICE_ROLE_KEY=${service_role_key}" -echo "" -echo "SECRET_KEY_BASE=${secret_key_base}" -echo "REALTIME_DB_ENC_KEY=${realtime_db_enc_key}" -echo "VAULT_ENC_KEY=${vault_enc_key}" -echo "PG_META_CRYPTO_KEY=${pg_meta_crypto_key}" -echo "LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}" -echo "LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}" -echo "S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}" -echo "S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}" -echo "MINIO_ROOT_PASSWORD=${minio_root_password}" -echo "" - -postgres_password=$(gen_hex 16) -dashboard_password=$(gen_hex 16) - -echo "POSTGRES_PASSWORD=${postgres_password}" -echo "DASHBOARD_PASSWORD=${dashboard_password}" -echo "" - -if [ "$1" = "--update-env" ]; then - update_env=true -elif test -t 0; then - printf "Update .env file? (y/N) " - read -r REPLY - case "$REPLY" in - [Yy]) update_env=true ;; - *) update_env=false ;; - esac -else - echo "Running non-interactively. Pass --update-env to write to .env." - update_env=false -fi - -if [ "$update_env" != "true" ]; then - exit 0 -fi - -echo "Updating .env..." - -sed \ - -i.old \ - -e "s|^JWT_SECRET=.*$|JWT_SECRET=${jwt_secret}|" \ - -e "s|^ANON_KEY=.*$|ANON_KEY=${anon_key}|" \ - -e "s|^SERVICE_ROLE_KEY=.*$|SERVICE_ROLE_KEY=${service_role_key}|" \ - -e "s|^SECRET_KEY_BASE=.*$|SECRET_KEY_BASE=${secret_key_base}|" \ - -e "s|^REALTIME_DB_ENC_KEY=.*$|REALTIME_DB_ENC_KEY=${realtime_db_enc_key}|" \ - -e "s|^VAULT_ENC_KEY=.*$|VAULT_ENC_KEY=${vault_enc_key}|" \ - -e "s|^PG_META_CRYPTO_KEY=.*$|PG_META_CRYPTO_KEY=${pg_meta_crypto_key}|" \ - -e "s|^LOGFLARE_PUBLIC_ACCESS_TOKEN=.*$|LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}|" \ - -e "s|^LOGFLARE_PRIVATE_ACCESS_TOKEN=.*$|LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}|" \ - -e "s|^S3_PROTOCOL_ACCESS_KEY_ID=.*$|S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}|" \ - -e "s|^S3_PROTOCOL_ACCESS_KEY_SECRET=.*$|S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}|" \ - -e "s|^MINIO_ROOT_PASSWORD=.*$|MINIO_ROOT_PASSWORD=${minio_root_password}|" \ - -e "s|^POSTGRES_PASSWORD=.*$|POSTGRES_PASSWORD=${postgres_password}|" \ - -e "s|^DASHBOARD_PASSWORD=.*$|DASHBOARD_PASSWORD=${dashboard_password}|" \ - .env diff --git a/supabase/utils/upgrade-pg17.sh b/supabase/utils/upgrade-pg17.sh deleted file mode 100644 index c05a7b2571..0000000000 --- a/supabase/utils/upgrade-pg17.sh +++ /dev/null @@ -1,822 +0,0 @@ -#!/usr/bin/env bash -# -# Requires bash (not sh) for pipefail, which ensures failures in piped -# commands are caught during the upgrade. -# -# Upgrade self-hosted Supabase Postgres from 15 to 17. -# -# Uses Supabase's pg_upgrade scripts (initiate.sh + complete.sh) inside a -# temporary PG 15 container, then swaps data directories and starts Postgres 17. -# -# Usage (must be run as root or with sudo): -# cd docker/ -# sudo bash utils/upgrade-pg17.sh # Interactive (prompts for confirmation) -# sudo bash utils/upgrade-pg17.sh --yes # Non-interactive (skip all prompts) -# -# Requirements: -# - Docker with Docker Compose (docker compose, not docker-compose) -# - Running Supabase self-hosted setup with Postgres 15 -# - At least 2x current database size + 5 GB free disk space -# -# Backup: -# The original Postgres 15 data directory is preserved as -# ./volumes/db/data.bak.pg15 during the upgrade. -# DO NOT DELETE it until you have verified the upgrade was successful. -# -# Rollback (if the upgrade fails or you want to revert): -# 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down -# 2. rm -rf ./volumes/db/data -# 3. mv ./volumes/db/data.bak.pg15 ./volumes/db/data -# 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/ -# 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d -# - -# Ensure we're running under bash (not sh/zsh/dash). -# Check that $BASH ends with /bash (not /sh, /zsh, etc.). -case "${BASH:-}" in - */bash) ;; - *) echo "Error: This script requires bash. Run it with: sudo bash $0" >&2; exit 1 ;; -esac - -set -euo pipefail - -AUTO_CONFIRM=false -for arg in "$@"; do - case "$arg" in - --yes|-y) AUTO_CONFIRM=true ;; - esac -done - -# --- Configuration ---------------------------------------------------------- - -# Image used for the upgrade tarball + complete.sh container. -# Must share glibc with PG 15 (the extracted ELF binaries run inside PG15). -# Pinned to .063: later images bumped glibc, which breaks the ELF extraction. -PG17_UPGRADE_IMAGE="supabase/postgres:17.6.1.063" -# Tag in supabase/postgres repo matching the upgrade image (for downloading scripts) -PG17_SCRIPTS_REF="17.6.1.063" - -# Final Postgres 17 image that runs after the upgrade. Pinned here (not read from -# the compose files). Keep in sync with the db image in docker-compose.yml (and -# docker-compose.pg17.yml). Used for pulling, chowning data/db-config to the -# target's postgres UID, and running the post-upgrade migrations. -PG17_TARGET_IMAGE="supabase/postgres:17.6.1.136" - -DB_CONTAINER="supabase-db" -UPGRADE_CONTAINER="supabase-pg-upgrade" -COMPLETE_CONTAINER="supabase-pg-complete" - -DATA_DIR="./volumes/db/data" -BACKUP_DIR="./volumes/db/data.bak.pg15" -# Include image tag in cache filename so changing PG17_UPGRADE_IMAGE invalidates it -PG17_TAG="${PG17_UPGRADE_IMAGE##*:}" -TARBALL_CACHE="./volumes/db/pg17_upgrade_bin_${PG17_TAG}.tar.gz" -# initiate.sh writes pg_upgrade output here: pgdata/, conf/, sql/ -MIGRATION_DIR="./volumes/db/data_migration" - -# --- Helpers ---------------------------------------------------------------- - -die() { printf 'Error: %s\n' "$*" >&2; exit 1; } -info() { printf '\n==> %s\n' "$*"; } -warn() { printf 'Warning: %s\n' "$*" >&2; } - -# Temp dir on host for tarball + scripts (mounted into containers) -staging_dir="" -pg_password="" -current_image="" -drop_extensions="" -db_config_vol="" - -# Remove leftover containers and staging dir on exit. -# Uses an alpine container for rm because the tarball build runs as root -# inside Docker - the resulting files are root-owned and can't be deleted -# by the host user on macOS. -cleanup() { - docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true - docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true - if [ -n "$staging_dir" ] && [ -d "$staging_dir" ]; then - docker run --rm -v "$staging_dir:/cleanup" alpine rm -rf /cleanup 2>/dev/null || true - rm -rf "$staging_dir" 2>/dev/null || true - fi -} -trap cleanup EXIT - -on_interrupt() { - echo "" - warn "Interrupted. Cleaning up..." - # If db-config was chowned to PG17, restore for PG 15 rollback - if [ -n "$db_config_vol" ] && [ -n "$current_image" ]; then - docker run --rm -v "${db_config_vol}:/vol" "$current_image" \ - chown -R postgres:postgres /vol/ 2>/dev/null || true - fi - die "Interrupted." -} -trap on_interrupt INT - -confirm() { - if [ "$AUTO_CONFIRM" = true ]; then return 0; fi - if ! test -t 0; then - die "This script must be run interactively, or use --yes to skip prompts." - fi - printf '%s (y/N) ' "$1" - read -r reply - case "$reply" in - [Yy]*) return 0 ;; - *) echo "Aborted."; exit 0 ;; - esac -} - -run_sql_on() { - local container=$1; shift - docker exec -i \ - -e PGPASSWORD="$pg_password" \ - "$container" \ - psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 "$@" -} - -wait_for_healthy() { - local container=$1 retries=30 - while [ $retries -gt 0 ]; do - if docker exec "$container" pg_isready -U postgres -h localhost >/dev/null 2>&1; then - return 0 - fi - retries=$((retries - 1)) - sleep 1 - done - die "Postgres in '$container' did not become ready in 30 seconds." -} - -# --- Pre-flight checks ----------------------------------------------------- - -preflight() { - info "Running pre-flight checks" - - if [ "$(id -u)" -ne 0 ]; then - die "This script must be run as root (e.g. sudo bash $0)." - fi - - docker compose version >/dev/null 2>&1 || die "Docker Compose not found." - command -v curl >/dev/null 2>&1 || die "curl is required (for downloading upgrade scripts)." - [ -f docker-compose.yml ] || die "Run this script from the docker/ directory." - [ -f docker-compose.pg17.yml ] || die "Missing docker-compose.pg17.yml." - [ -f .env ] || die "Missing .env file." - - # Resolve db-config volume (exact match on _db-config suffix or bare db-config) - db_config_vol=$(docker volume ls --filter "name=db-config" --format '{{.Name}}' \ - | grep -E '^db-config$|_db-config$' | head -n 1) - [ -n "$db_config_vol" ] || die "Could not find db-config volume. Is Supabase running?" - - pg_password=$(grep '^POSTGRES_PASSWORD=' .env | cut -d '=' -f 2- | sed "s/^['\"]//;s/['\"]$//" | head -n 1) - [ -n "$pg_password" ] || die "POSTGRES_PASSWORD not set in .env." - - docker inspect "$DB_CONTAINER" >/dev/null 2>&1 \ - || die "Container '$DB_CONTAINER' not found. Is Supabase running?" - - current_image=$(docker inspect "$DB_CONTAINER" --format '{{.Config.Image}}') - case "$current_image" in - supabase/postgres:15.*|supabase.postgres:15.*) ;; - supabase/postgres:17.*|supabase.postgres:17.*) die "Already running Postgres 17 ($current_image)." ;; - *) die "Unexpected database image: $current_image" ;; - esac - - local status - status=$(docker inspect "$DB_CONTAINER" --format '{{.State.Status}}') - [ "$status" = "running" ] || die "'$DB_CONTAINER' is not running (status: $status)." - [ -d "$DATA_DIR" ] || die "Data directory not found: $DATA_DIR" - - if [ -d "$BACKUP_DIR" ]; then - warn "Backup directory already exists: $BACKUP_DIR" - warn "This is likely from a previous upgrade attempt." - warn "If you haven't verified that previous upgrade, roll back first:" - warn " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" - warn " 2. rm -rf $DATA_DIR" - warn " 3. mv $BACKUP_DIR $DATA_DIR" - warn " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" - warn " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" - echo "" - warn "Continuing will DELETE the existing backup permanently." - confirm "Delete $BACKUP_DIR and start a fresh upgrade?" - rm -rf "$BACKUP_DIR" - fi - if [ -d "$MIGRATION_DIR" ]; then - rm -rf "$MIGRATION_DIR" - fi - - # Disk space - local data_size_kb data_size_mb avail_kb avail_mb needed_mb - data_size_kb=$(du -sk "$DATA_DIR" 2>/dev/null | cut -f1) - [ -n "$data_size_kb" ] || die "Could not calculate data size for $DATA_DIR" - data_size_mb=$((data_size_kb / 1024)) - avail_kb=$(df -k "$(dirname "$DATA_DIR")" | awk 'NR==2 { print $4 }') - [ -n "$avail_kb" ] || die "Could not calculate available disk space for $(dirname "$DATA_DIR")" - avail_mb=$((avail_kb / 1024)) - needed_mb=$((data_size_mb * 2 + 5000)) - echo " Data size: ${data_size_mb} MB" - echo " Available space: ${avail_mb} MB" - echo " Estimated need: ${needed_mb} MB" - if [ "$avail_mb" -lt "$needed_mb" ]; then - warn "Disk space may be insufficient." - warn "pg_upgrade copies data; need ~2x data size + ~5 GB for the upgrade tarball." - confirm "Continue anyway?" - fi - - # Incompatible extensions - info "Checking for incompatible extensions" - local incompatible - incompatible=$(run_sql_on "$DB_CONTAINER" -A -t -c " - SELECT string_agg(extname, ', ') - FROM pg_extension - WHERE extname IN ('timescaledb', 'plv8', 'plcoffee', 'plls'); - " 2>/dev/null | tr -d '[:space:]') || true - - if [ -n "$incompatible" ]; then - warn "Incompatible extensions found: $incompatible" - warn "These do not exist in Postgres 17 and must be dropped before upgrading." - warn "If you proceed, they will be dropped automatically." - warn "The original data is preserved as a backup so you can roll back." - confirm "Drop these extensions and continue with the upgrade?" - drop_extensions="$incompatible" - fi - - echo "" - echo "This script will:" - echo " 1. Pull the Postgres 17 image" - echo " 2. Build an upgrade tarball from the image (~1.2 GB compressed, temporary)" - echo " 3. Stop all Supabase services" - echo " 4. Run pg_upgrade (Postgres 15 -> 17)" - echo " 5. Apply post-upgrade patches" - echo " 6. Start Supabase with Postgres 17" - echo " 7. Apply additional migrations" - echo "" - echo " Current image: $current_image" - echo " Target image: $PG17_TARGET_IMAGE" - echo " Upgrade image: $PG17_UPGRADE_IMAGE" - echo " Data directory: $DATA_DIR" - echo " Backup location: $BACKUP_DIR" - echo "" - confirm "Proceed with the upgrade?" -} - -# --- Step 1: Pull Postgres 17 image ---------------------------------------- - -pull_image() { - info "Pulling Postgres 17 images" - docker pull "$PG17_UPGRADE_IMAGE" - if [ "$PG17_TARGET_IMAGE" != "$PG17_UPGRADE_IMAGE" ]; then - docker pull "$PG17_TARGET_IMAGE" - fi -} - -# --- Step 2: Build upgrade tarball ----------------------------------------- -# -# Extracts PG 17 binaries, libraries, share data, and upgrade scripts from -# the PG 17 Docker image into a tarball that initiate.sh can consume. -# -# The tarball uses the "non-nix" layout (17/bin, 17/lib, 17/share - no -# nix_flake_version file), so initiate.sh sets LD_LIBRARY_PATH to find -# the bundled libraries. - -build_tarball() { - local tmpbase="${TMPDIR:-/tmp}" - staging_dir=$(mktemp -d "${tmpbase%/}/supabase-pg17-upgrade.XXXXXX") - # World-writable so Docker containers can write to bind mounts on macOS, - # where the VM's root user has no special access to host directories. - chmod 777 "$staging_dir" - echo " Staging directory: $staging_dir" - - # Download upgrade scripts from the supabase/postgres repo (pinned to PG17_SCRIPTS_REF). - # These are no longer bundled in the latest PG 17 Docker images. - info "Downloading upgrade scripts (ref: $PG17_SCRIPTS_REF)" - local scripts_base="https://raw.githubusercontent.com/supabase/postgres/${PG17_SCRIPTS_REF}/ansible/files/admin_api_scripts/pg_upgrade_scripts" - mkdir -p "$staging_dir/scripts" - for script in initiate.sh complete.sh common.sh pgsodium_getkey.sh check.sh prepare.sh; do - curl -fsSL "$scripts_base/$script" -o "$staging_dir/scripts/$script" \ - || die "Failed to download $script from GitHub" - done - - if [ -f "$TARBALL_CACHE" ]; then - info "Using cached upgrade tarball: $TARBALL_CACHE" - cp "$TARBALL_CACHE" "$staging_dir/pg_upgrade_bin.tar.gz" - return - fi - - info "Building upgrade tarball from Postgres 17 image (first run)" - docker run --rm --user root --entrypoint bash \ - -v "$staging_dir:/export" \ - "$PG17_UPGRADE_IMAGE" \ - -c ' - set -euo pipefail - mkdir -p /export/17/bin /export/17/lib /export/17/share - - echo " Copying binaries..." - # Binaries in the nix profile are either ELF binaries or shell - # wrappers that exec a .xxx-wrapped ELF from the nix store. - # Extract the actual ELF binaries so they work outside nix. - BIN_DIR=$(dirname $(readlink -f /usr/lib/postgresql/bin/postgres)) - for f in "$BIN_DIR"/*; do - name=$(basename "$f") - - # Skip nix wrapper-internal files - case "$name" in .*-wrapped) continue ;; esac - - # Check for ELF - if [ -x "$f" ] && file -b "$f" | grep -q "ELF .* executable"; then - cp "$f" /export/17/bin/"$name" - else - # Shell wrapper - extract the real .xxx-wrapped ELF path - wrapped=$(grep -o "/nix/store/[^ \"]*-wrapped" "$f" 2>/dev/null | head -n 1 || true) - if [ -n "$wrapped" ] && [ -f "$wrapped" ]; then - cp "$wrapped" /export/17/bin/"$name" - else - cp "$f" /export/17/bin/"$name" - fi - fi - done - - echo " Copying libraries..." - PKGLIBDIR=$(pg_config --pkglibdir) - LIBDIR=$(pg_config --libdir) - - # These paths may overlap (PKGLIBDIR and LIBDIR often point to the - # same nix store path). Use cp -Lf to handle overwrites from - # read-only nix store source files. - cp -Lf "$PKGLIBDIR"/*.so /export/17/lib/ || echo " Warning: cp from $PKGLIBDIR failed" >&2 - cp -Lf "$LIBDIR"/*.so* /export/17/lib/ || echo " Warning: cp from $LIBDIR failed" >&2 - cp -Lf /nix/var/nix/profiles/default/lib/*.so* /export/17/lib/ || echo " Warning: cp from nix profile lib failed" >&2 - - echo " Copying share data..." - - # Nix-built binaries resolve share dir relative to their location: - # /../share/postgresql/ - # so we need share/postgresql/ not just share/ - mkdir -p /export/17/share/postgresql - - # Remove cyclic symlink (timezonesets/timezonesets -> timezonesets). - rm -f /usr/share/postgresql/timezonesets/timezonesets 2>/dev/null || true - - # Pre-create subdirectories so cp -rL does not need to mkdir them. - # On macOS with Docker Desktop bind mount rejects mkdir with the nix store - # read-only (dr-xr-xr-x) permissions; pre-creating with default - # writable permissions fixed this - mkdir -p /export/17/share/postgresql/{extension,timezonesets,tsearch_data} - mkdir -p /export/17/share/postgresql/extension/{functions,procedures,tables,types} - - cp -rL /usr/share/postgresql/* /export/17/share/postgresql/ || echo " Warning: cp share data had errors" >&2 - - # initiate.sh copies .control/.sql from PGLIBNEW to PGSHARENEW/extension/ - echo " Copying extension definitions to lib..." - SHAREDIR=$(pg_config --sharedir) - cp "$SHAREDIR"/extension/*.control /export/17/lib/ || echo " Warning: cp .control from $SHAREDIR/extension failed" >&2 - cp "$SHAREDIR"/extension/*.sql /export/17/lib/ || echo " Warning: cp .sql from $SHAREDIR/extension failed" >&2 - - # Verify critical files before creating tarball - echo " Checking for key files..." - [ -f /export/17/bin/postgres ] || { echo "Error: bin/postgres missing"; exit 1; } - [ -f /export/17/share/postgresql/timezonesets/Default ] || { echo "Error: timezonesets/Default missing"; exit 1; } - ls /export/17/share/postgresql/extension/*.control >/dev/null 2>&1 || { echo "Error: no .control files in extension/"; exit 1; } - ls /export/17/lib/*.so >/dev/null 2>&1 || { echo "Error: no .so files in lib/"; exit 1; } - - echo " Creating tarball (this may take several minutes)..." - cd /export && tar czf pg_upgrade_bin.tar.gz 17/ - - echo " Tarball: $(du -sh /export/pg_upgrade_bin.tar.gz | cut -f1)" - ' - - # Cache for next run - cp "$staging_dir/pg_upgrade_bin.tar.gz" "$TARBALL_CACHE" - info "Tarball cached at $TARBALL_CACHE" -} - -# --- Step 3: Drop incompatible extensions ---------------------------------- - -drop_incompatible_extensions() { - if [ -z "$drop_extensions" ]; then - return - fi - info "Dropping incompatible extensions" - - local ext - echo "$drop_extensions" | tr ',' '\n' | while read -r ext; do - ext=$(echo "$ext" | tr -d '[:space:]') - [ -z "$ext" ] && continue - echo " DROP EXTENSION $ext CASCADE" - run_sql_on "$DB_CONTAINER" -c "DROP EXTENSION IF EXISTS \"$ext\" CASCADE;" - done -} - -# --- Step 4: Stop services and back up ------------------------------------- - -stop_and_backup() { - info "Backing up pgsodium root key" - local key_backup="./volumes/db/pgsodium_root.key.bak.pg15" - docker run --rm -v "${db_config_vol}:/src:ro" -v "$(pwd)/volumes/db:/dst" \ - alpine cp /src/pgsodium_root.key /dst/pgsodium_root.key.bak.pg15 \ - || die "Failed to back up pgsodium root key from db-config volume." - echo " Saved to: $key_backup" - - info "Stopping all Supabase services" - docker compose down - - echo " Original data will be preserved as: $BACKUP_DIR" -} - -# --- Step 5: Run pg_upgrade via initiate.sh + complete.sh ------------------ -# -# Host directories are mounted at non-standard paths (/mnt/host-*) with -# symlinks at the paths the upgrade scripts expect. This lets complete.sh's -# CI wrapper (which does rm/mv/ln on /var/lib/postgresql/data and -# /data_migration) operate on symlinks rather than bind mounts. - -run_upgrade() { - local abs_data_dir abs_migration_dir - - mkdir -p "$MIGRATION_DIR" - # World-writable for macOS Docker bind mount compatibility (see build_tarball) - chmod 777 "$MIGRATION_DIR" - abs_data_dir=$(cd "$DATA_DIR" && pwd) - abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) - - info "Starting upgrade container" - docker run -d --name "$UPGRADE_CONTAINER" \ - --entrypoint sleep \ - -v "${abs_data_dir}:/mnt/host-pgdata" \ - -v "${abs_migration_dir}:/mnt/host-migration" \ - -v "${db_config_vol}:/etc/postgresql-custom" \ - -v "${staging_dir}:/tmp/staging:ro" \ - -e PGPASSWORD="$pg_password" \ - "$current_image" \ - infinity - - info "Preparing upgrade environment" - docker exec "$UPGRADE_CONTAINER" bash -c ' - # Symlink bind mounts to the paths the upgrade scripts expect - rm -rf /var/lib/postgresql/data - ln -s /mnt/host-pgdata /var/lib/postgresql/data - ln -s /mnt/host-migration /data_migration - - mkdir -p /tmp/persistent /tmp/upgrade /tmp/pg_upgrade - cp /tmp/staging/pg_upgrade_bin.tar.gz /tmp/persistent/ - cp /tmp/staging/scripts/*.sh /tmp/upgrade/ - chmod +x /tmp/upgrade/*.sh - - # Patch CI_start_postgres to use "restart" instead of "start" so it - # is idempotent (initiate.sh starts postgres for top-level queries, - # then handle_extensions calls CI_start_postgres again) - sed -i "s/pg_ctl start -o/pg_ctl restart -o/g" /tmp/upgrade/common.sh - - # Patch PGSHARENEW to match nix binary expectations (share/postgresql/) - sed -i "s|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share\"|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share/postgresql\"|" /tmp/upgrade/initiate.sh - ' - - info "Starting Postgres 15 in upgrade container" - docker exec "$UPGRADE_CONTAINER" bash -c ' - su postgres -c "pg_ctl start -o \"-c config_file=/etc/postgresql/postgresql.conf\" -l /tmp/postgres.log" - ' - wait_for_healthy "$UPGRADE_CONTAINER" - - # initiate.sh expects the PG 17 binaries tarball at /tmp/persistent/pg_upgrade_bin.tar.gz - # (hardcoded path - copied there during container setup above). - # - # Env vars for the unwrapped nix ELF binaries in the tarball: - # LD_LIBRARY_PATH - find libpq, libssl, etc. (RUNPATH points to absent nix store paths) - # NIX_PGLIBDIR - postgres uses this to find extension .so files - - info "Running initiate.sh (pg_upgrade: Postgres 15 -> 17)" - echo " This may take several minutes depending on database size..." - echo "" - if ! docker exec \ - -e IS_CI=true \ - -e PG_MAJOR_VERSION=17 \ - -e PGPASSWORD="$pg_password" \ - -e LD_LIBRARY_PATH=/tmp/pg_upgrade_bin/17/lib \ - -e NIX_PGLIBDIR=/tmp/pg_upgrade_bin/17/lib \ - "$UPGRADE_CONTAINER" \ - /tmp/upgrade/initiate.sh 17; then - echo "" - warn "initiate.sh failed. Its cleanup may have restored the original state" - warn "(re-enabled extensions, revoked superuser). Your data directory is" - warn "unchanged - no data was moved or deleted." - warn "" - warn "Check the output above for the root cause, fix it, and re-run." - docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true - die "initiate.sh failed" - fi - - info "initiate.sh completed successfully" - docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true -} - -# --- Step 6: Run complete.sh in a native PG 17 container ------------------- -# -# complete.sh applies post-upgrade patches (pg_net grants, vault re-encryption, -# pg_cron, predefined roles, vacuumdb, etc.). We run it in a PG17 container -# where the binaries are native - no nix extraction or LD_LIBRARY_PATH needed. - -run_complete() { - local abs_migration_dir - - abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) - - info "Starting PG17 container for complete.sh" - docker run -d --name "$COMPLETE_CONTAINER" \ - --entrypoint sleep \ - -v "${abs_migration_dir}:/mnt/host-migration" \ - -v "${db_config_vol}:/etc/postgresql-custom" \ - -v "${staging_dir}:/tmp/staging:ro" \ - -e PGPASSWORD="$pg_password" \ - "$PG17_UPGRADE_IMAGE" \ - infinity - - info "Preparing complete.sh environment" - # Save original db-config ownership so we can restore it if complete.sh fails. - # complete.sh needs PG 17 ownership to start postgres, but if it fails the - # user needs to fall back to PG 15 which uses a different uid. - docker exec "$COMPLETE_CONTAINER" bash -c ' - stat -c "%u:%g" /etc/postgresql-custom/pgsodium_root.key 2>/dev/null > /tmp/dbconfig_owner || true - ' - - docker exec "$COMPLETE_CONTAINER" bash -c ' - # Symlink bind mount so complete.sh CI wrapper can mv/rm/ln - ln -s /mnt/host-migration /data_migration - - # Remove the image default data dir (complete.sh creates a symlink here) - rm -rf /var/lib/postgresql/data - - # Fix ownership on db-config volume (PG15 uid differs from PG17) - chown -R postgres:postgres /etc/postgresql-custom/ - - # PG17 config includes this directory; may not exist from PG15 - mkdir -p /etc/postgresql-custom/conf.d - - mkdir -p /tmp/upgrade - - # Copy upgrade scripts - cp /tmp/staging/scripts/*.sh /tmp/upgrade/ - chmod +x /tmp/upgrade/*.sh - - # Patch --new-bin to use native bindir (we are in a PG17 container, - # no need for /tmp/pg_upgrade_bin/ paths) - sed -i "s|BINDIR=\"/tmp/pg_upgrade_bin/\$PG_MAJOR_VERSION/bin\"|BINDIR=\$(pg_config --bindir)|g" /tmp/upgrade/common.sh - ' - - info "Running complete.sh (post-upgrade patches, vacuum analyze)" - docker exec \ - -e IS_CI=true \ - -e PG_MAJOR_VERSION=17 \ - -e PGPASSWORD="$pg_password" \ - "$COMPLETE_CONTAINER" \ - /tmp/upgrade/complete.sh || true - - # complete.sh's ERR trap exits with 0 in some cases; check status file - local status - status=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/pg-upgrade-status 2>/dev/null || echo "unknown") - if [ "$status" != "complete" ]; then - warn "complete.sh failed. Postgres log:" - docker exec "$COMPLETE_CONTAINER" cat /tmp/postgres.log 2>/dev/null || true - echo "" - # Restore db-config ownership so PG 15 can start for rollback - warn "Restoring db-config ownership for PG15..." - local orig_owner - orig_owner=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/dbconfig_owner 2>/dev/null || true) - if [ -n "$orig_owner" ]; then - docker exec "$COMPLETE_CONTAINER" chown -R "$orig_owner" /etc/postgresql-custom/ 2>/dev/null || true - fi - docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true - echo "" - echo " Your Postgres 15 data is unchanged (data swap has not happened yet)." - echo " To restart Postgres 15:" - echo " rm -rf $MIGRATION_DIR" - echo " docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" - echo "" - die "complete.sh failed (status: $status)" - fi - - info "complete.sh finished successfully" - docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true -} - -# --- Step 7: Swap data directories ----------------------------------------- - -swap_data() { - info "Swapping data directories" - - echo " $DATA_DIR -> $BACKUP_DIR" - mv "$DATA_DIR" "$BACKUP_DIR" - - echo " $MIGRATION_DIR/pgdata -> $DATA_DIR" - mv "$MIGRATION_DIR/pgdata" "$DATA_DIR" - rm -rf "$MIGRATION_DIR" -} - -# --- Step 8: Start Postgres 17 --------------------------------------------- - -start_pg17() { - info "Starting Supabase with Postgres 17" - - # Ensure db-config volume has correct ownership and structure for PG 17. - # complete.sh does this too, but just in case of partial - # failures from previous runs. - docker run --rm -v "${db_config_vol}:/vol" "$PG17_TARGET_IMAGE" sh -c ' - mkdir -p /vol/conf.d - chown -R postgres:postgres /vol/ - ' - - docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d - - echo " Waiting for Postgres 17 to be ready..." - local retries=60 - while [ $retries -gt 0 ]; do - if docker exec "$DB_CONTAINER" pg_isready -U postgres -h localhost >/dev/null 2>&1; then - break - fi - retries=$((retries - 1)) - sleep 2 - done - [ $retries -gt 0 ] || die "Postgres 17 did not start within 120 seconds." - - local new_version - new_version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SHOW server_version;" 2>/dev/null | head -n 1) - echo " Postgres version: $new_version" - case "$new_version" in - 17.*) ;; - *) die "Expected Postgres 17.x, got: $new_version" ;; - esac -} - -# --- Step 9: Apply migrations not covered by complete.sh ------------------- -# -# These PG 17 migrations run on fresh installs via initdb but not after -# pg_upgrade (init scripts don't rerun when PG_VERSION already exists). -# complete.sh doesn't cover them either. On the platform a tracked migration -# runner (dbmate) applies them; self-hosted has no such tracking, so we apply -# the idempotent, safe-on-an-existing-DB ones directly here. -# -# Source: postgres/migrations/db/migrations/ -# - 20250710151649_supabase_read_only_user_default_transaction_read_only.sql -# - 20251001204436_predefined_role_grants.sql (supabase_etl_admin + pg_monitor) -# - 20251105172723_grant_pg_reload_conf_to_postgres.sql -# - 20251121132723_correct_search_path_pgbouncer.sql -# - 20260211120934_supabase_privileged_role.sql (target image's supautils.conf -# references this role; it must exist or supautils config is broken) -# - 20260413000000_fix-authenticator-session-preload-libraries.sql -# - 20260421000001_rescope_pg_graphql_access_trigger.sql -# -# NOT applied: -# - 20260421000000_pg_graphql-off-by-default.sql: drops pg_graphql. Safe on a -# fresh install, but destructive on an existing DB that uses pg_graphql. -# -# This step also reconciles extension versions (complete.sh ran with the .063 -# binaries) and the pg_cron 1.6 -> 1.6.4 version label. See below. - -apply_role_migrations() { - info "Applying Postgres 17 migrations" - - # Fix collation version mismatch first (upgrade used glibc 2.39, target - # image may use glibc 2.40). Do this before any other SQL to suppress - # the noisy warnings on every subsequent command. - for db in postgres template1 _supabase; do - docker exec -i -e PGPASSWORD="$pg_password" "$DB_CONTAINER" \ - psql -h localhost -U supabase_admin -d "$db" \ - -c "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION;" || true - done - - # Create supabase_etl_admin role (doesn't exist in PG 15 images). - # Must be created before running predefined_role_grants.sql which - # assumes it exists. - run_sql_on "$DB_CONTAINER" -c " - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'supabase_etl_admin') THEN - CREATE USER supabase_etl_admin WITH LOGIN REPLICATION; - GRANT pg_read_all_data TO supabase_etl_admin; - GRANT CREATE ON DATABASE postgres TO supabase_etl_admin; - END IF; - END - \$\$;" || true - - # Run the migration files directly from the PG 17 container image. - # They're idempotent (IF EXISTS / IF NOT EXISTS guards). - local migration_dir="/docker-entrypoint-initdb.d/migrations" - local migrations=" - 20250710151649_supabase_read_only_user_default_transaction_read_only.sql - 20251001204436_predefined_role_grants.sql - 20251105172723_grant_pg_reload_conf_to_postgres.sql - 20251121132723_correct_search_path_pgbouncer.sql - 20260211120934_supabase_privileged_role.sql - 20260413000000_fix-authenticator-session-preload-libraries.sql - 20260421000001_rescope_pg_graphql_access_trigger.sql - " - - for m in $migrations; do - echo " Running: $m" - docker exec -i \ - -e PGPASSWORD="$pg_password" \ - "$DB_CONTAINER" \ - psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 \ - -f "${migration_dir}/${m}" || warn " $m failed (non-fatal)" - done - - # Reconcile extension versions to the target image. pg_upgrade generates - # update_extensions.sql, but complete.sh runs it inside the .063 container, so - # extensions only get updated to .063's versions. The final image ships newer - # .so files, leaving the catalog version behind what's loaded. - # - # Mirror what update_extensions.sql does, but against the running target image - # and for every installed extension: ALTER EXTENSION ... UPDATE brings each one - # up to the image's default version. This is generic on purpose - it tracks the - # image (matching platform behavior) and stays correct across future image - # bumps without maintaining a hardcoded list. ALTER ... UPDATE is a no-op when - # already at the default. Per-extension exceptions are caught so one extension - # with no update path (e.g. pg_cron 1.6 -> 1.6.4, handled below) does not abort - # the rest. - info "Reconciling extension versions to the target image" - run_sql_on "$DB_CONTAINER" -c " - DO \$\$ - DECLARE r record; - BEGIN - FOR r IN SELECT extname FROM pg_extension LOOP - BEGIN - EXECUTE format('ALTER EXTENSION %I UPDATE', r.extname); - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'skipped extension %: %', r.extname, SQLERRM; - END; - END LOOP; - END - \$\$;" || warn "extension version reconcile had errors (non-fatal)" - - # pg_cron: PG 15 registers the extension as '1.6'; PG 17 images package it as - # '1.6.4' with no '1.6' -> '1.6.4' update path, so ALTER EXTENSION ... UPDATE - # has no path to follow. complete.sh only fixes this when pg_cron is owned by - # 'postgres' (it drops + recreates). Manually-enabled instances are often - # owned by supabase_admin, where that path is skipped. Reconcile the catalog - # label directly: the loaded .so and the SQL objects are already - # 1.6.4-equivalent (no SQL delta between 1.6 and 1.6.4). No DROP, so any - # scheduled jobs are untouched. - run_sql_on "$DB_CONTAINER" -c " - DO \$\$ - DECLARE want text; - BEGIN - SELECT default_version INTO want FROM pg_available_extensions WHERE name = 'pg_cron'; - IF want IS NOT NULL - AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron' AND extversion <> want) THEN - UPDATE pg_extension SET extversion = want WHERE extname = 'pg_cron'; - END IF; - END - \$\$;" || warn "pg_cron version reconcile had errors (non-fatal)" -} - -# --- Step 10: Verify ------------------------------------------------------ - -verify() { - info "Verification" - - local version - version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SELECT version();" 2>/dev/null | head -n 1) - echo " $version" - - echo "" - echo " Extensions:" - run_sql_on "$DB_CONTAINER" -c \ - "SELECT extname, extversion FROM pg_extension ORDER BY extname;" - - echo "" - info "Upgrade complete!" - echo "" - echo " To use Postgres 17 going forward, always include the override:" - echo " docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d" - echo "" - echo " Postgres 15 backup: $BACKUP_DIR" - echo " pgsodium key backup: ./volumes/db/pgsodium_root.key.bak.pg15" - echo " Once satisfied, you can reclaim space:" - echo " rm -rf $BACKUP_DIR ./volumes/db/pg17_upgrade_bin_*.tar.gz" - echo "" - echo " Rollback (if needed):" - echo " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" - echo " 2. rm -rf $DATA_DIR" - echo " 3. mv $BACKUP_DIR $DATA_DIR" - echo " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" - echo " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" - echo "" -} - -# --- Main ------------------------------------------------------------------- - -main() { - echo "" - echo "Supabase Self-Hosted: Postgres 15 -> 17 Upgrade" - echo "================================================" - - preflight - pull_image - build_tarball - drop_incompatible_extensions - stop_and_backup - run_upgrade - run_complete - swap_data - start_pg17 - apply_role_migrations - verify -} - -main "$@" diff --git a/supabase/volumes/api/envoy/cds.yaml b/supabase/volumes/api/envoy/cds.yaml deleted file mode 100644 index 2d47842d6b..0000000000 --- a/supabase/volumes/api/envoy/cds.yaml +++ /dev/null @@ -1,223 +0,0 @@ -resources: - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: auth - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: auth - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: auth - port_value: 9999 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: /health - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: rest - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: rest - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: rest - port_value: 3000 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: / - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: realtime - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: realtime - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: realtime-dev.supabase-realtime - port_value: 4000 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: / - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: storage - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: storage - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: storage - port_value: 5000 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: /status - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: functions - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: functions - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: functions - port_value: 9000 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - tcp_health_check: {} - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: meta - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: meta - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: meta - port_value: 8080 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: /health - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - - - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - name: studio - connect_timeout: 5s - type: STRICT_DNS - dns_refresh_rate: 5s - dns_failure_refresh_rate: - base_interval: 1s - max_interval: 1s - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: studio - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: studio - port_value: 3000 - health_checks: - - timeout: 2s - interval: 5s - unhealthy_threshold: 3 - healthy_threshold: 2 - http_health_check: - path: /project/default - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 diff --git a/supabase/volumes/api/envoy/docker-entrypoint.sh b/supabase/volumes/api/envoy/docker-entrypoint.sh deleted file mode 100755 index 2c67584a30..0000000000 --- a/supabase/volumes/api/envoy/docker-entrypoint.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -set -e - -# Generate SHA1 base64 hash for Envoy basic auth user list -PASSWORD_HASH=$(printf '%s' "${DASHBOARD_PASSWORD}" | openssl sha1 -binary | openssl base64) -DASHBOARD_BASIC_AUTH="${DASHBOARD_USERNAME}:{SHA}${PASSWORD_HASH}" - -echo "Generating Envoy configuration..." - -# Process the lds.yaml template with environment variables using sed -# Using | as delimiter since JWT tokens contain / -sed -e "s|\${ANON_KEY}|${ANON_KEY}|g" \ - -e "s|\${ANON_KEY_ASYMMETRIC}|${ANON_KEY_ASYMMETRIC}|g" \ - -e "s|\${SERVICE_ROLE_KEY}|${SERVICE_ROLE_KEY}|g" \ - -e "s|\${SERVICE_ROLE_KEY_ASYMMETRIC}|${SERVICE_ROLE_KEY_ASYMMETRIC}|g" \ - -e "s|\${SUPABASE_PUBLISHABLE_KEY}|${SUPABASE_PUBLISHABLE_KEY}|g" \ - -e "s|\${SUPABASE_SECRET_KEY}|${SUPABASE_SECRET_KEY}|g" \ - -e "s|\${SUPABASE_PUBLIC_URL}|${SUPABASE_PUBLIC_URL}|g" \ - -e "s|\${DASHBOARD_BASIC_AUTH}|${DASHBOARD_BASIC_AUTH}|g" \ - /etc/envoy/lds.template.yaml > /etc/envoy/lds.yaml - -if [ -n "$SUPABASE_SECRET_KEY" ] && \ - [ -n "$SUPABASE_PUBLISHABLE_KEY" ] && \ - [ -n "$SERVICE_ROLE_KEY_ASYMMETRIC" ] && \ - [ -n "$ANON_KEY_ASYMMETRIC" ]; then - echo "Envoy sb_ key translation enabled" -else - echo "Envoy running in legacy API key mode (sb_ keys disabled)" -fi - -echo "Envoy configuration generated successfully" -echo "Starting Envoy..." - -# Start Envoy -exec envoy -c /etc/envoy/envoy.yaml "$@" diff --git a/supabase/volumes/api/envoy/envoy.yaml b/supabase/volumes/api/envoy/envoy.yaml deleted file mode 100644 index def443b688..0000000000 --- a/supabase/volumes/api/envoy/envoy.yaml +++ /dev/null @@ -1,27 +0,0 @@ -dynamic_resources: - cds_config: - path_config_source: - path: /etc/envoy/cds.yaml - resource_api_version: V3 - lds_config: - path_config_source: - path: /etc/envoy/lds.yaml - resource_api_version: V3 - -node: - cluster: supabase_cluster - id: supabase_node - -overload_manager: - resource_monitors: - - name: envoy.resource_monitors.global_downstream_max_connections - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig - max_active_downstream_connections: 30000 - -admin: - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 diff --git a/supabase/volumes/api/envoy/lds.template.yaml b/supabase/volumes/api/envoy/lds.template.yaml deleted file mode 100644 index 1a3c345188..0000000000 --- a/supabase/volumes/api/envoy/lds.template.yaml +++ /dev/null @@ -1,1101 +0,0 @@ -resources: - - '@type': type.googleapis.com/envoy.config.listener.v3.Listener - name: supabase - per_connection_buffer_limit_bytes: 32768 # 32 KiB - - address: - socket_address: - address: 0.0.0.0 - port_value: 8000 - - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - normalize_path: true - merge_slashes: true - path_with_escaped_slashes_action: REJECT_REQUEST - use_remote_address: true - common_http_protocol_options: - headers_with_underscores_action: REJECT_REQUEST - upgrade_configs: - - upgrade_type: websocket - access_log: - - name: envoy.access_loggers.stdout - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog - log_format: - text_format_source: - inline_string: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% - - [%START_TIME(%d/%b/%Y:%H:%M:%S %z)%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %BYTES_SENT% \"%REQ(REFERER)%\" \"%REQ(USER-AGENT)%\"\n" - - route_config: - name: supabase_route - virtual_hosts: - - name: supabase_host - domains: - - '*' - cors: - allow_origin_string_match: - - safe_regex: - regex: ".*" - allow_methods: "GET,POST,PUT,PATCH,DELETE,OPTIONS,HEAD,CONNECT,TRACE" - allow_headers: "*" - expose_headers: "*" - max_age: "3600" - request_headers_to_add: - - header: - key: X-Forwarded-Host - value: "%REQ(:AUTHORITY)%" - append_action: ADD_IF_ABSENT - - header: - key: X-Forwarded-Port - value: "%DOWNSTREAM_LOCAL_PORT%" - append_action: ADD_IF_ABSENT - routes: - - match: - prefix: /auth/v1/verify - route: - cluster: auth - prefix_rewrite: /verify - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/verify - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /auth/v1/callback - route: - cluster: auth - prefix_rewrite: /callback - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/callback - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /auth/v1/authorize - route: - cluster: auth - prefix_rewrite: /authorize - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/authorize - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /auth/v1/.well-known/jwks.json - route: - cluster: auth - prefix_rewrite: /.well-known/jwks.json - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/.well-known/jwks.json - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /.well-known/oauth-authorization-server - route: - cluster: auth - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /.well-known/oauth-authorization-server - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /auth/v1/sso/saml/acs - route: - cluster: auth - prefix_rewrite: /sso/saml/acs - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/sso/saml/acs - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /auth/v1/sso/saml/metadata - route: - cluster: auth - prefix_rewrite: /sso/saml/metadata - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/sso/saml/metadata - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - name: functions-v1-all - match: - prefix: /functions/v1/ - route: - cluster: functions - prefix_rewrite: / - timeout: 150s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /functions/v1/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /storage/v1/ - route: - cluster: storage - prefix_rewrite: / - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /storage/v1 - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - - name: auth-v1-protected - match: - prefix: /auth/v1/ - route: - cluster: auth - prefix_rewrite: / - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /auth/v1/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - - - name: rest-v1-openapi-protected - match: - path: /rest/v1/ - route: - cluster: rest - prefix_rewrite: / - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /rest/v1/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - admin: - permissions: - - any: true - principals: - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY}' - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' - - - name: rest-v1-protected - match: - prefix: /rest/v1/ - route: - cluster: rest - prefix_rewrite: / - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /rest/v1/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - - - name: graphql-v1-protected - match: - prefix: /graphql/v1 - route: - cluster: rest - prefix_rewrite: /rpc/graphql - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /graphql/v1 - append_action: ADD_IF_ABSENT - - header: - key: Content-Profile - value: graphql_public - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - - - name: realtime-v1-api-openapi-blocked - match: - prefix: /realtime/v1/api/openapi - route: - cluster: realtime - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /realtime/v1/api/openapi - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: DENY - policies: - deny_all: - permissions: - - any: true - principals: - - any: true - - - name: realtime-v1-api-tenants-blocked - match: - prefix: /realtime/v1/api/tenants - route: - cluster: realtime - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /realtime/v1/api/tenants - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: DENY - policies: - deny_all: - permissions: - - any: true - principals: - - any: true - - - name: realtime-v1-api-protected - match: - prefix: /realtime/v1/api - route: - cluster: realtime - prefix_rewrite: /api - timeout: 30s - host_rewrite_literal: realtime-dev.supabase-realtime - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /realtime/v1/api - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - - - name: realtime-v1-ws-protected - match: - prefix: /realtime/v1/ - route: - cluster: realtime - prefix_rewrite: /socket/ - timeout: 30s - host_rewrite_literal: realtime-dev.supabase-realtime - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /realtime/v1/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - - - name: pg-protected - match: - prefix: /pg/ - route: - cluster: meta - prefix_rewrite: / - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /pg/ - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.cors: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy - allow_origin_string_match: - - exact: '${SUPABASE_PUBLIC_URL}' - - safe_regex: - regex: 'https?://(localhost|127\.0\.0\.1)(:[0-9]+)?' - allow_methods: "GET,POST,PUT,PATCH,DELETE,OPTIONS,HEAD" - allow_headers: "*" - expose_headers: "*" - max_age: "3600" - - - match: - prefix: /api/mcp - route: - cluster: studio - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /api/mcp - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: DENY - policies: - deny_all: - permissions: - - any: true - principals: - - any: true - - - match: - prefix: /mcp - route: - cluster: studio - prefix_rewrite: /api/mcp - timeout: 30s - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: /mcp - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.basic_auth: - '@type': >- - type.googleapis.com/envoy.config.route.v3.FilterConfig - disabled: true - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - # Block access to /mcp by default - rbac: - rules: - action: DENY - policies: - deny_all: - permissions: - - any: true - principals: - - any: true - # Enable local access (danger zone!) - # 1. Comment out the 'rbac' block above. - # 2. Uncomment and adjust the 'rbac' block below. - # 3. Add or adjust your local IPs in 'principals'. - #rbac: - # rules: - # action: ALLOW - # policies: - # allow_local: - # permissions: - # - any: true - # principals: - # - direct_remote_ip: - # address_prefix: 127.0.0.1 - # prefix_len: 32 - # - direct_remote_ip: - # address_prefix: ::1 - # prefix_len: 128 - - - match: - prefix: / - route: - cluster: studio - timeout: 30s - request_headers_to_remove: - - authorization - request_headers_to_add: - - header: - key: X-Forwarded-Prefix - value: / - append_action: ADD_IF_ABSENT - typed_per_filter_config: - envoy.filters.http.rbac: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - rbac: - rules: - action: ALLOW - policies: - allow_all: - permissions: - - any: true - principals: - - any: true - - http_filters: - - name: envoy.filters.http.cors - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - - name: envoy.filters.http.basic_auth - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth - users: - inline_string: '${DASHBOARD_BASIC_AUTH}' - - # Copies ?apikey=... from the URL into the apikey header when clients omit the header. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local FUNCTIONS_ROUTE = "functions-v1-all" - local FUNCTIONS_PREFIX = "/functions/v1/" - - local function is_functions_request(request_handle, headers) - if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then - return true - end - - local path = headers:get(":path") - if path == nil then - return false - end - - return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX - end - - function envoy_on_request(request_handle) - local headers = request_handle:headers() - if is_functions_request(request_handle, headers) then - return - end - - if headers:get("apikey") ~= nil then - return - end - - local path = headers:get(":path") - local query_start = string.find(path, "?", 1, true) - if query_start == nil then - return - end - - local query = string.sub(path, query_start + 1) - for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do - if key == "apikey" and value ~= "" then - headers:add("apikey", value) - return - end - end - end - - # Returns 401 for missing/invalid API keys on protected API routes. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local ANON_KEY = "${ANON_KEY}" - local SERVICE_ROLE_KEY = "${SERVICE_ROLE_KEY}" - local SUPABASE_PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" - local SUPABASE_SECRET_KEY = "${SUPABASE_SECRET_KEY}" - local ANON_KEY_ASYMMETRIC = "${ANON_KEY_ASYMMETRIC}" - local SERVICE_ROLE_KEY_ASYMMETRIC = "${SERVICE_ROLE_KEY_ASYMMETRIC}" - local TRANSLATION_ENABLED = SUPABASE_SECRET_KEY ~= "" and SUPABASE_PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_KEY_ASYMMETRIC ~= "" and ANON_KEY_ASYMMETRIC ~= "" - - local PROTECTED_ROUTES = { - ["auth-v1-protected"] = true, - ["rest-v1-openapi-protected"] = true, - ["rest-v1-protected"] = true, - ["graphql-v1-protected"] = true, - ["realtime-v1-api-protected"] = true, - ["realtime-v1-ws-protected"] = true, - ["pg-protected"] = true, - } - - local function is_protected_route(route_name) - if route_name == nil or route_name == "" then - return false - end - - return PROTECTED_ROUTES[route_name] == true - end - - local function is_valid_apikey(apikey) - if apikey == nil or apikey == "" then - return false - end - - if SERVICE_ROLE_KEY ~= "" and apikey == SERVICE_ROLE_KEY then - return true - end - - if ANON_KEY ~= "" and apikey == ANON_KEY then - return true - end - - if TRANSLATION_ENABLED and apikey == SUPABASE_SECRET_KEY then - return true - end - - if TRANSLATION_ENABLED and apikey == SUPABASE_PUBLISHABLE_KEY then - return true - end - - return false - end - - function envoy_on_request(request_handle) - local headers = request_handle:headers() - local route_name = request_handle:streamInfo():routeName() - if not is_protected_route(route_name) then - return - end - - if is_valid_apikey(headers:get("apikey")) then - return - end - - request_handle:respond({ - [":status"] = "401", - ["content-type"] = "text/plain", - }, "Unauthorized") - end - - # Translates the query parameter apikey into the matching internal JWT and rewrites the URL so only JWTs propagate downstream. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local FUNCTIONS_ROUTE = "functions-v1-all" - local FUNCTIONS_PREFIX = "/functions/v1/" - local SECRET_KEY = "${SUPABASE_SECRET_KEY}" - local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" - local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" - local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" - local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" - - local function is_functions_request(request_handle, headers) - if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then - return true - end - - local path = headers:get(":path") - if path == nil then - return false - end - - return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX - end - - local function translate_apikey(apikey) - if apikey == nil or apikey == "" then - return nil - end - - if not TRANSLATION_ENABLED then - return nil - end - - if apikey == SECRET_KEY then - return SERVICE_ROLE_JWT - end - - if apikey == PUBLISHABLE_KEY then - return ANON_JWT - end - - return nil - end - - local function extract_query_apikey(path) - if path == nil or path == "" then - return nil - end - - local query_start = string.find(path, "?", 1, true) - if query_start == nil then - return nil - end - - local query = string.sub(path, query_start + 1) - for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do - if key == "apikey" and value ~= "" then - return value - end - end - - return nil - end - - local function replace_query_apikey(path, new_value) - if path == nil or path == "" or new_value == nil or new_value == "" then - return nil - end - - local query_start = string.find(path, "?", 1, true) - if query_start == nil then - return nil - end - - local base = string.sub(path, 1, query_start) - local query = string.sub(path, query_start + 1) - local updated = {} - local replaced = false - - for part in string.gmatch(query, "([^&]+)") do - local key, value = string.match(part, "([^=]+)=(.*)") - if key == "apikey" then - part = key .. "=" .. new_value - replaced = true - end - table.insert(updated, part) - end - - if not replaced then - return nil - end - - return base .. table.concat(updated, "&") - end - - function envoy_on_request(request_handle) - local headers = request_handle:headers() - if is_functions_request(request_handle, headers) then - return - end - - local path = headers:get(":path") - local apikey = extract_query_apikey(path) - local translated = translate_apikey(apikey) - - if translated == nil then - return - end - - headers:replace("apikey", translated) - - local rewritten_path = replace_query_apikey(path, translated) - if rewritten_path ~= nil then - headers:replace(":path", rewritten_path) - end - end - - # Translates an apikey header into the appropriate internal JWT for downstream RBAC checks. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local FUNCTIONS_ROUTE = "functions-v1-all" - local FUNCTIONS_PREFIX = "/functions/v1/" - local SECRET_KEY = "${SUPABASE_SECRET_KEY}" - local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" - local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" - local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" - local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" - - local function is_functions_request(request_handle, headers) - if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then - return true - end - - local path = headers:get(":path") - if path == nil then - return false - end - - return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX - end - - local function translate_apikey(apikey) - if apikey == nil or apikey == "" then - return nil - end - - if not TRANSLATION_ENABLED then - return nil - end - - if apikey == SECRET_KEY then - return SERVICE_ROLE_JWT - end - - if apikey == PUBLISHABLE_KEY then - return ANON_JWT - end - - return nil - end - - function envoy_on_request(request_handle) - local headers = request_handle:headers() - if is_functions_request(request_handle, headers) then - return - end - - local translated = translate_apikey(headers:get("apikey")) - if translated ~= nil and translated ~= "" then - headers:replace("apikey", translated) - end - end - - # Mirrors apikey into x-api-key for realtime WS compatibility. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" - - function envoy_on_request(request_handle) - local route_name = request_handle:streamInfo():routeName() - if route_name ~= REALTIME_WS_ROUTE then - return - end - - local headers = request_handle:headers() - local apikey = headers:get("apikey") - if apikey == nil or apikey == "" then - return - end - - headers:replace("x-api-key", apikey) - end - - # Synthesizes an Authorization header (Bearer …) from apikey when callers don’t provide a real JWT header. - - name: envoy.filters.http.lua - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - local FUNCTIONS_ROUTE = "functions-v1-all" - local FUNCTIONS_PREFIX = "/functions/v1/" - local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" - - local function is_functions_request(request_handle, headers) - if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then - return true - end - - local path = headers:get(":path") - if path == nil then - return false - end - - return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX - end - - local function has_real_jwt(auth_header) - if auth_header == nil or auth_header == "" then - return false - end - - if string.sub(auth_header, 1, 7) ~= "Bearer " then - return false - end - - return string.sub(auth_header, 1, 10) ~= "Bearer sb_" - end - - local function format_authorization(value) - if value == nil or value == "" then - return nil - end - - if string.sub(value, 1, 7) == "Bearer " then - return value - end - - return "Bearer " .. value - end - - function envoy_on_request(request_handle) - local headers = request_handle:headers() - if request_handle:streamInfo():routeName() == REALTIME_WS_ROUTE then - return - end - - if is_functions_request(request_handle, headers) then - return - end - - if has_real_jwt(headers:get("authorization")) then - return - end - - local apikey = headers:get("apikey") - local authorization_value = format_authorization(apikey) - if authorization_value ~= nil then - headers:replace("authorization", authorization_value) - end - end - - - name: envoy.filters.http.rbac - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC - rules: - action: ALLOW - policies: - admin: - permissions: - - url_path: - path: - prefix: /pg/ - principals: - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY}' - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' - apikey: - permissions: - - url_path: - path: - prefix: /auth/v1/ - - url_path: - path: - prefix: /rest/v1/ - - url_path: - path: - prefix: /realtime/v1/api - - url_path: - path: - prefix: /realtime/v1/ - - url_path: - path: - prefix: /graphql/v1 - principals: - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY}' - - header: - name: apikey - string_match: - exact: '${ANON_KEY}' - - header: - name: apikey - string_match: - exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' - - header: - name: apikey - string_match: - exact: '${ANON_KEY_ASYMMETRIC}' - - name: envoy.filters.http.router - typed_config: - '@type': >- - type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/supabase/volumes/db/_supabase.sql b/supabase/volumes/db/_supabase.sql deleted file mode 100644 index 6236ae1bcc..0000000000 --- a/supabase/volumes/db/_supabase.sql +++ /dev/null @@ -1,3 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -CREATE DATABASE _supabase WITH OWNER :pguser; diff --git a/supabase/volumes/db/jwt.sql b/supabase/volumes/db/jwt.sql deleted file mode 100644 index 09a7283666..0000000000 --- a/supabase/volumes/db/jwt.sql +++ /dev/null @@ -1,3 +0,0 @@ -\set jwt_exp `echo "$JWT_EXP"` - -ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; diff --git a/supabase/volumes/db/logs.sql b/supabase/volumes/db/logs.sql deleted file mode 100644 index 255c0f407b..0000000000 --- a/supabase/volumes/db/logs.sql +++ /dev/null @@ -1,6 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -\c _supabase -create schema if not exists _analytics; -alter schema _analytics owner to :pguser; -\c postgres diff --git a/supabase/volumes/db/pooler.sql b/supabase/volumes/db/pooler.sql deleted file mode 100644 index 162c5b96aa..0000000000 --- a/supabase/volumes/db/pooler.sql +++ /dev/null @@ -1,6 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -\c _supabase -create schema if not exists _supavisor; -alter schema _supavisor owner to :pguser; -\c postgres diff --git a/supabase/volumes/db/realtime.sql b/supabase/volumes/db/realtime.sql deleted file mode 100644 index 4d4b9ffb91..0000000000 --- a/supabase/volumes/db/realtime.sql +++ /dev/null @@ -1,4 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -create schema if not exists _realtime; -alter schema _realtime owner to :pguser; diff --git a/supabase/volumes/db/roles.sql b/supabase/volumes/db/roles.sql deleted file mode 100644 index 8f7161a6db..0000000000 --- a/supabase/volumes/db/roles.sql +++ /dev/null @@ -1,8 +0,0 @@ --- NOTE: change to your own passwords for production environments -\set pgpass `echo "$POSTGRES_PASSWORD"` - -ALTER USER authenticator WITH PASSWORD :'pgpass'; -ALTER USER pgbouncer WITH PASSWORD :'pgpass'; -ALTER USER supabase_auth_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_functions_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; diff --git a/supabase/volumes/db/webhooks.sql b/supabase/volumes/db/webhooks.sql deleted file mode 100644 index fe273decb8..0000000000 --- a/supabase/volumes/db/webhooks.sql +++ /dev/null @@ -1,212 +0,0 @@ -BEGIN; - -- Create pg_net extension - CREATE EXTENSION IF NOT EXISTS pg_net SCHEMA extensions; - -- Create supabase_functions schema - CREATE SCHEMA supabase_functions AUTHORIZATION supabase_admin; - GRANT USAGE ON SCHEMA supabase_functions TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON TABLES TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON FUNCTIONS TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON SEQUENCES TO postgres, anon, authenticated, service_role; - -- supabase_functions.migrations definition - CREATE TABLE supabase_functions.migrations ( - version text PRIMARY KEY, - inserted_at timestamptz NOT NULL DEFAULT NOW() - ); - -- Initial supabase_functions migration - INSERT INTO supabase_functions.migrations (version) VALUES ('initial'); - -- supabase_functions.hooks definition - CREATE TABLE supabase_functions.hooks ( - id bigserial PRIMARY KEY, - hook_table_id integer NOT NULL, - hook_name text NOT NULL, - created_at timestamptz NOT NULL DEFAULT NOW(), - request_id bigint - ); - CREATE INDEX supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); - CREATE INDEX supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); - COMMENT ON TABLE supabase_functions.hooks IS 'Supabase Functions Hooks: Audit trail for triggered hooks.'; - CREATE FUNCTION supabase_functions.http_request() - RETURNS trigger - LANGUAGE plpgsql - AS $function$ - DECLARE - request_id bigint; - payload jsonb; - url text := TG_ARGV[0]::text; - method text := TG_ARGV[1]::text; - headers jsonb DEFAULT '{}'::jsonb; - params jsonb DEFAULT '{}'::jsonb; - timeout_ms integer DEFAULT 1000; - BEGIN - IF url IS NULL OR url = 'null' THEN - RAISE EXCEPTION 'url argument is missing'; - END IF; - - IF method IS NULL OR method = 'null' THEN - RAISE EXCEPTION 'method argument is missing'; - END IF; - - IF TG_ARGV[2] IS NULL OR TG_ARGV[2] = 'null' THEN - headers = '{"Content-Type": "application/json"}'::jsonb; - ELSE - headers = TG_ARGV[2]::jsonb; - END IF; - - IF TG_ARGV[3] IS NULL OR TG_ARGV[3] = 'null' THEN - params = '{}'::jsonb; - ELSE - params = TG_ARGV[3]::jsonb; - END IF; - - IF TG_ARGV[4] IS NULL OR TG_ARGV[4] = 'null' THEN - timeout_ms = 1000; - ELSE - timeout_ms = TG_ARGV[4]::integer; - END IF; - - CASE - WHEN method = 'GET' THEN - SELECT http_get INTO request_id FROM net.http_get( - url, - params, - headers, - timeout_ms - ); - WHEN method = 'POST' THEN - payload = jsonb_build_object( - 'old_record', OLD, - 'record', NEW, - 'type', TG_OP, - 'table', TG_TABLE_NAME, - 'schema', TG_TABLE_SCHEMA - ); - - SELECT http_post INTO request_id FROM net.http_post( - url, - payload, - params, - headers, - timeout_ms - ); - ELSE - RAISE EXCEPTION 'method argument % is invalid', method; - END CASE; - - INSERT INTO supabase_functions.hooks - (hook_table_id, hook_name, request_id) - VALUES - (TG_RELID, TG_NAME, request_id); - - RETURN NEW; - END - $function$; - -- Supabase super admin - DO - $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_roles - WHERE rolname = 'supabase_functions_admin' - ) - THEN - CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; - END IF; - END - $$; - GRANT ALL PRIVILEGES ON SCHEMA supabase_functions TO supabase_functions_admin; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA supabase_functions TO supabase_functions_admin; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA supabase_functions TO supabase_functions_admin; - ALTER USER supabase_functions_admin SET search_path = "supabase_functions"; - ALTER table "supabase_functions".migrations OWNER TO supabase_functions_admin; - ALTER table "supabase_functions".hooks OWNER TO supabase_functions_admin; - ALTER function "supabase_functions".http_request() OWNER TO supabase_functions_admin; - GRANT supabase_functions_admin TO postgres; - -- http_request() is SECURITY DEFINER and runs as supabase_functions_admin. Serializing - -- NEW/OLD can touch objects in the extensions schema (PostGIS geometry reads - -- extensions.spatial_ref_sys), so the role needs USAGE on it. - GRANT USAGE ON SCHEMA extensions TO supabase_functions_admin; - -- Remove unused supabase_pg_net_admin role - DO - $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_roles - WHERE rolname = 'supabase_pg_net_admin' - ) - THEN - REASSIGN OWNED BY supabase_pg_net_admin TO supabase_admin; - DROP OWNED BY supabase_pg_net_admin; - DROP ROLE supabase_pg_net_admin; - END IF; - END - $$; - -- pg_net grants when extension is already enabled - DO - $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_extension - WHERE extname = 'pg_net' - ) - THEN - GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - END IF; - END - $$; - -- Event trigger for pg_net - CREATE OR REPLACE FUNCTION extensions.grant_pg_net_access() - RETURNS event_trigger - LANGUAGE plpgsql - AS $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_event_trigger_ddl_commands() AS ev - JOIN pg_extension AS ext - ON ev.objid = ext.oid - WHERE ext.extname = 'pg_net' - ) - THEN - GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - END IF; - END; - $$; - COMMENT ON FUNCTION extensions.grant_pg_net_access IS 'Grants access to pg_net'; - DO - $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_event_trigger - WHERE evtname = 'issue_pg_net_access' - ) THEN - CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end WHEN TAG IN ('CREATE EXTENSION') - EXECUTE PROCEDURE extensions.grant_pg_net_access(); - END IF; - END - $$; - INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants'); - ALTER function supabase_functions.http_request() SECURITY DEFINER; - ALTER function supabase_functions.http_request() SET search_path = supabase_functions; - REVOKE ALL ON FUNCTION supabase_functions.http_request() FROM PUBLIC; - GRANT EXECUTE ON FUNCTION supabase_functions.http_request() TO postgres, anon, authenticated, service_role; -COMMIT; diff --git a/supabase/volumes/functions/deno.jsonc b/supabase/volumes/functions/deno.jsonc deleted file mode 100644 index db206e8f46..0000000000 --- a/supabase/volumes/functions/deno.jsonc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "imports": { - "@supabase/functions-js": "jsr:@supabase/functions-js@^2", - "@supabase/server": "npm:@supabase/server@^1" - } -} diff --git a/supabase/volumes/functions/hello/index.ts b/supabase/volumes/functions/hello/index.ts deleted file mode 100644 index 50d0e6be8c..0000000000 --- a/supabase/volumes/functions/hello/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Follow this setup guide to integrate the Deno language server with your editor: -// https://deno.land/manual/getting_started/setup_your_environment -// This enables autocomplete, go to definition, etc. - -// Setup type definitions for built-in Supabase Runtime APIs -import "@supabase/functions-js/edge-runtime.d.ts" -import { withSupabase } from "@supabase/server" - -// Logs are visible from 'functions' container inspector -console.log("Hello from Functions!"); - -// This endpoint uses 'publishable' | 'secret' access, apiKey is required. -// Use publishable for Client-facing, key-validated endpoints -// Use secret for Server-to-server, internal calls -export default { - fetch: withSupabase({ auth: ["publishable", "secret"] }, async (req, ctx) => { - // Called by another service with a secret key - // ctx.supabaseAdmin bypasses RLS — use for privileged operations - /* - if (ctx.authMode === "secret") { - const { user_id } = await req.json(); - const { data } = await ctx.supabaseAdmin.auth.admin.getUserById(user_id); - - return Response.json({ - email: data?.user?.email, - }); - } - */ - - return Response.json({ message: "Hello from Edge Functions!" }); - }), -}; - -// To invoke: -// curl 'http://localhost:/functions/v1/hello' \ -// --header 'apiKey: ' - diff --git a/supabase/volumes/functions/main/index.ts b/supabase/volumes/functions/main/index.ts deleted file mode 100644 index 6bfad388a8..0000000000 --- a/supabase/volumes/functions/main/index.ts +++ /dev/null @@ -1,232 +0,0 @@ -import * as jose from 'jsr:@panva/jose@6' - -console.log('main function started') - -const JWT_SECRET = Deno.env.get('JWT_SECRET') -const SUPABASE_JWKS = parseJwks(Deno.env.get('SUPABASE_JWKS')) -const LOCAL_JWKS = SUPABASE_JWKS ? jose.createLocalJWKSet(SUPABASE_JWKS) : null -const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' - -type AuthFailure = { - code: RequestErrors - message?: string -} - -export enum RequestErrors { - InvalidLegacyJWT = 'UNAUTHORIZED_LEGACY_JWT', - InvalidAsymmetricJWT = 'UNAUTHORIZED_ASYMMETRIC_JWT', - InvalidTokenFormat = 'UNAUTHORIZED_INVALID_JWT_FORMAT', - UnsupportedTokenAlgorithm = 'UNAUTHORIZED_UNSUPPORTED_TOKEN_ALGORITHM', - MissingAuthHeader = 'UNAUTHORIZED_NO_AUTH_HEADER', -} - -// NOTE:(kallebysantos) We don't check for valid keys but just the bare array parsing, -// let this for 'jose' lib verification -export function parseJwks(raw: string | undefined): jose.JSONWebKeySet | null { - if (!raw) return null - try { - const parsed = JSON.parse(raw) - if (parsed?.keys && Array.isArray(parsed.keys)) { - return parsed as jose.JSONWebKeySet - } - return null - } catch { - return null - } -} - -/** - * Extract JWT token from Authorization header - * - * Parses the Authorization header to extract the Bearer token. - * Expects format: "Bearer " - * - * @param req - The HTTP request object - * @returns The JWT token string or an authentication failure - */ -function getAuthToken(req: Request): string | AuthFailure { - const authHeader = req.headers.get('authorization') - if (!authHeader) { - return { - code: RequestErrors.MissingAuthHeader, - message: 'Missing authorization header', - } - } - const tokenParts = authHeader.trim().split(/\s+/) - const [bearer, token] = tokenParts - if (bearer.toLowerCase() !== 'bearer' || tokenParts.length !== 2 || !token) { - return { - code: RequestErrors.InvalidTokenFormat, - message: 'Invalid JWT format', - } - } - return token -} - -function getAuthErrorResponse({ code, message = 'Invalid JWT' }: AuthFailure) { - return Response.json( - { - code, - message, - // DEPRECATED: Retained for backward compatibility. - msg: message, - }, - { - status: 401, - headers: { - 'sb-error-code': code, - 'Access-Control-Expose-Headers': 'sb-error-code', - }, - } - ) -} - -async function isValidLegacyJWT(jwt: string): Promise { - if (!JWT_SECRET) { - console.error('JWT_SECRET not available for HS256 token verification') - return { code: RequestErrors.InvalidLegacyJWT } - } - - const encoder = new TextEncoder(); - const secretKey = encoder.encode(JWT_SECRET); - - try { - await jose.jwtVerify(jwt, secretKey); - } catch (e) { - console.error('Symmetric Legacy JWT verification error', e); - return { code: RequestErrors.InvalidLegacyJWT } - } - return null -} - -async function isValidJWT(jwt: string): Promise { - if (!LOCAL_JWKS) { - console.error('JWKS not available for ES256/RS256 token verification') - return { code: RequestErrors.InvalidAsymmetricJWT } - } - - try { - await jose.jwtVerify(jwt, LOCAL_JWKS); - } catch (e) { - console.error('Asymmetric JWT verification error', e); - return { code: RequestErrors.InvalidAsymmetricJWT } - } - - return null -} - -/** - * Verify JWT token, handling both legacy (HS256) and newer (ES256/RS256) algorithms - * - * This function automatically detects the algorithm used in the token and applies - * the appropriate verification method: - * - HS256: Uses JWT_SECRET (symmetric key) - * - ES256/RS256: Uses JWKS endpoint (asymmetric public keys) - * - * This fix ensures compatibility with both legacy tokens and newer asymmetric tokens, - * resolving the "Key for the ES256 algorithm must be of type CryptoKey" error. - * - * @param jwt - The JWT token string to verify - * @returns Authentication failure details, or null when verification succeeds - */ -async function isValidHybridJWT(jwt: string): Promise { - let jwtAlgorithm: string | undefined - try { - jwtAlgorithm = jose.decodeProtectedHeader(jwt).alg - } catch (e) { - console.error('JWT format error', e) - return { - code: RequestErrors.InvalidTokenFormat, - message: 'Invalid JWT format', - } - } - - if (!jwtAlgorithm) { - return { - code: RequestErrors.InvalidTokenFormat, - message: 'Invalid JWT format', - } - } - - if (jwtAlgorithm === 'HS256') { - console.log(`Legacy token type detected, attempting ${jwtAlgorithm} verification.`) - - return await isValidLegacyJWT(jwt) - } - - if (jwtAlgorithm === 'ES256' || jwtAlgorithm === 'RS256') { - return await isValidJWT(jwt) - } - - return { - code: RequestErrors.UnsupportedTokenAlgorithm, - message: `Unsupported JWT algorithm ${jwtAlgorithm}`, - } -} - -Deno.serve(async (req: Request) => { - if (req.method !== 'OPTIONS' && VERIFY_JWT) { - try { - const token = getAuthToken(req) - if (typeof token !== 'string') { - return getAuthErrorResponse(token) - } - const authFailure = await isValidHybridJWT(token) - if (authFailure) { - return getAuthErrorResponse(authFailure) - } - } catch (e) { - console.error(e) - return getAuthErrorResponse({ - code: RequestErrors.InvalidTokenFormat, - message: 'Invalid JWT format', - }) - } - } - - const url = new URL(req.url) - const { pathname } = url - const path_parts = pathname.split('/') - const service_name = path_parts[1] - - if (!service_name || service_name === '') { - const error = { msg: 'missing function name in request' } - return new Response(JSON.stringify(error), { - status: 400, - headers: { 'Content-Type': 'application/json' }, - }) - } - - const servicePath = `/home/deno/functions/${service_name}` - console.error(`serving the request with ${servicePath}`) - - const memoryLimitMb = 150 - const workerTimeoutMs = 1 * 60 * 1000 - const noModuleCache = false - // Using a common Import Map for all functions - // to use a scope 'deno.json' it must be dinamically resolved base on the 'service_name' - const importMapPath = `/home/deno/functions/deno.jsonc` - // SUPABASE_FUNCTION_SLUG is listed after the container env snapshot so - // nothing in it can shadow the value, and it is per-request because only this - // worker knows which function the request resolved to. - const envVarsObj = { ...Deno.env.toObject(), SUPABASE_FUNCTION_SLUG: service_name } - const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) - - try { - const worker = await EdgeRuntime.userWorkers.create({ - servicePath, - memoryLimitMb, - workerTimeoutMs, - noModuleCache, - importMapPath, - envVars, - }) - return await worker.fetch(req) - } catch (e) { - const error = { msg: e.toString() } - return new Response(JSON.stringify(error), { - status: 500, - headers: { 'Content-Type': 'application/json' }, - }) - } -}) diff --git a/supabase/volumes/pooler/pooler.exs b/supabase/volumes/pooler/pooler.exs deleted file mode 100644 index 06f05e1b4a..0000000000 --- a/supabase/volumes/pooler/pooler.exs +++ /dev/null @@ -1,30 +0,0 @@ -{:ok, _} = Application.ensure_all_started(:supavisor) - -{:ok, version} = - case Supavisor.Repo.query!("select version()") do - %{rows: [[ver]]} -> Supavisor.Helpers.parse_pg_version(ver) - _ -> nil - end - -params = %{ - "external_id" => System.get_env("POOLER_TENANT_ID"), - "db_host" => System.get_env("POSTGRES_HOST") || "db", - "db_port" => System.get_env("POSTGRES_PORT"), - "db_database" => System.get_env("POSTGRES_DB"), - "require_user" => false, - "auth_query" => "SELECT * FROM pgbouncer.get_auth($1)", - "default_max_clients" => System.get_env("POOLER_MAX_CLIENT_CONN"), - "default_pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), - "default_parameter_status" => %{"server_version" => version}, - "users" => [%{ - "db_user" => "pgbouncer", - "db_password" => System.get_env("POSTGRES_PASSWORD"), - "mode_type" => System.get_env("POOLER_POOL_MODE"), - "pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), - "is_manager" => true - }] -} - -if !Supavisor.Tenants.get_tenant_by_external_id(params["external_id"]) do - {:ok, _} = Supavisor.Tenants.create_tenant(params) -end From c8174baae8f4383d12ecf5e3dfd604c6a1d7ebf4 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:08:17 -0600 Subject: [PATCH 04/14] Preserve image processing access to migrated files --- supabase/MIGRATION.md | 3 +++ supabase/migrate-storage.py | 6 ++++++ supabase/tests/migrate-storage.spec.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index e0ac0a54c4..df3e8a603b 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -222,6 +222,9 @@ sh run.sh start Replace `stub` with your preserved backend bucket and tenant values. The helper checks all export hashes before writing, rejects unsafe paths/conflicting files, and preserves Content-Type and Cache-Control using Linux extended attributes. +Installed directories use mode 755 and files 644 so the separate imgproxy user +can read them; restrict server access to this storage directory. Export backups +retain their private permissions. It can be rerun with the same verified export after fixing an installation error. ## 4. Verify before reopening writes diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index d6b5220181..5e6b4cecaf 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -85,7 +85,13 @@ def install(directory, root, bucket, tenant): for obj in manifest: target = destination(root, bucket, tenant, obj) target.parent.mkdir(parents=True, exist_ok=True) + # imgproxy runs as a different user; private backup umasks must not + # prevent it from traversing the installed tree or reading payloads. + for directory_path in [target.parent, *target.parent.parents]: + if directory_path.is_relative_to(root): + directory_path.chmod(0o755) shutil.copyfile(directory / obj["file"], target) + target.chmod(0o644) # Storage reads HTTP metadata from Linux extended attributes. os.setxattr(target, "user.supabase.content-type", obj["content_type"].encode()) os.setxattr(target, "user.supabase.cache-control", obj["cache_control"].encode()) diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py index 4359ae94cc..137e5f3ba1 100644 --- a/supabase/tests/migrate-storage.spec.py +++ b/supabase/tests/migrate-storage.spec.py @@ -1,5 +1,6 @@ import hashlib import json +import os from io import BytesIO from pathlib import Path import runpy @@ -40,6 +41,20 @@ def test_preserves_versioned_layout_and_http_metadata(self): attributes.assert_any_call(result, "user.supabase.content-type", b"application/json") attributes.assert_any_call(result, "user.supabase.cache-control", b"max-age=3600") + def test_installed_files_are_readable_by_imgproxy_with_private_backup_umask(self): + self.manifest([self.obj]) + self.target.mkdir(mode=0o700) + previous = os.umask(0o077) + try: + migration["install"](self.export, self.target, "stub", "stub") + finally: + os.umask(previous) + result = migration["destination"](self.target, "stub", "stub", self.obj) + self.assertEqual(result.stat().st_mode & 0o777, 0o644) + for directory in result.parents: + if directory.is_relative_to(self.target): + self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + def test_corruption_is_rejected_before_installing_any_object(self): self.manifest([self.obj, {**self.obj, "file": "00000001.bin"}]) (self.export / "00000001.bin").write_bytes(b"corrupted") From 00abe236adfebea6786c59f114b31ac4e878c4ab Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:08:40 -0600 Subject: [PATCH 05/14] Prepare readable storage for empty migrations --- supabase/migrate-storage.py | 2 ++ supabase/tests/migrate-storage.spec.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index 5e6b4cecaf..dd372afe93 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -82,6 +82,8 @@ def install(directory, root, bucket, tenant): target = destination(root, bucket, tenant, obj) if target.exists() and digest(target) != obj["sha256"]: raise ValueError("Destination already contains different data") + root.mkdir(parents=True, exist_ok=True) + root.chmod(0o755) for obj in manifest: target = destination(root, bucket, tenant, obj) target.parent.mkdir(parents=True, exist_ok=True) diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py index 137e5f3ba1..fadd1c299a 100644 --- a/supabase/tests/migrate-storage.spec.py +++ b/supabase/tests/migrate-storage.spec.py @@ -55,6 +55,16 @@ def test_installed_files_are_readable_by_imgproxy_with_private_backup_umask(self if directory.is_relative_to(self.target): self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + def test_empty_export_prepares_readable_storage_root(self): + self.manifest([]) + self.target.mkdir(mode=0o700) + previous = os.umask(0o077) + try: + migration["install"](self.export, self.target, "stub", "stub") + finally: + os.umask(previous) + self.assertEqual(self.target.stat().st_mode & 0o777, 0o755) + def test_corruption_is_rejected_before_installing_any_object(self): self.manifest([self.obj, {**self.obj, "file": "00000001.bin"}]) (self.export / "00000001.bin").write_bytes(b"corrupted") From aba866c085f03ffa5f5eeba8a8f5b131c3def5e7 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:29:18 -0600 Subject: [PATCH 06/14] Close Supabase migration validation gaps --- supabase/MIGRATION.md | 92 ++++++++++++++++++++++---- supabase/README.md | 9 +++ supabase/migrate-storage.py | 17 ++--- supabase/tests/migrate-storage.spec.py | 15 ++++- 4 files changed, 111 insertions(+), 22 deletions(-) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index df3e8a603b..6f76f83cce 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -40,6 +40,7 @@ From that old directory, create a private backup outside the deployment: ```sh umask 077 +postgres_database=postgres # Set this to the existing .env POSTGRES_DB. migration_backup="$(cd .. && pwd)/supabase-migration-backup" mkdir "$migration_backup" cp .env docker-compose.yml "$migration_backup/" @@ -63,11 +64,12 @@ shell history or in the application's `VITE_*` variables. ```sh python3 migrate-storage.py export "$migration_backup/objects" \ - --url http://localhost:8000 + --url http://localhost:8000 --database "$postgres_database" ``` -Use your actual gateway URL. Export includes all live objects in every Supabase -bucket, not only `revisit`, and fails on missing objects or size mismatches. +Use your actual gateway URL and preserved `POSTGRES_DB` for both export and +verification. Keep `postgres_database` in the same shell session. Export includes +all live objects in every Supabase bucket, not only `revisit`, and fails on missing objects or size mismatches. The export directory must not already exist. An interrupted export must be repeated into a new directory; only a completed export contains `manifest.json`. It does not copy orphaned MinIO objects absent from the database. @@ -123,7 +125,7 @@ The upstream scripts now live in this deployment; reVISit does not maintain copies of them. Merge new `.env.example` settings into your existing private `.env`. Keep the -existing database password, `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY`, and +existing `POSTGRES_DB`, database password, `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY`, and encryption keys. **Do not run `generate-keys.sh` during migration:** rotating these values can invalidate sessions or make encrypted data unreadable. @@ -137,8 +139,19 @@ these values can invalidate sessions or make encrypted data unreadable. these secure Storage’s S3 API even when its backend is local files. Generate only missing values using the commands/lengths in `.env.example`; never use the template’s default secrets or credentials. -- Preserve URLs, SMTP, signup settings, and OAuth provider credentials. The old - GitHub settings are preserved by `sh run.sh config add github`. This optional +- Preserve `SITE_URL`, application redirects, SMTP, signup settings, and OAuth + provider credentials. Convert `API_EXTERNAL_URL` to the full Auth endpoint: + `https://your-domain` becomes `https://your-domain/auth/v1` (do not append the + suffix twice). Keep the final `GITHUB_OAUTH_REDIRECT_URI` registered with + GitHub unchanged at `https://your-domain/auth/v1/callback`. Any custom OAuth + override derived from `${API_EXTERNAL_URL}/auth/v1/callback` must now use + `${API_EXTERNAL_URL}/callback`. SAML deployments must update their IdP's ACS + and metadata endpoints to `/auth/v1/sso/saml/acs` and + `/auth/v1/sso/saml/metadata`. If configured, convert `SAML_EXTERNAL_URL` + (mapped to `GOTRUE_SAML_EXTERNAL_URL`) to the full `/auth/v1` base too; it + overrides `API_EXTERNAL_URL`. Re-fetch provider metadata afterwards. See + [upstream Auth URL migration guidance](https://supabase.com/changelog/47093-self-hosted-supabase-api-external-url-to-include-auth-v1). + The old GitHub settings are preserved by `sh run.sh config add github`. This optional override reads the existing `ENABLE_GITHUB_OAUTH`, `GITHUB_OAUTH_CLIENT_ID`, `GITHUB_OAUTH_SECRET`, and `GITHUB_OAUTH_REDIRECT_URI` variables. Keep the same callback URL and registered GitHub OAuth application. @@ -179,7 +192,7 @@ docker compose --project-directory "$PWD" \ -f "$migration_backup/docker-compose.yml" up -d --wait db docker exec supabase-db pg_isready -U postgres # Our previous Compose did not initialize the upstream Realtime schema. -docker exec -i supabase-db psql -U supabase_admin -d postgres \ +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ -v ON_ERROR_STOP=1 < volumes/db/realtime.sql sudo bash utils/upgrade-pg17.sh ``` @@ -191,9 +204,56 @@ required role/extension changes, and starts the new stack. See On a host with multiple `db-config` volumes, stop and resolve the target before running: this upstream script selects a matching volume by name. -Confirm the script reaches **Upgrade complete** and all services start. Do not -rerun it blindly after a partial upgrade: it refuses an already upgraded database. -Resolve startup failures and use the preserved backups to rehearse again. +Require a successful script exit and healthy services. +**Upgrade complete is not sufficient:** the pinned script warns about failed +required SQL migrations and can still report success. Keep writes blocked and +**stop here** on missing migration files, SQL errors, or migration/extension +reconciliation failures. Resolve each failure and reapply only its affected SQL +from `/docker-entrypoint-initdb.d/migrations/` in the target database container +with `psql -v ON_ERROR_STOP=1`; inspect the pinned `utils/upgrade-pg17.sh` for the +required files and effects. Do not rerun the whole database upgrade: it refuses +an already upgraded database. Resolve startup failures or restore the preserved +backups to rehearse again. + +The upstream script applies its database-local SQL to `postgres`. If your +`POSTGRES_DB` differs, separately rehearse and apply the required database-local +migrations there as well; the helper's database option does not adapt the +upstream upgrade script. Before proceeding, require the following role/grant +check to succeed in the configured application database, and confirm every +required migration completed without errors: + +```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -v ON_ERROR_STOP=1 <<'SQL' +DO $$ +BEGIN + IF NOT ( + pg_has_role('postgres', 'supabase_privileged_role', 'MEMBER') + AND pg_has_role('supabase_etl_admin', 'supabase_privileged_role', 'MEMBER') + AND pg_has_role('supabase_etl_admin', 'pg_read_all_data', 'MEMBER') + AND has_database_privilege('supabase_etl_admin', current_database(), 'CREATE') + AND pg_has_role('supabase_etl_admin', 'pg_monitor', 'MEMBER') + AND pg_has_role('supabase_read_only_user', 'pg_monitor', 'MEMBER') + AND EXISTS ( + SELECT 1 FROM pg_auth_members m + JOIN pg_roles granted ON granted.oid = m.roleid + JOIN pg_roles member_role ON member_role.oid = m.member + WHERE granted.rolname = 'pg_create_subscription' + AND member_role.rolname = 'postgres' AND m.admin_option + ) + AND has_function_privilege('postgres', 'pg_catalog.pg_reload_conf()', + 'EXECUTE WITH GRANT OPTION') + ) THEN + RAISE EXCEPTION 'Required PG17 roles/grants missing; keep writes blocked'; + END IF; +END +$$; +SQL +``` + +A missing role or grant is a migration failure; resolve it before continuing. +These checks complement review of all migration output, including database-local +function/search-path, role settings, GraphQL trigger, and extension changes. The target image changes the libc collation version. While traffic is still blocked, rebuild indexes in each retained database (include any custom databases): @@ -231,7 +291,7 @@ It can be rerun with the same verified export after fixing an installation error ```sh python3 migrate-storage.py verify "$migration_backup/objects" \ - --url http://localhost:8000 + --url http://localhost:8000 --database "$postgres_database" sh run.sh status docker exec supabase-db psql -U supabase_admin -d postgres \ -c 'SHOW server_version;' @@ -244,7 +304,15 @@ metadata. Keep the legacy application URL/key unchanged when possible. Check Stu existing participants/answers/configurations, recordings, exports, access policies, and a new participant submission. Restart the stack and confirm both old and new data persist. Retain database/file/encryption backups and old images -until the deployment has been accepted. Then reopen application writes. +until the deployment has been accepted. After verifying your external database/file/encryption backups and accepting +the migration, move `volumes/db/data.bak.pg15` and +`volumes/storage.minio-backup` outside the deployment (retain them for rollback). +Do this **before any upstream update**: its configuration archive excludes only +the exact live `data` and `storage` directories, includes these retained copies, +and can continue after archive errors. Keep their new locations in your rollback +inventory. Also move or remove the disposable `volumes/db/pg17_upgrade_bin_*.tar.gz` +cache after acceptance; it is also included in that archive and can exceed 1 GB. +Retain the encryption/key backups. Then reopen application writes. ## Rollback diff --git a/supabase/README.md b/supabase/README.md index 08736b3f5c..f3c04e9abe 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -135,6 +135,15 @@ For deployments requiring object storage, use [upstream S3 configuration](https://supabase.com/docs/guides/self-hosting/self-hosted-s3). Use `sh run.sh stop`, `start`, `status`, and `logs` to operate the stack. +Before any update, verify external backups and move retained migration copies +`volumes/db/data.bak.pg15` and `volumes/storage.minio-backup` outside the deployment; +keep them for rollback. Also move or remove the disposable +`volumes/db/pg17_upgrade_bin_*.tar.gz` cache after accepting migration, while +retaining encryption/key backups. Upstream's configuration backup excludes only the exact +live-data paths, so these copies would be archived too, potentially filling the +disk. Archive errors only warn: check the resulting configuration archive before +relying on it, and stop if it is incomplete. + Review future tagged updates using `sh update.sh --dry-run --to ` and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). The update tool belongs to the downloaded upstream bundle. Review its plan and diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index dd372afe93..9853fe00e0 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -15,11 +15,11 @@ import urllib.request -def objects(): +def objects(database): sql = "SELECT coalesce(json_agg(o ORDER BY bucket_id, name), '[]'::json) FROM storage.objects o" result = subprocess.run( ["docker", "exec", "supabase-db", "psql", "-U", "supabase_admin", - "-d", "postgres", "-At", "-v", "ON_ERROR_STOP=1", "-c", sql], + "-d", database, "-At", "-v", "ON_ERROR_STOP=1", "-c", sql], check=True, capture_output=True, text=True, ) return json.loads(result.stdout) @@ -40,9 +40,9 @@ def digest(file): return hashlib.file_digest(stream, "sha256").hexdigest() -def export(url, directory): +def export(url, directory, database="postgres"): directory.mkdir(parents=True, exist_ok=False) - manifest = objects() + manifest = objects(database) for index, obj in enumerate(manifest): obj["file"] = f"{index:08d}.bin" target = directory / obj["file"] @@ -100,9 +100,9 @@ def install(directory, root, bucket, tenant): print(f"Installed {len(manifest)} objects; database metadata unchanged") -def verify(url, directory): +def verify(url, directory, database="postgres"): manifest = json.loads((directory / "manifest.json").read_text()) - current = objects() + current = objects(database) exported_columns = set(manifest[0]) - {"file", "sha256", "content_type", "cache_control"} if manifest else set() original = {obj["id"]: {key: obj[key] for key in exported_columns} for obj in manifest} retained = {obj["id"]: {key: obj.get(key) for key in exported_columns} for obj in current} @@ -127,6 +127,7 @@ def main(): parser.add_argument("operation", choices=["export", "install", "verify"]) parser.add_argument("directory", type=Path) parser.add_argument("--url", default="http://localhost:8000") + parser.add_argument("--database", default="postgres", help="POSTGRES_DB for export and verify") parser.add_argument("--destination", type=Path) parser.add_argument("--bucket", default="stub", help="GLOBAL_S3_BUCKET, not the application bucket") parser.add_argument("--tenant", default="stub", help="STORAGE_TENANT_ID (previously TENANT_ID)") @@ -136,9 +137,9 @@ def main(): parser.error("install requires --destination") install(args.directory, args.destination, args.bucket, args.tenant) elif args.operation == "export": - export(args.url, args.directory) + export(args.url, args.directory, args.database) else: - verify(args.url, args.directory) + verify(args.url, args.directory, args.database) if __name__ == "__main__": diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py index fadd1c299a..320c8ae716 100644 --- a/supabase/tests/migrate-storage.spec.py +++ b/supabase/tests/migrate-storage.spec.py @@ -32,6 +32,17 @@ def setUp(self): def manifest(self, objects): (self.export / "manifest.json").write_text(json.dumps(objects)) + def test_export_and_verify_query_the_selected_database(self): + directory = self.export / "empty" + with patch("subprocess.run") as query: + query.return_value.stdout = "[]" + for operation in ("export", "verify"): + with patch("sys.argv", ["migrate-storage.py", operation, str(directory), + "--database", "study_database"]): + migration["main"]() + command = query.call_args.args[0] + self.assertEqual(command[command.index("-d") + 1], "study_database") + def test_preserves_versioned_layout_and_http_metadata(self): self.manifest([self.obj]) with patch("os.setxattr", create=True) as attributes: @@ -84,10 +95,10 @@ def download(url, obj): return response with patch.dict(migration["verify"].__globals__, - objects=lambda: list(reversed(objects)), download=download): + objects=lambda database: list(reversed(objects)), download=download): migration["verify"]("http://example.test", self.export) changed = [{**objects[0], "version": "changed"}, objects[1]] - with patch.dict(migration["verify"].__globals__, objects=lambda: changed): + with patch.dict(migration["verify"].__globals__, objects=lambda database: changed): with self.assertRaisesRegex(ValueError, "metadata changed"): migration["verify"]("http://example.test", self.export) From 2584871f9c5bd191f297ecb5a084de1fc306d080 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:35:55 -0600 Subject: [PATCH 07/14] Make Supabase storage installation retryable --- supabase/MIGRATION.md | 2 ++ supabase/migrate-storage.py | 15 ++++++++----- supabase/tests/migrate-storage.spec.py | 31 ++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 6f76f83cce..89f987b879 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -285,6 +285,8 @@ and preserves Content-Type and Cache-Control using Linux extended attributes. Installed directories use mode 755 and files 644 so the separate imgproxy user can read them; restrict server access to this storage directory. Export backups retain their private permissions. +Each payload and its HTTP metadata are staged on the destination filesystem +before an atomic rename, so a failed copy leaves the final object untouched. It can be rerun with the same verified export after fixing an installation error. ## 4. Verify before reopening writes diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index 9853fe00e0..53c17cca2a 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -11,6 +11,7 @@ from pathlib import Path import shutil import subprocess +import tempfile import urllib.parse import urllib.request @@ -92,11 +93,15 @@ def install(directory, root, bucket, tenant): for directory_path in [target.parent, *target.parent.parents]: if directory_path.is_relative_to(root): directory_path.chmod(0o755) - shutil.copyfile(directory / obj["file"], target) - target.chmod(0o644) - # Storage reads HTTP metadata from Linux extended attributes. - os.setxattr(target, "user.supabase.content-type", obj["content_type"].encode()) - os.setxattr(target, "user.supabase.cache-control", obj["cache_control"].encode()) + # Publish only complete bytes and metadata; failed copies remain retryable. + with tempfile.TemporaryDirectory(dir=target.parent, prefix=".revisit-migration-") as staging: + payload = Path(staging) / "payload" + shutil.copyfile(directory / obj["file"], payload) + payload.chmod(0o644) + # Storage reads HTTP metadata from Linux extended attributes. + os.setxattr(payload, "user.supabase.content-type", obj["content_type"].encode()) + os.setxattr(payload, "user.supabase.cache-control", obj["cache_control"].encode()) + os.replace(payload, target) print(f"Installed {len(manifest)} objects; database metadata unchanged") diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py index 320c8ae716..46cc01c1d4 100644 --- a/supabase/tests/migrate-storage.spec.py +++ b/supabase/tests/migrate-storage.spec.py @@ -4,6 +4,7 @@ from io import BytesIO from pathlib import Path import runpy +import shutil import tempfile import unittest from unittest.mock import patch @@ -49,8 +50,34 @@ def test_preserves_versioned_layout_and_http_metadata(self): migration["install"](self.export, self.target, "stub", "stub") result = self.target / "stub/stub/revisit/study/participant data.json/existing-version" self.assertEqual(result.read_bytes(), b"existing participant data") - attributes.assert_any_call(result, "user.supabase.content-type", b"application/json") - attributes.assert_any_call(result, "user.supabase.cache-control", b"max-age=3600") + self.assertEqual({call.args[1]: call.args[2] for call in attributes.call_args_list}, { + "user.supabase.content-type": b"application/json", + "user.supabase.cache-control": b"max-age=3600", + }) + + def test_interrupted_copy_preserves_destination_and_can_be_retried(self): + self.manifest([self.obj]) + + def interrupted_copy(source, target): + Path(target).write_bytes(b"partial copy") + raise OSError("copy interrupted") + + for existing in (False, True): + with self.subTest(existing=existing): + root = self.target / str(existing) + target = migration["destination"](root, "stub", "stub", self.obj) + if existing: + target.parent.mkdir(parents=True) + shutil.copyfile(self.export / self.obj["file"], target) + with patch("shutil.copyfile", side_effect=interrupted_copy): + with self.assertRaisesRegex(OSError, "copy interrupted"): + migration["install"](self.export, root, "stub", "stub") + if existing: + self.assertEqual(target.read_bytes(), b"existing participant data") + else: + self.assertFalse(target.exists()) + migration["install"](self.export, root, "stub", "stub") + self.assertEqual(target.read_bytes(), b"existing participant data") def test_installed_files_are_readable_by_imgproxy_with_private_backup_umask(self): self.manifest([self.obj]) From 504f7c0651edb67fa0d9103dd3c0b15513d275cd Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 22:51:31 -0600 Subject: [PATCH 08/14] Fix Supabase setup and migration command sequencing --- supabase/MIGRATION.md | 27 ++++++++++++++++++++++----- supabase/README.md | 4 +++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 89f987b879..6e33304bec 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -162,6 +162,8 @@ Before modifying database files, pull the target images and check the two Node services start on this host: ```sh +# The existing ENABLE_GITHUB_OAUTH value controls whether GitHub sign-in is enabled. +sh run.sh config add github sh run.sh pull docker compose run --rm --no-deps studio node --version docker compose run --rm --no-deps meta node --version @@ -256,15 +258,30 @@ These checks complement review of all migration output, including database-local function/search-path, role settings, GraphQL trigger, and extension changes. The target image changes the libc collation version. While traffic is still -blocked, rebuild indexes in each retained database (include any custom databases): +blocked, rebuild indexes in each retained database. The block includes the +configured application database once; add any other retained databases to the +`set --` list: ```sh -for database in postgres template1 _supabase; do - docker exec supabase-db psql -U supabase_admin -d "$database" \ - -v ON_ERROR_STOP=1 -c "REINDEX DATABASE \"$database\";" -done +( + set -- postgres template1 _supabase + case "$postgres_database" in + postgres|template1|_supabase) ;; + *) set -- "$@" "$postgres_database" ;; + esac + for database do + docker exec -i supabase-db psql -U supabase_admin -d "$database" \ + -v ON_ERROR_STOP=1 -v database="$database" <<'SQL' || exit 1 +REINDEX DATABASE :"database"; +SQL + done +) ``` +Require this block to exit successfully for **every** database. On any failure, +stop, resolve the error, and rerun it before installing files or reopening writes. +The subshell stops at the first failed command without closing your login shell. + The upstream script refreshes the version marker; that alone does not rebuild indexes. See [PostgreSQL collation guidance](https://www.postgresql.org/docs/17/sql-altercollation.html). Allow time and temporary disk space for this step. diff --git a/supabase/README.md b/supabase/README.md index f3c04e9abe..0b3183f0bf 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -94,10 +94,12 @@ names: run only one copy per Docker host, or deliberately isolate it. 4. Initialize the reVISit table and private bucket **on a fresh database only**: ```sh - docker exec -i supabase-db psql -U supabase_admin -d postgres \ + docker exec -i supabase-db psql -U supabase_admin \ -v ON_ERROR_STOP=1 < "$revisit_support/revisit.sql" ``` + `psql` uses the container's `PGDATABASE`, which upstream sets from + `POSTGRES_DB`; this initializes the configured application database. This applies the same schema and access model as the existing [reVISit setup instructions](https://revisit.dev/docs/data-and-deployment/supabase/setup/). Those policies allow anyone holding the public application key to read and From c4a8e6f12ae61a16460f3005043e5fddfa33e8c7 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Tue, 6 Oct 2026 23:06:21 -0600 Subject: [PATCH 09/14] Run migration tests in CI and enforce Supabase bundle pin checks --- .github/workflows/vitest.yml | 3 +++ supabase/MIGRATION.md | 8 ++++---- supabase/README.md | 10 +++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 3eb55bca67..eab1e344a3 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -24,3 +24,6 @@ jobs: - name: Run unit tests run: yarn unittest + + - name: Run storage migration tests + run: python3 supabase/tests/migrate-storage.spec.py diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 6e33304bec..4c8a9f91af 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -108,13 +108,13 @@ old deployment directory, replace the support path with your new reVISit checkou ```sh revisit_support=/absolute/path/to/new-checkout/supabase git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ - https://github.com/supabase/supabase.git ../supabase-upstream -git -C ../supabase-upstream sparse-checkout set docker + https://github.com/supabase/supabase.git ../supabase-upstream && +git -C ../supabase-upstream sparse-checkout set docker && test "$(git -C ../supabase-upstream rev-parse HEAD)" = \ - 564eab8ad7840b13324f68b1bfac074ef8d51c21 + 564eab8ad7840b13324f68b1bfac074ef8d51c21 && rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ --exclude='volumes/db/data*' --exclude='volumes/storage*' \ - ../supabase-upstream/docker/ ./ + ../supabase-upstream/docker/ ./ && cp "$revisit_support/docker-compose.github.yml" . ``` diff --git a/supabase/README.md b/supabase/README.md index 0b3183f0bf..38fa69bce9 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -31,12 +31,12 @@ names: run only one copy per Docker host, or deliberately isolate it. ```sh revisit_support=/absolute/path/to/revisit-study/supabase git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ - https://github.com/supabase/supabase.git supabase-upstream - git -C supabase-upstream sparse-checkout set docker + https://github.com/supabase/supabase.git supabase-upstream && + git -C supabase-upstream sparse-checkout set docker && test "$(git -C supabase-upstream rev-parse HEAD)" = \ - 564eab8ad7840b13324f68b1bfac074ef8d51c21 - mkdir supabase-project - cp -a supabase-upstream/docker/. supabase-project/ + 564eab8ad7840b13324f68b1bfac074ef8d51c21 && + mkdir supabase-project && + cp -a supabase-upstream/docker/. supabase-project/ && cd supabase-project ``` From 627d5b49ef80edd0afc06782be99e76e0dea4af8 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Sun, 18 Oct 2026 18:48:20 -0600 Subject: [PATCH 10/14] Support pgsty migration to patched upstream Supabase --- README.md | 2 +- supabase/MIGRATION.md | 74 +++++++++++++++++++++++++++---------- supabase/README.md | 24 +++++++----- supabase/UPSTREAM | 6 +-- supabase/migrate-storage.py | 4 +- tests/supabase-live.spec.ts | 33 +++++++++++++++++ 6 files changed, 108 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 808b1adbeb..ec0130207d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ To run this demo experiment locally, you will need to install Node.js on your co ## Self-hosted Supabase -See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for a version-pinned upstream Supabase deployment. Existing MinIO installations must migrate before starting the new configuration. +See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for a version-pinned upstream Supabase deployment. Existing MinIO or pgsty-backed installations must migrate before starting the new configuration. ## Adding Tests diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 4c8a9f91af..e675787660 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -1,8 +1,9 @@ -# Migrate the bundled MinIO deployment +# Migrate the bundled MinIO or pgsty deployment This procedure targets the previous bundled stack (Postgres 15.8.1.060, -Supabase Storage v1.11.13, and MinIO) and moves it to the release in [UPSTREAM](UPSTREAM), -using Postgres 17 and Storage v1.74.0's default file backend. +Supabase Storage v1.11.13, and either the original MinIO images or `main`'s +`pgsty/silo` + `pgsty/mc` replacement images) and moves it to the release in +[UPSTREAM](UPSTREAM), using Postgres 17 and Storage v1.81.0's default file backend. Rehearse against a restored copy of your deployment first. New installations use [README.md](README.md); hosted Supabase users are unaffected. @@ -11,21 +12,28 @@ window**. Storage object IDs, versions, owners, timestamps, bucket definitions, and policies remain in Postgres. `migrate-storage.py` exports object bytes through the old API and installs their versioned files with HTTP metadata in the new backend. It never writes database records or re-uploads objects through the API. +Storage v1.81.0 itself applies one-way database migrations: after it starts, an +older Storage image cannot safely use that database. Rollback requires the +coordinated pre-upgrade database, object-store, encryption and configuration +backups below; changing image tags alone is insufficient. Requirements: a Linux server, Git, Docker Compose, Bash, curl, OpenSSL, Python 3.11+, root access for the upstream database upgrade, and enough space for the database backup/upgrade and two copies of object data. Pre-pull the target images before starting the upgrade; allow at least 15 GB additional temporary space for the upgrade images, extracted binaries, and compressed bundle (also inside the -Docker VM when using Docker Desktop). The destination filesystem must -support Linux extended attributes. Do not reuse MinIO's on-disk directory as +Docker VM when using Docker Desktop). The command below puts build files next +to the backup, avoiding a small or memory-backed `/tmp`. The destination +filesystem must support Linux extended attributes. Do not reuse MinIO's on-disk directory as Supabase file storage. Custom S3 layouts, custom object-version separators, or multiple Supabase stacks on one Docker host require a separately rehearsed plan. ## 1. Inventory and export while the old stack is running Keep the old Compose/configuration files and images available for rollback. -Save cached MinIO server/client images if the registry can no longer supply them. +Save the exact cached server/client images used by the old containers, including +the pgsty images when migrating `main`. Do not replace Silo with an old MinIO +image or change object-store versions during this migration. Record your database extensions, OAuth/SMTP settings, proxy configuration, bucket limits, storage tenant, backend bucket, and any custom access policies. Restrict application traffic and pause participant/designer/background writes @@ -45,8 +53,20 @@ migration_backup="$(cd .. && pwd)/supabase-migration-backup" mkdir "$migration_backup" cp .env docker-compose.yml "$migration_backup/" cp -a volumes/api "$migration_backup/api" -docker image save minio/minio minio/mc \ - -o "$migration_backup/minio-images.tar" +storage_server_image=$(docker inspect "$(docker compose ps -q minio)" \ + --format '{{.Image}}') +storage_client_image=$(docker inspect "$(docker compose ps -a -q minio-createbucket)" \ + --format '{{.Image}}') +test -n "$storage_server_image" && test -n "$storage_client_image" && +docker image save "$storage_server_image" "$storage_client_image" \ + -o "$migration_backup/storage-images.tar" && +cat > "$migration_backup/docker-compose.storage-images.yml" < "$migration_backup/database.sql" db_config_volume=$(docker inspect supabase-db --format \ @@ -57,6 +77,14 @@ docker run --rm -v "$db_config_volume:/source:ro" \ alpine:3.22 tar -cf /backup/db-config.tar -C /source . ``` +Require every backup command to succeed before proceeding. The storage-image +archive and override use immutable IDs from the existing containers, so rollback +does not depend on mutable `latest` tags or registry availability. The bucket +setup container may be stopped; keep it until its image has been saved. If it +was already removed, resolve its original client image ID from your deployment +inventory and save it with the server image before continuing. The Compose +service names remain `minio` and `minio-createbucket` with the pgsty images. + Privately set `SUPABASE_SERVICE_ROLE_KEY` to the **old** `.env`'s `SERVICE_ROLE_KEY` (for example, use `read -rs SUPABASE_SERVICE_ROLE_KEY` in Bash, then `export SUPABASE_SERVICE_ROLE_KEY`; paste the key when prompted). Do not put the key in @@ -107,11 +135,11 @@ old deployment directory, replace the support path with your new reVISit checkou ```sh revisit_support=/absolute/path/to/new-checkout/supabase -git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ +git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.3 \ https://github.com/supabase/supabase.git ../supabase-upstream && git -C ../supabase-upstream sparse-checkout set docker && test "$(git -C ../supabase-upstream rev-parse HEAD)" = \ - 564eab8ad7840b13324f68b1bfac074ef8d51c21 && + bc27d2661edd09a5d0b8d76ce33da6d4071a756e && rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ --exclude='volumes/db/data*' --exclude='volumes/storage*' \ ../supabase-upstream/docker/ ./ && @@ -169,9 +197,7 @@ docker compose run --rm --no-deps studio node --version docker compose run --rm --no-deps meta node --version ``` -If either crashes, resolve host/image compatibility before upgrading. Our ARM64 -Docker Desktop rehearsal required the amd64 variants of these two images, as -noted in [README.md](README.md). +If either crashes, resolve host/image compatibility before upgrading. Move the old MinIO data aside and create an empty file-storage directory: @@ -196,7 +222,9 @@ docker exec supabase-db pg_isready -U postgres # Our previous Compose did not initialize the upstream Realtime schema. docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ -v ON_ERROR_STOP=1 < volumes/db/realtime.sql -sudo bash utils/upgrade-pg17.sh +mkdir -p "$migration_backup/upgrade-tmp" +sudo env TMPDIR="$migration_backup/upgrade-tmp" bash utils/upgrade-pg17.sh && +sudo rmdir "$migration_backup/upgrade-tmp" ``` Read all output, including migration warnings. This script uses upstream's @@ -314,7 +342,7 @@ python3 migrate-storage.py verify "$migration_backup/objects" \ sh run.sh status docker exec supabase-db psql -U supabase_admin -d postgres \ -c 'SHOW server_version;' -printf 'ref=self-hosted/v0.8.2\n' > .supabase-version +printf 'ref=self-hosted/v0.8.3\n' > .supabase-version ``` Verification streams every migrated object through the new API and checks its @@ -340,10 +368,18 @@ old Compose/API configuration and `.env`, the quiesced Postgres 15 data, the MinIO directory, and the matching `db-config` encryption/configuration volume. The database upgrade changes the volume's ownership/configuration, so restoring only database files is insufficient. Use the physical backup and saved volume -archive (and load saved MinIO images with `docker image load -i`), or follow the -upstream rollback steps together with the original configuration. Start the old stack with the saved original images and verify -data and login before reopening writes. Keep the failed new data separately -for diagnosis. +archive, then load the saved images and use the pinned storage override: + +```sh +docker image load -i "$migration_backup/storage-images.tar" +docker compose -f docker-compose.yml \ + -f "$migration_backup/docker-compose.storage-images.yml" up -d +``` + +Run this from the restored old deployment directory with its original `.env` +and Compose file. This restores the same pgsty or MinIO image IDs that were +running before migration. Verify data and login before reopening writes. Keep +the failed new data separately for diagnosis. After new writes are accepted, restoring the old backups loses those writes. Pause and plan how to retain them rather than treating rollback as lossless. diff --git a/supabase/README.md b/supabase/README.md index 38fa69bce9..61c73e5038 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -1,7 +1,7 @@ # Self-hosted Supabase for reVISit This directory contains the reVISit-specific setup and migration files. Obtain -the deployment bundle directly from **Supabase self-hosted/v0.8.2** (see +the deployment bundle directly from **Supabase self-hosted/v0.8.3** (see [UPSTREAM](UPSTREAM)), following the upstream manual installation below. Its default Storage backend uses persistent local files; MinIO is not required. For an existing deployment, follow [MIGRATION.md](MIGRATION.md) **before starting @@ -15,10 +15,8 @@ resources and operating-system details, follow the [upstream installation guide](https://supabase.com/docs/guides/self-hosting/docker). The storage filesystem must support Linux extended attributes. Mac shared folders in Docker Desktop may reject these attributes; use a Docker-managed -Linux volume for a local rehearsal and back it up separately. The pinned ARM64 -Studio and postgres-meta images also crashed on our Apple Silicon Docker Desktop -host; their amd64 variants worked under emulation. Verify image startup on your -actual host before a migration. +Linux volume for a local rehearsal and back it up separately. Verify image +startup on your actual host before a migration. Keep the deployment outside the reVISit checkout so application updates do not replace its configuration or data. The upstream stack uses fixed container @@ -30,11 +28,11 @@ names: run only one copy per Docker host, or deliberately isolate it. ```sh revisit_support=/absolute/path/to/revisit-study/supabase - git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.2 \ + git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.3 \ https://github.com/supabase/supabase.git supabase-upstream && git -C supabase-upstream sparse-checkout set docker && test "$(git -C supabase-upstream rev-parse HEAD)" = \ - 564eab8ad7840b13324f68b1bfac074ef8d51c21 && + bc27d2661edd09a5d0b8d76ce33da6d4071a756e && mkdir supabase-project && cp -a supabase-upstream/docker/. supabase-project/ && cd supabase-project @@ -48,7 +46,7 @@ names: run only one copy per Docker host, or deliberately isolate it. cp .env.example .env sh utils/generate-keys.sh --update-env sh utils/add-new-auth-keys.sh --update-env - printf 'ref=self-hosted/v0.8.2\n' > .supabase-version + printf 'ref=self-hosted/v0.8.3\n' > .supabase-version ``` These helpers print secrets; keep their output private. Never commit `.env`. @@ -113,7 +111,7 @@ names: run only one copy per Docker host, or deliberately isolate it. ```dotenv VITE_STORAGE_ENGINE="supabase" VITE_SUPABASE_URL="http://localhost:8000" - VITE_SUPABASE_ANON_KEY="" + VITE_SUPABASE_ANON_KEY="" ``` The application variable retains its existing name and accepts the new @@ -150,7 +148,13 @@ Review future tagged updates using `sh update.sh --dry-run --to ` and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). The update tool belongs to the downloaded upstream bundle. Review its plan and your GitHub override before applying an update; this guide and migration helper -target the pinned release above. +target the pinned release above. This release includes Storage and Realtime +security fixes. Its Storage database migrations are one-way: reverting an image +alone is unsafe; restore coordinated pre-upgrade backups for rollback. Review +[upstream release notes](https://github.com/supabase/supabase/blob/self-hosted/v0.8.3/docker/CHANGELOG.md) +before upgrading. If you use Realtime Postgres Changes, the release notes also +document a connection-restart issue and the `REGION: us-east-1` workaround; +reVISit's storage engine does not subscribe to Postgres Changes. ## Local regression checks diff --git a/supabase/UPSTREAM b/supabase/UPSTREAM index 9f3ba99505..9181311d6c 100644 --- a/supabase/UPSTREAM +++ b/supabase/UPSTREAM @@ -1,6 +1,6 @@ -Supabase self-hosted/v0.8.2 -Commit: 564eab8ad7840b13324f68b1bfac074ef8d51c21 -Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.2/docker +Supabase self-hosted/v0.8.3 +Commit: bc27d2661edd09a5d0b8d76ce33da6d4071a756e +Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.3/docker The deployment configuration and scripts are obtained directly from upstream. This directory contains only reVISit setup, migration, and integration files. diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index 53c17cca2a..c851c5fe61 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Offline MinIO -> Supabase v1.74.0 file-backend migration. No database writes. +"""Offline MinIO/pgsty -> Supabase v1.81.0 file-backend migration. No database writes. Export runs with the old stack online and application writes stopped. Install runs on Linux with the new Storage service stopped. See MIGRATION.md. @@ -60,7 +60,7 @@ def export(url, directory, database="postgres"): def destination(root, bucket, tenant, obj): - # These match v1.74.0's default location and withOptionalVersion helpers. + # These match v1.81.0's default location and withOptionalVersion helpers. parts = [bucket, tenant, obj["bucket_id"], *obj["name"].split("/")] if obj.get("version"): parts.append(obj["version"]) diff --git a/tests/supabase-live.spec.ts b/tests/supabase-live.spec.ts index 5e90f6c407..a59f5b0418 100644 --- a/tests/supabase-live.spec.ts +++ b/tests/supabase-live.spec.ts @@ -110,4 +110,37 @@ test.describe('Live self-hosted Supabase', () => { await expect(page.getByPlaceholder('Select an option')).toHaveValue('A'); expect(colorAnswers(await readParticipant(participantId!))).toEqual(colorAnswers(participant)); }); + + test('supports binary files, overwrites and signed downloads with a participant session', async ({ request }) => { + const client = createClient(process.env.VITE_SUPABASE_URL!, process.env.VITE_SUPABASE_ANON_KEY!, { + auth: { persistSession: false, autoRefreshToken: false }, + }); + const { error: authError } = await client.auth.signInAnonymously(); + expect(authError).toBeNull(); + const bucket = client.storage.from('revisit'); + const path = `${studyId}/recording checks/${randomUUID()}.webm`; + const original = Uint8Array.from([0, 1, 127, 128, 255]); + const replacement = Uint8Array.from([255, 128, 0, 42]); + try { + const { error: uploadError } = await bucket.upload(path, original, { contentType: 'video/webm' }); + expect(uploadError).toBeNull(); + const { data: downloaded, error: downloadError } = await bucket.download(path); + expect(downloadError).toBeNull(); + expect(new Uint8Array(await downloaded!.arrayBuffer())).toEqual(original); + + const { error: overwriteError } = await bucket.upload(path, replacement, { upsert: true, contentType: 'video/webm' }); + expect(overwriteError).toBeNull(); + const { data: signed, error: signError } = await bucket.createSignedUrl(path, 60); + expect(signError).toBeNull(); + const response = await request.get(signed!.signedUrl); + expect(response.ok()).toBeTruthy(); + expect(response.headers()['content-type']).toContain('video/webm'); + expect(new Uint8Array(await response.body())).toEqual(replacement); + } finally { + const { error } = await bucket.remove([path]); + expect(error).toBeNull(); + } + const { error: removedError } = await bucket.download(path); + expect(removedError).not.toBeNull(); + }); }); From 1669a47fa63860b39723b4e92a10f73dbbb8936c Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Sun, 18 Oct 2026 19:11:20 -0600 Subject: [PATCH 11/14] Document and rehearse complete Supabase rollback --- supabase/MIGRATION.md | 94 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index e675787660..f6fc096e62 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -72,6 +72,10 @@ docker exec supabase-db pg_dumpall -U supabase_admin \ db_config_volume=$(docker inspect supabase-db --format \ '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}') test -n "$db_config_volume" +printf '%s\n' "$db_config_volume" > "$migration_backup/db-config-volume" +docker inspect supabase-db --format \ + '{{index .Config.Labels "com.docker.compose.project"}}' \ + > "$migration_backup/compose-project" docker run --rm -v "$db_config_volume:/source:ro" \ -v "$migration_backup:/backup" \ alpine:3.22 tar -cf /backup/db-config.tar -C /source . @@ -363,23 +367,91 @@ Retain the encryption/key backups. Then reopen application writes. ## Rollback -While writes are still paused, stop the new stack without `-v`. Restore the saved -old Compose/API configuration and `.env`, the quiesced Postgres 15 data, the -MinIO directory, and the matching `db-config` encryption/configuration volume. -The database upgrade changes the volume's ownership/configuration, so restoring -only database files is insufficient. Use the physical backup and saved volume -archive, then load the saved images and use the pinned storage override: +Keep writes paused. Run the following in **Bash from the migrated deployment +directory**, using the same absolute `migration_backup` path as above. If using +a new shell, set that variable again first. The commands preserve the failed +deployment's data/configuration for diagnosis, then restore the coordinated +Postgres 15, MinIO/pgsty, and `db-config` backups. Restoring only database files +is insufficient: the upgrade also changes ownership/configuration in the +encryption volume. + +The saved Compose project name preserves the original named-volume mapping, +even if the deployment directory was renamed. The volume name comes from the +original database container, rather than guessing a name. Restore any additional +custom configuration/overrides from your inventory before restarting. ```sh +( +set -eu +: "${migration_backup:?Set migration_backup to the original absolute backup path}" +db_config_volume=$(cat "$migration_backup/db-config-volume") +compose_project=$(cat "$migration_backup/compose-project") +test -n "$db_config_volume" && test -n "$compose_project" +test -d "$migration_backup/postgres15-data" +test -d "$migration_backup/minio-data" +test -f "$migration_backup/.env" +test -f "$migration_backup/docker-compose.yml" +test -d "$migration_backup/api" +tar -tf "$migration_backup/db-config.tar" > /dev/null +docker volume inspect "$db_config_volume" > /dev/null docker image load -i "$migration_backup/storage-images.tar" -docker compose -f docker-compose.yml \ + +# Stop the upgraded stack with its current configuration. Never add -v. +docker compose --project-name "$compose_project" down +volume_users=$(docker ps -q --filter "volume=$db_config_volume") +if [ -n "$volume_users" ]; then + echo 'db-config is still in use; stop leftover upgrade containers before restoring.' >&2 + exit 1 +fi +umask 077 +failed_upgrade="$migration_backup/failed-upgrade-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir "$failed_upgrade" +cp -a .env docker-compose.yml "$failed_upgrade/" +docker run --rm -v "$db_config_volume:/source:ro" \ + -v "$failed_upgrade:/backup" \ + alpine:3.22 tar -cf /backup/db-config.tar -C /source . +for directory in volumes/db/data volumes/storage volumes/api; do + if [ -e "$directory" ]; then + sudo mv "$directory" "$failed_upgrade/$(basename "$directory")" + fi +done + +sudo cp -a "$migration_backup/postgres15-data" volumes/db/data +sudo cp -a "$migration_backup/minio-data" volumes/storage +sudo cp -a "$migration_backup/api" volumes/api +cp "$migration_backup/.env" .env +cp "$migration_backup/docker-compose.yml" docker-compose.yml + +# Remove PG17-only files, including hidden files, before restoring PG15's +# numeric ownership, permissions, encryption keys, and configuration. +docker run --rm -v "$db_config_volume:/restore" \ + -v "$migration_backup:/backup:ro" alpine:3.22 sh -ec ' + rm -rf /restore/* /restore/.[!.]* /restore/..?* + tar -xpf /backup/db-config.tar -C /restore + ' + +docker compose --project-name "$compose_project" -f docker-compose.yml \ + -f "$migration_backup/docker-compose.storage-images.yml" up -d --wait db +docker exec supabase-db psql -U supabase_admin -d postgres \ + -v ON_ERROR_STOP=1 -c 'SHOW server_version;' +docker compose --project-name "$compose_project" -f docker-compose.yml \ -f "$migration_backup/docker-compose.storage-images.yml" up -d +) +``` + +The subshell stops on any failed command; do not restart services after a failed +restore. It restores the same pgsty or MinIO image IDs that were running before +migration. Once the Auth and Storage APIs are ready, verify the restored objects +with the old service-role key and gateway URL, using the original `POSTGRES_DB`: + +```sh +python3 migrate-storage.py verify "$migration_backup/objects" \ + --url http://localhost:8000 --database "$postgres_database" ``` -Run this from the restored old deployment directory with its original `.env` -and Compose file. This restores the same pgsty or MinIO image IDs that were -running before migration. Verify data and login before reopening writes. Keep -the failed new data separately for diagnosis. +Check existing participants/answers and Study Designer login before reopening +writes. Keep the failed-upgrade directory and original backups until rollback +has been accepted. These are local copies, not automatic off-host backups. After new writes are accepted, restoring the old backups loses those writes. Pause and plan how to retain them rather than treating rollback as lossless. From 3492cbcb4da42a35bcc5ea9d1f85db5f0c94cdf0 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Thu, 22 Oct 2026 10:00:49 -0600 Subject: [PATCH 12/14] Pin upstream Auth TLS fix and verify migration activation --- supabase/MIGRATION.md | 36 +++++++++++++++++++++++++++++++- supabase/README.md | 16 ++++++++++++-- supabase/UPSTREAM | 4 ++++ supabase/docker-compose.auth.yml | 4 ++++ 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 supabase/docker-compose.auth.yml diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index f6fc096e62..1238dd5a2c 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -4,6 +4,10 @@ This procedure targets the previous bundled stack (Postgres 15.8.1.060, Supabase Storage v1.11.13, and either the original MinIO images or `main`'s `pgsty/silo` + `pgsty/mc` replacement images) and moves it to the release in [UPSTREAM](UPSTREAM), using Postgres 17 and Storage v1.81.0's default file backend. +Auth uses the required upstream `v2.198.0-rc.25` override to include the eMudhra +TLS fix; keep it enabled after migration and container recreation. This is a +release candidate, selected because the pinned bundle's stable Auth predates the +fix. See [README.md](README.md) for the upstream security change. Rehearse against a restored copy of your deployment first. New installations use [README.md](README.md); hosted Supabase users are unaffected. @@ -147,6 +151,7 @@ test "$(git -C ../supabase-upstream rev-parse HEAD)" = \ rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ --exclude='volumes/db/data*' --exclude='volumes/storage*' \ ../supabase-upstream/docker/ ./ && +cp "$revisit_support/docker-compose.auth.yml" . && cp "$revisit_support/docker-compose.github.yml" . ``` @@ -194,6 +199,8 @@ Before modifying database files, pull the target images and check the two Node services start on this host: ```sh +# Auth is required even if GitHub OAuth is disabled. +sh run.sh config add auth # The existing ENABLE_GITHUB_OAUTH value controls whether GitHub sign-in is enabled. sh run.sh config add github sh run.sh pull @@ -228,9 +235,17 @@ docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ -v ON_ERROR_STOP=1 < volumes/db/realtime.sql mkdir -p "$migration_backup/upgrade-tmp" sudo env TMPDIR="$migration_backup/upgrade-tmp" bash utils/upgrade-pg17.sh && -sudo rmdir "$migration_backup/upgrade-tmp" +sudo rmdir "$migration_backup/upgrade-tmp" && +sh run.sh recreate auth ``` +The pinned upgrade script starts services with explicit Compose files, briefly +using the bundle's older Auth without the GitHub override. The final command +immediately recreates Auth with the required TLS and GitHub overrides. Keep +traffic blocked throughout; if the script fails, keep Auth unused until the +fixed image is running. Do not use the script's printed startup commands for +normal operation; use `sh run.sh start` so all saved overrides apply. + Read all output, including migration warnings. This script uses upstream's `pg_upgrade` procedure, preserves `data.bak.pg15` and the encryption key, applies required role/extension changes, and starts the new stack. See @@ -344,6 +359,8 @@ It can be rerun with the same verified export after fixing an installation error python3 migrate-storage.py verify "$migration_backup/objects" \ --url http://localhost:8000 --database "$postgres_database" sh run.sh status +test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ + supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 docker exec supabase-db psql -U supabase_admin -d postgres \ -c 'SHOW server_version;' printf 'ref=self-hosted/v0.8.3\n' > .supabase-version @@ -449,6 +466,23 @@ python3 migrate-storage.py verify "$migration_backup/objects" \ --url http://localhost:8000 --database "$postgres_database" ``` +**Rollback also restores the old vulnerable Auth image.** Keep application +traffic blocked and GitHub OAuth/secure email unused until Auth is protected. +For the original bundled `v2.171.0` image, apply the certificate mitigation: + +```sh +compose_project=$(cat "$migration_backup/compose-project") +docker compose --project-name "$compose_project" -f docker-compose.yml \ + exec -T --user root auth apk add --no-cache --upgrade \ + 'ca-certificates>=20261020-r0' 'ca-certificates-bundle>=20261020-r0' && +docker compose --project-name "$compose_project" -f docker-compose.yml restart auth +``` + +Require a successful package update and healthy Auth before testing sign-in. +This mitigation is discarded if the old Auth container is recreated; complete the fixed-image migration before +normal operation, or reapply the mitigation after every recreation. If your +Auth image or certificate mounts differ, verify a deployment-specific fix first. + Check existing participants/answers and Study Designer login before reopening writes. Keep the failed-upgrade directory and original backups until rollback has been accepted. These are local copies, not automatic off-host backups. diff --git a/supabase/README.md b/supabase/README.md index 61c73e5038..158b9a9af7 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -7,6 +7,14 @@ default Storage backend uses persistent local files; MinIO is not required. For an existing deployment, follow [MIGRATION.md](MIGRATION.md) **before starting this configuration**. Hosted Supabase deployments do not need this migration. +The required Auth override pins upstream `v2.198.0-rc.25` by digest. This release +candidate includes Supabase's [Go 1.27.3 TLS fix](https://github.com/supabase/auth/pull/2883) +for the [eMudhra intermediate CA compromise](https://bugzilla.mozilla.org/show_bug.cgi?id=2083449). +The pinned bundle's default Auth `v2.197.0` predates that fix. Keep this override +enabled, including when recreating containers or updating the upstream bundle; +review a replacement stable Auth release before removing it. GitHub OAuth +configuration remains optional; the TLS fix is required for every deployment. + ## First-time setup Requirements: a Linux host with Git, Docker Engine, Docker Compose v2 or newer, OpenSSL, @@ -47,6 +55,8 @@ names: run only one copy per Docker host, or deliberately isolate it. sh utils/generate-keys.sh --update-env sh utils/add-new-auth-keys.sh --update-env printf 'ref=self-hosted/v0.8.3\n' > .supabase-version + cp "$revisit_support/docker-compose.auth.yml" . + sh run.sh config add auth ``` These helpers print secrets; keep their output private. Never commit `.env`. @@ -81,6 +91,8 @@ names: run only one copy per Docker host, or deliberately isolate it. sh run.sh pull sh run.sh start sh run.sh status + test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ + supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 ``` Open Studio at `http://localhost:8000` and use your generated @@ -147,8 +159,8 @@ relying on it, and stop if it is incomplete. Review future tagged updates using `sh update.sh --dry-run --to ` and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). The update tool belongs to the downloaded upstream bundle. Review its plan and -your GitHub override before applying an update; this guide and migration helper -target the pinned release above. This release includes Storage and Realtime +the required Auth override and your GitHub settings before applying an update; +this guide and migration helper target the pinned release above. This release includes Storage and Realtime security fixes. Its Storage database migrations are one-way: reverting an image alone is unsafe; restore coordinated pre-upgrade backups for rollback. Review [upstream release notes](https://github.com/supabase/supabase/blob/self-hosted/v0.8.3/docker/CHANGELOG.md) diff --git a/supabase/UPSTREAM b/supabase/UPSTREAM index 9181311d6c..87844a7268 100644 --- a/supabase/UPSTREAM +++ b/supabase/UPSTREAM @@ -4,3 +4,7 @@ Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.3/docker The deployment configuration and scripts are obtained directly from upstream. This directory contains only reVISit setup, migration, and integration files. + +Required override: docker-compose.auth.yml pins supabase/gotrue:v2.198.0-rc.25 +Digest: sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 +Reason: https://github.com/supabase/auth/pull/2883 (Go 1.27.3 eMudhra TLS fix) diff --git a/supabase/docker-compose.auth.yml b/supabase/docker-compose.auth.yml new file mode 100644 index 0000000000..7b7beab284 --- /dev/null +++ b/supabase/docker-compose.auth.yml @@ -0,0 +1,4 @@ +# Required until the pinned upstream bundle includes the eMudhra TLS fix. +services: + auth: + image: supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 From 7c663258595c53ac32cda20418a48d91afaf61da Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Thu, 22 Oct 2026 10:04:52 -0600 Subject: [PATCH 13/14] Stop release verification when the Auth image check fails --- supabase/MIGRATION.md | 14 +++++++++----- supabase/README.md | 9 ++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 1238dd5a2c..06690bc0bf 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -21,7 +21,7 @@ older Storage image cannot safely use that database. Rollback requires the coordinated pre-upgrade database, object-store, encryption and configuration backups below; changing image tags alone is insufficient. -Requirements: a Linux server, Git, Docker Compose, Bash, curl, OpenSSL, Python 3.11+, +Requirements: a Linux server, Git, Docker Compose, Bash, curl, rsync, OpenSSL, Python 3.11+, root access for the upstream database upgrade, and enough space for the database backup/upgrade and two copies of object data. Pre-pull the target images before starting the upgrade; allow at least 15 GB additional temporary space for the @@ -357,15 +357,19 @@ It can be rerun with the same verified export after fixing an installation error ```sh python3 migrate-storage.py verify "$migration_backup/objects" \ - --url http://localhost:8000 --database "$postgres_database" -sh run.sh status + --url http://localhost:8000 --database "$postgres_database" && +sh run.sh status && test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ - supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 + supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 && docker exec supabase-db psql -U supabase_admin -d postgres \ - -c 'SHOW server_version;' + -c 'SHOW server_version;' && printf 'ref=self-hosted/v0.8.3\n' > .supabase-version ``` +Require the entire verification block to succeed. If the Auth image check +fails, keep writes blocked, enable the required Auth override, run +`sh run.sh recreate auth`, and repeat verification. + Verification streams every migrated object through the new API and checks its SHA-256, Content-Type, and Cache-Control, plus the original database object metadata. Keep the legacy application URL/key unchanged when possible. Check Study Designer login, diff --git a/supabase/README.md b/supabase/README.md index 158b9a9af7..af6d7cb076 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -88,13 +88,16 @@ names: run only one copy per Docker host, or deliberately isolate it. 3. Pull and start the stack: ```sh - sh run.sh pull - sh run.sh start - sh run.sh status + sh run.sh pull && + sh run.sh start && + sh run.sh status && test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 ``` + Require this block to succeed, including the exact Auth image check. + If it fails, keep the deployment closed to users until corrected. + Open Studio at `http://localhost:8000` and use your generated `DASHBOARD_USERNAME` and `DASHBOARD_PASSWORD`. On a server, expose the API through HTTPS and restrict administration/database ports. The upstream From 5dc9fc5b53e9716efb8735ba5744e707bf5b9d81 Mon Sep 17 00:00:00 2001 From: Jack Wilburn Date: Thu, 22 Oct 2026 11:34:48 -0600 Subject: [PATCH 14/14] Harden Supabase migration preflight and verify recovery paths --- .github/workflows/vitest.yml | 4 +- supabase/MIGRATION.md | 111 +++++++++++++++++--- supabase/README.md | 16 +++ supabase/migrate-storage.py | 37 +++++-- supabase/prepare-pg17-upgrade.py | 34 ++++++ supabase/tests/migrate-storage.spec.py | 101 +++++++++++++++++- supabase/tests/prepare-pg17-upgrade.spec.py | 46 ++++++++ tests/supabase-live.spec.ts | 94 +++++++++++++++++ 8 files changed, 418 insertions(+), 25 deletions(-) create mode 100644 supabase/prepare-pg17-upgrade.py create mode 100644 supabase/tests/prepare-pg17-upgrade.spec.py diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index eab1e344a3..60b2fbecbd 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -26,4 +26,6 @@ jobs: run: yarn unittest - name: Run storage migration tests - run: python3 supabase/tests/migrate-storage.spec.py + run: | + python3 supabase/tests/migrate-storage.spec.py + python3 supabase/tests/prepare-pg17-upgrade.spec.py diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md index 06690bc0bf..369273f872 100644 --- a/supabase/MIGRATION.md +++ b/supabase/MIGRATION.md @@ -22,8 +22,12 @@ coordinated pre-upgrade database, object-store, encryption and configuration backups below; changing image tags alone is insufficient. Requirements: a Linux server, Git, Docker Compose, Bash, curl, rsync, OpenSSL, Python 3.11+, -root access for the upstream database upgrade, and enough space for the database -backup/upgrade and two copies of object data. Pre-pull the target images before +root access for the upstream database upgrade, and sufficient free disk space. +Allow at least **three additional object-data copies** beyond the current +object store (physical backup, API export, and new file storage), plus **two +additional database copies** (external PG15 backup and PG17 data). Allow extra +space for filesystem/per-object metadata and a safety margin; measure the +existing on-disk directories rather than relying only on logical object sizes. Pre-pull the target images before starting the upgrade; allow at least 15 GB additional temporary space for the upgrade images, extracted binaries, and compressed bundle (also inside the Docker VM when using Docker Desktop). The command below puts build files next @@ -100,16 +104,51 @@ shell history or in the application's `VITE_*` variables. ```sh python3 migrate-storage.py export "$migration_backup/objects" \ - --url http://localhost:8000 --database "$postgres_database" + --url http://localhost:8000 --database "$postgres_database" && +python3 migrate-storage.py check "$migration_backup/objects" \ + --bucket stub --tenant stub ``` +Use your preserved `GLOBAL_S3_BUCKET` and `TENANT_ID` instead of `stub` for +`check` and installation. The check validates every export checksum and rejects +S3 keys that collide as file/directory paths, before any database upgrade. Such +a layout requires retaining an S3 backend with a separately rehearsed plan. +Export also refuses to publish a manifest if object metadata changes while it +runs; keep all writers paused and repeat into a new export directory. + Use your actual gateway URL and preserved `POSTGRES_DB` for both export and verification. Keep `postgres_database` in the same shell session. Export includes -all live objects in every Supabase bucket, not only `revisit`, and fails on missing objects or size mismatches. +all live objects in every Supabase bucket, not only `revisit`, and fails on +missing objects, truncated HTTP responses, or size mismatches. The export directory must not already exist. An interrupted export must be repeated into a new directory; only a completed export contains `manifest.json`. It does not copy orphaned MinIO objects absent from the database. +Snapshot the bundled application records, bucket settings, and access policies +while writes remain paused. Keep this private alongside the other backups: + +```sh +cat > "$migration_backup/check-application.sql" <<'SQL' +SELECT json_build_object( + 'records', (SELECT json_agg(r ORDER BY "studyId", "docId") FROM public.revisit r), + 'buckets', (SELECT json_agg(b ORDER BY id) FROM ( + SELECT id, name, owner, owner_id, public, created_at, updated_at, + file_size_limit, allowed_mime_types + FROM storage.buckets + ) b), + 'policies', (SELECT json_agg(p ORDER BY schemaname, tablename, policyname) + FROM pg_policies p WHERE schemaname IN ('public', 'storage')) +); +SQL +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-before.json" +``` + +The final verification repeats this query and compares its output. Add separate +checks for any custom application tables, schemas, extensions, or encrypted data +identified in your inventory; the bundled snapshot does not certify those. + Stop the old stack **using its old Compose file**, without deleting volumes: ```sh @@ -215,8 +254,22 @@ Move the old MinIO data aside and create an empty file-storage directory: ```sh mv volumes/storage volumes/storage.minio-backup mkdir volumes/storage +sudo python3 - <<'PYTHON' +import os +from pathlib import Path +import tempfile +with tempfile.TemporaryDirectory(dir="volumes/storage") as temporary: + probe = Path(temporary) / "xattr-probe" + probe.write_bytes(b"storage preflight") + os.setxattr(probe, "user.supabase.content-type", b"application/octet-stream") + assert os.getxattr(probe, "user.supabase.content-type") == b"application/octet-stream" +PYTHON ``` +Require the filesystem probe to succeed before upgrading the database. An +unsupported or unwritable filesystem must be fixed while the original database +and object-store backups are still untouched. + Do **not** apply `revisit.sql`: existing application schema, buckets, and policies are retained by the database upgrade. @@ -233,29 +286,38 @@ docker exec supabase-db pg_isready -U postgres # Our previous Compose did not initialize the upstream Realtime schema. docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ -v ON_ERROR_STOP=1 < volumes/db/realtime.sql -mkdir -p "$migration_backup/upgrade-tmp" -sudo env TMPDIR="$migration_backup/upgrade-tmp" bash utils/upgrade-pg17.sh && +python3 "$revisit_support/prepare-pg17-upgrade.py" \ + utils/upgrade-pg17.sh "$migration_backup/upgrade-pg17.revisit.sh" && +mkdir -p "$migration_backup/upgrade-tmp" && +sudo env TMPDIR="$migration_backup/upgrade-tmp" \ + bash "$migration_backup/upgrade-pg17.revisit.sh" && sudo rmdir "$migration_backup/upgrade-tmp" && sh run.sh recreate auth ``` -The pinned upgrade script starts services with explicit Compose files, briefly -using the bundle's older Auth without the GitHub override. The final command -immediately recreates Auth with the required TLS and GitHub overrides. Keep -traffic blocked throughout; if the script fails, keep Auth unused until the -fixed image is running. Do not use the script's printed startup commands for -normal operation; use `sh run.sh start` so all saved overrides apply. +The preparation helper leaves the downloaded upstream script unchanged and +writes a deployment copy that selects the `db-config` volume actually mounted +by `supabase-db`, starts services using the saved Auth/GitHub overrides, and +aborts on a failed required SQL migration. It rejects an unexpected upstream +script before writing the prepared copy. Do not run the unprepared script: +it guesses the encryption volume, ignores saved overrides during startup, and +can report success after required migration errors. The final Auth recreation +and image check confirm the selected configuration. Keep traffic blocked +throughout; use `sh run.sh start` for normal operation rather than the original +script's printed commands. Read all output, including migration warnings. This script uses upstream's `pg_upgrade` procedure, preserves `data.bak.pg15` and the encryption key, applies required role/extension changes, and starts the new stack. See [upstream database upgrade guidance](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17). -On a host with multiple `db-config` volumes, stop and resolve the target before -running: this upstream script selects a matching volume by name. +The prepared script selects the running database's actual named encryption +volume even when unrelated or leftover `db-config` volumes exist. A missing +volume mount is an error; do not substitute another volume. Require a successful script exit and healthy services. -**Upgrade complete is not sufficient:** the pinned script warns about failed -required SQL migrations and can still report success. Keep writes blocked and +**Upgrade complete is not sufficient:** the prepared copy stops on required +migration-file failures, but upstream extension reconciliation can still warn +and continue. Keep writes blocked and **stop here** on missing migration files, SQL errors, or migration/extension reconciliation failures. Resolve each failure and reapply only its affected SQL from `/docker-entrypoint-initdb.d/migrations/` in the target database container @@ -352,10 +414,19 @@ retain their private permissions. Each payload and its HTTP metadata are staged on the destination filesystem before an atomic rename, so a failed copy leaves the final object untouched. It can be rerun with the same verified export after fixing an installation error. +A forcibly terminated process can leave `.revisit-migration-*` staging +directories beside its target objects. Once no migration process is running, +remove only those incomplete staging directories if they consume needed disk +space; preserve the exported backup and final object paths. Rerun installation +and verification before reopening writes. ## 4. Verify before reopening writes ```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-after.json" && +diff "$migration_backup/application-before.json" "$migration_backup/application-after.json" && python3 migrate-storage.py verify "$migration_backup/objects" \ --url http://localhost:8000 --database "$postgres_database" && sh run.sh status && @@ -366,7 +437,9 @@ docker exec supabase-db psql -U supabase_admin -d postgres \ printf 'ref=self-hosted/v0.8.3\n' > .supabase-version ``` -Require the entire verification block to succeed. If the Auth image check +Require the entire verification block to succeed. Any unexplained record, +bucket, or policy difference must be resolved before reopening writes. Run +these comparisons before the smoke tests create new participant data. If the Auth image check fails, keep writes blocked, enable the required Auth override, run `sh run.sh recreate auth`, and repeat verification. @@ -466,6 +539,10 @@ migration. Once the Auth and Storage APIs are ready, verify the restored objects with the old service-role key and gateway URL, using the original `POSTGRES_DB`: ```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-restored.json" && +diff "$migration_backup/application-before.json" "$migration_backup/application-restored.json" && python3 migrate-storage.py verify "$migration_backup/objects" \ --url http://localhost:8000 --database "$postgres_database" ``` diff --git a/supabase/README.md b/supabase/README.md index af6d7cb076..7cb8226f65 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -178,6 +178,7 @@ Python 3.11+ and no third-party Python packages: ```sh python3 supabase/tests/migrate-storage.spec.py +python3 supabase/tests/prepare-pg17-upgrade.spec.py ``` The opt-in Chromium test is `tests/supabase-live.spec.ts`. It needs an isolated @@ -188,3 +189,18 @@ repository root with `yarn exec playwright test tests/supabase-live.spec.ts --pr Use `PW_SUPABASE_PARTICIPANT_ID` to retain a known UUID before migration and `PW_SUPABASE_EXISTING_PARTICIPANT` to verify it afterwards. The test creates real participant records; do not point it at production. + +For an unfinished-participant rehearsal, set `PW_SUPABASE_RESUME_PARTICIPANT` to +a fresh UUID and `PW_SUPABASE_SEED_RESUME=1` before migration. Afterwards retain +the UUID and remove `PW_SUPABASE_SEED_RESUME`; the test retrieves the earlier +answer in a fresh browser and completes the study. This consumes the unfinished +fixture, so omit that UUID on subsequent recreation runs. + +The Designer export check additionally needs private +`PW_SUPABASE_DESIGNER_EMAIL` / `PW_SUPABASE_DESIGNER_PASSWORD` values for a +confirmed test account authorized in the fixture's user-management settings, +and `PW_SUPABASE_EXISTING_PARTICIPANT` for the original completed participant. +It uses real password Auth locally to check the retained account and protected +JSON/CSV export UI. The application's GitHub sign-in button still requires a +separate real-provider smoke test after deployment; this local test does not +claim external OAuth or SMTP delivery coverage. diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py index c851c5fe61..4e48d282f9 100644 --- a/supabase/migrate-storage.py +++ b/supabase/migrate-storage.py @@ -43,18 +43,24 @@ def digest(file): def export(url, directory, database="postgres"): directory.mkdir(parents=True, exist_ok=False) - manifest = objects(database) + original = objects(database) + manifest = [dict(obj) for obj in original] for index, obj in enumerate(manifest): obj["file"] = f"{index:08d}.bin" target = directory / obj["file"] with download(url, obj) as response, target.open("wb") as output: + content_length = response.headers.get("Content-Length") obj["content_type"] = response.headers.get("Content-Type", "application/octet-stream") obj["cache_control"] = response.headers.get("Cache-Control", "no-cache") shutil.copyfileobj(response, output) + if content_length is not None and target.stat().st_size != int(content_length): + raise ValueError("Downloaded object size differs from HTTP Content-Length") obj["sha256"] = digest(target) size = (obj.get("metadata") or {}).get("size") if size is not None and target.stat().st_size != int(size): raise ValueError("Downloaded object size differs from database metadata") + if objects(database) != original: + raise ValueError("Storage metadata changed during export; keep writes paused and export again") (directory / "manifest.json").write_text(json.dumps(manifest, indent=2)) print(f"Exported {len(manifest)} objects") @@ -72,17 +78,34 @@ def destination(root, bucket, tenant, obj): return target -def install(directory, root, bucket, tenant): - if not hasattr(os, "setxattr"): - raise RuntimeError("Install requires Linux extended attributes; see MIGRATION.md") +def validate_export(directory, root, bucket, tenant): manifest = json.loads((directory / "manifest.json").read_text()) + targets = set() # Validate the entire export before changing the destination. for index, obj in enumerate(manifest): if obj["file"] != f"{index:08d}.bin" or digest(directory / obj["file"]) != obj["sha256"]: raise ValueError("Export checksum mismatch") target = destination(root, bucket, tenant, obj) + if target in targets: + raise ValueError("Storage objects map to the same file path") + targets.add(target) if target.exists() and digest(target) != obj["sha256"]: raise ValueError("Destination already contains different data") + if any(parent in targets for target in targets for parent in target.parents): + raise ValueError("Storage object paths conflict as files and directories; retain S3 storage") + return manifest + + +def check(directory, bucket, tenant): + with tempfile.TemporaryDirectory() as temporary: + manifest = validate_export(directory, Path(temporary) / "storage", bucket, tenant) + print(f"Checked {len(manifest)} objects for file-backend compatibility") + + +def install(directory, root, bucket, tenant): + if not hasattr(os, "setxattr"): + raise RuntimeError("Install requires Linux extended attributes; see MIGRATION.md") + manifest = validate_export(directory, root, bucket, tenant) root.mkdir(parents=True, exist_ok=True) root.chmod(0o755) for obj in manifest: @@ -129,7 +152,7 @@ def verify(url, directory, database="postgres"): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=["export", "install", "verify"]) + parser.add_argument("operation", choices=["export", "check", "install", "verify"]) parser.add_argument("directory", type=Path) parser.add_argument("--url", default="http://localhost:8000") parser.add_argument("--database", default="postgres", help="POSTGRES_DB for export and verify") @@ -137,7 +160,9 @@ def main(): parser.add_argument("--bucket", default="stub", help="GLOBAL_S3_BUCKET, not the application bucket") parser.add_argument("--tenant", default="stub", help="STORAGE_TENANT_ID (previously TENANT_ID)") args = parser.parse_args() - if args.operation == "install": + if args.operation == "check": + check(args.directory, args.bucket, args.tenant) + elif args.operation == "install": if args.destination is None: parser.error("install requires --destination") install(args.directory, args.destination, args.bucket, args.tenant) diff --git a/supabase/prepare-pg17-upgrade.py b/supabase/prepare-pg17-upgrade.py new file mode 100644 index 0000000000..bbcff05b41 --- /dev/null +++ b/supabase/prepare-pg17-upgrade.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Prepare the pinned upstream upgrade script for this deployment's saved overrides.""" +import argparse +from pathlib import Path + + +VOLUME_SELECTION = '''db_config_vol=$(docker volume ls --filter "name=db-config" --format '{{.Name}}' \\ + | grep -E '^db-config$|_db-config$' | head -n 1)''' +CONTAINER_VOLUME = '''db_config_vol=$(docker inspect "$DB_CONTAINER" --format \\ + '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}')''' +STARTUP = 'docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d' +FAILED_MIGRATION = '|| warn " $m failed (non-fatal)"' + + +def prepare(source, target): + if source.resolve() == target.resolve(): + raise ValueError("Keep the original upstream script unchanged") + text = source.read_text() + # Fail before writing if a different upstream release changes any assumption. + for old, new in ((f" {VOLUME_SELECTION}\n", f" {CONTAINER_VOLUME}\n"), + (f" {STARTUP}\n", " docker compose up -d\n"), + (FAILED_MIGRATION, '|| die "Required migration $m failed; keep writes blocked"')): + if text.count(old) != 1: + raise ValueError("Unexpected upstream upgrade script; review this release before upgrading") + text = text.replace(old, new) + target.write_text(text) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("target", type=Path) + args = parser.parse_args() + prepare(args.source, args.target) diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py index 46cc01c1d4..42ee2c4ac2 100644 --- a/supabase/tests/migrate-storage.spec.py +++ b/supabase/tests/migrate-storage.spec.py @@ -7,7 +7,7 @@ import shutil import tempfile import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch migration = runpy.run_path(str(Path(__file__).parents[1] / "migrate-storage.py")) @@ -129,6 +129,105 @@ def download(url, obj): with self.assertRaisesRegex(ValueError, "metadata changed"): migration["verify"]("http://example.test", self.export) + def test_file_directory_collisions_fail_before_installing_any_payload(self): + second = {**self.obj, "file": "00000001.bin", + "name": "study/participant data.json/existing-version/child"} + (self.export / second["file"]).write_bytes(b"existing participant data") + self.manifest([self.obj, second]) + with self.assertRaisesRegex(ValueError, "conflict as files"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + with self.assertRaisesRegex(ValueError, "conflict as files"): + migration["check"](self.export, "stub", "stub") + + def test_duplicate_paths_and_missing_exports_fail_before_install(self): + for second in ({**self.obj, "file": "00000001.bin"}, + {**self.obj, "file": "00000001.bin", "name": "different"}): + with self.subTest(name=second["name"]): + self.manifest([self.obj, second]) + if second["name"] == self.obj["name"]: + (self.export / second["file"]).write_bytes(b"existing participant data") + with self.assertRaisesRegex(ValueError, "same file path"): + migration["install"](self.export, self.target, "stub", "stub") + (self.export / second["file"]).unlink() + else: + with self.assertRaises(FileNotFoundError): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + + def test_symlink_escape_is_rejected_without_writing_outside_storage(self): + self.manifest([self.obj]) + self.target.mkdir() + outside = Path(self.temp.name) / "outside" + outside.mkdir() + (self.target / "stub").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "escapes destination"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(list(outside.iterdir()), []) + + def test_metadata_failure_keeps_payload_unpublished_and_allows_retry(self): + self.manifest([self.obj]) + with patch("os.setxattr", side_effect=OSError("extended attributes unavailable")): + with self.assertRaisesRegex(OSError, "extended attributes"): + migration["install"](self.export, self.target, "stub", "stub") + target = migration["destination"](self.target, "stub", "stub", self.obj) + self.assertFalse(target.exists()) + self.assertFalse(list(self.target.rglob(".revisit-migration-*"))) + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(target.read_bytes(), b"existing participant data") + + def test_concurrent_writes_and_interrupted_export_never_publish_a_manifest(self): + def response(url, obj): + stream = BytesIO(b"existing participant data") + stream.headers = {"Content-Type": "application/json", "Cache-Control": "max-age=3600"} + return stream + + original = [{"id": "original", "bucket_id": "revisit", "name": "recording", + "metadata": {"size": len(b"existing participant data")}}] + directory = self.export / "incomplete" + with patch.dict(migration["export"].__globals__, download=response, + objects=Mock(side_effect=[original, [{**original[0], "version": "new"}]])): + with self.assertRaisesRegex(ValueError, "metadata changed during export"): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + with self.assertRaises(FileExistsError): + migration["export"]("http://example.test", directory) + for failure in (OSError("connection interrupted"), ValueError("size differs")): + directory = directory.with_name(directory.name + "-retry") + with patch.dict(migration["export"].__globals__, objects=lambda database: original, + download=Mock(side_effect=failure)): + with self.assertRaises(type(failure)): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + + def test_truncated_http_response_without_database_size_never_completes_export(self): + stream = BytesIO(b"partial") + stream.headers = {"Content-Length": "100"} + directory = self.export / "truncated" + obj = {"id": "legacy", "bucket_id": "revisit", "name": "legacy", "metadata": None} + with patch.dict(migration["export"].__globals__, objects=lambda database: [obj], + download=lambda url, item: stream): + with self.assertRaisesRegex(ValueError, "Content-Length"): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + + def test_verify_rejects_wrong_bytes_headers_and_added_objects(self): + obj = {**self.obj, "id": "existing"} + self.manifest([obj]) + for body, content_type, cache in ((b"corrupted", obj["content_type"], obj["cache_control"]), + (b"existing participant data", "text/plain", obj["cache_control"]), + (b"existing participant data", obj["content_type"], "no-cache")): + stream = BytesIO(body) + stream.headers = {"Content-Type": content_type, "Cache-Control": cache} + with patch.dict(migration["verify"].__globals__, objects=lambda database: [obj], + download=lambda url, item: stream): + with self.assertRaises(ValueError): + migration["verify"]("http://example.test", self.export) + with patch.dict(migration["verify"].__globals__, + objects=lambda database: [obj, {**obj, "id": "new"}]): + with self.assertRaisesRegex(ValueError, "metadata changed"): + migration["verify"]("http://example.test", self.export) + def test_rejects_traversal_and_conflicting_destination(self): self.manifest([{**self.obj, "name": "../outside"}]) with self.assertRaisesRegex(ValueError, "Unsafe"): diff --git a/supabase/tests/prepare-pg17-upgrade.spec.py b/supabase/tests/prepare-pg17-upgrade.spec.py new file mode 100644 index 0000000000..58811f1229 --- /dev/null +++ b/supabase/tests/prepare-pg17-upgrade.spec.py @@ -0,0 +1,46 @@ +import runpy +from pathlib import Path +import tempfile +import unittest + + +helper = runpy.run_path(str(Path(__file__).parents[1] / "prepare-pg17-upgrade.py")) + + +class PrepareUpgradeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.source = Path(self.temp.name) / "upstream.sh" + self.target = Path(self.temp.name) / "prepared.sh" + self.original = (f" {helper['VOLUME_SELECTION']}\n" + f" {helper['STARTUP']}\n" + f" psql -f migration.sql {helper['FAILED_MIGRATION']}\n") + self.source.write_text(self.original) + + def test_preserves_source_selects_actual_volume_and_applies_saved_overrides(self): + helper["prepare"](self.source, self.target) + self.assertEqual(self.source.read_text(), self.original) + prepared = self.target.read_text() + self.assertIn(helper["CONTAINER_VOLUME"], prepared) + self.assertIn(" docker compose up -d\n", prepared) + self.assertNotIn('|| warn', prepared) + self.assertIn('|| die "Required migration $m failed; keep writes blocked"', prepared) + + def test_unknown_or_duplicated_upstream_commands_do_not_write_output(self): + for changed in (self.original.replace(helper["STARTUP"], "different command"), + self.original + f" {helper['VOLUME_SELECTION']}\n"): + with self.subTest(changed=changed): + self.source.write_text(changed) + with self.assertRaisesRegex(ValueError, "Unexpected upstream"): + helper["prepare"](self.source, self.target) + self.assertFalse(self.target.exists()) + + def test_rejects_overwriting_upstream_source(self): + with self.assertRaisesRegex(ValueError, "original upstream"): + helper["prepare"](self.source, self.source) + self.assertEqual(self.source.read_text(), self.original) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/supabase-live.spec.ts b/tests/supabase-live.spec.ts index a59f5b0418..81160c9f2c 100644 --- a/tests/supabase-live.spec.ts +++ b/tests/supabase-live.spec.ts @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ import { randomUUID } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; import { createClient } from '@supabase/supabase-js'; import { expect, Page, test } from '@playwright/test'; import { ParticipantData } from '../src/storage/types'; @@ -111,6 +112,99 @@ test.describe('Live self-hosted Supabase', () => { expect(colorAnswers(await readParticipant(participantId!))).toEqual(colorAnswers(participant)); }); + test('resumes an unfinished pre-migration participant and preserves the earlier answer', async ({ page }) => { + const participantId = process.env.PW_SUPABASE_RESUME_PARTICIPANT; + test.skip(!participantId, 'Set PW_SUPABASE_RESUME_PARTICIPANT for the unfinished participant rehearsal'); + const seed = process.env.PW_SUPABASE_SEED_RESUME === '1'; + const previous = seed ? null : await readParticipant(participantId!); + const earlierAnswer = Object.values(previous?.answers ?? {}) + .find((answer) => answer.componentName.startsWith('color-trial-') && Object.values(answer.answer).includes('A')); + if (!seed) { + expect(previous?.participantId).toBe(participantId); + expect(earlierAnswer).toBeTruthy(); + expect(colorAnswers(previous).flatMap((answer) => Object.values(answer))).not.toContain('B'); + } + + await selectParticipant(page, participantId!); + await page.goto('/demo-condition?condition=color'); + await nextClick(page); + const question = page.getByText(/Which color is the (lightest|darkest)\?/); + await expect(question).toBeVisible(); + const firstQuestion = await question.textContent(); + const response = page.getByPlaceholder('Select an option'); + if (seed) { + await response.click(); + await page.getByRole('option', { name: 'A', exact: true }).click(); + } else { + await expect(response).toHaveValue('A'); + } + await nextClick(page); + await expect(question).toBeVisible(); + await expect(question).not.toHaveText(firstQuestion!); + await expect(response).toHaveValue(''); + await expect.poll(async () => colorAnswers(await readParticipant(participantId!)) + .flatMap((answer) => Object.values(answer))).toContain('A'); + + if (!seed) { + await response.click(); + await page.getByRole('option', { name: 'B', exact: true }).click(); + await nextClick(page); + await waitForStudyEndMessage(page); + await expect.poll(async () => colorAnswers(await readParticipant(participantId!)) + .flatMap((answer) => Object.values(answer))).toEqual(expect.arrayContaining(['A', 'B'])); + const current = await readParticipant(participantId!); + expect(Object.values(current!.answers).find((answer) => answer.componentName === earlierAnswer!.componentName)?.answer) + .toEqual(earlierAnswer!.answer); + } + }); + + test('authenticates a retained Designer account and exports retained answers through the UI', async ({ page }) => { + const email = process.env.PW_SUPABASE_DESIGNER_EMAIL; + const password = process.env.PW_SUPABASE_DESIGNER_PASSWORD; + const participantId = process.env.PW_SUPABASE_EXISTING_PARTICIPANT; + test.skip(!email || !password || !participantId, 'Set Designer credentials and the retained participant ID'); + + await page.goto('/analysis/stats/demo-condition/manage'); + await expect(page.getByRole('button', { name: 'Sign In With Supabase' })).toBeVisible(); + + // The app's sign-in button uses external GitHub OAuth. Use a real Auth password + // session here to exercise account retention, authorization and exports locally. + const url = process.env.VITE_SUPABASE_URL!; + const client = createClient(url, process.env.VITE_SUPABASE_ANON_KEY!, { + auth: { persistSession: false, autoRefreshToken: false }, + }); + const { data, error } = await client.auth.signInWithPassword({ email: email!, password: password! }); + expect(error).toBeNull(); + expect(data.user?.email).toBe(email); + const storageKey = `sb-${new URL(url).hostname.split('.')[0]}-auth-token`; + await page.evaluate(({ key, session }) => localStorage.setItem(key, JSON.stringify(session)), { + key: storageKey, session: data.session, + }); + await page.goto('/analysis/stats/demo-condition/table'); + await expect(page.getByRole('columnheader', { name: /^ID Sort by ID/ })).toBeVisible(); + + const jsonButton = page.locator('button').filter({ has: page.locator('svg.tabler-icon-database-export') }); + await expect(jsonButton).toBeEnabled(); + const jsonDownload = page.waitForEvent('download'); + await jsonButton.click(); + const downloaded = await jsonDownload; + expect(await downloaded.failure()).toBeNull(); + const participants = JSON.parse(await readFile((await downloaded.path())!, 'utf8')) as ParticipantData[]; + const retained = participants.find((participant) => participant.participantId === participantId); + expect(retained).toBeTruthy(); + expect(retained?.answers).toEqual((await readParticipant(participantId!))?.answers); + + const csvButton = page.locator('button').filter({ has: page.locator('svg.tabler-icon-table-export') }); + await csvButton.click(); + const exporter = page.getByRole('dialog', { name: 'Tidy CSV Exporter' }); + await expect(exporter).toBeVisible(); + const csvDownload = page.waitForEvent('download'); + await exporter.getByRole('button', { name: 'Download', exact: true }).click(); + const csv = await csvDownload; + expect(await csv.failure()).toBeNull(); + expect(await readFile((await csv.path())!, 'utf8')).toContain(participantId!); + }); + test('supports binary files, overwrites and signed downloads with a participant session', async ({ request }) => { const client = createClient(process.env.VITE_SUPABASE_URL!, process.env.VITE_SUPABASE_ANON_KEY!, { auth: { persistSession: false, autoRefreshToken: false },