diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 3eb55bca67..60b2fbecbd 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -24,3 +24,8 @@ jobs: - name: Run unit tests run: yarn unittest + + - name: Run storage migration tests + run: | + python3 supabase/tests/migrate-storage.spec.py + python3 supabase/tests/prepare-pg17-upgrade.spec.py diff --git a/.gitignore b/.gitignore index 0289daf5b5..859f88a055 100644 --- a/.gitignore +++ b/.gitignore @@ -33,9 +33,7 @@ docsLibraries/ public/__revisit-widget src/public/__revisit-widget -supabase/volumes/* -!supabase/volumes/db/ -supabase/volumes/db/data -!supabase/volumes/api/ +supabase/volumes/ +/supabase-migration-backup/ coverage/ diff --git a/README.md b/README.md index 849de05186..ec0130207d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ To run this demo experiment locally, you will need to install Node.js on your co * To run locally, run `yarn serve`. * Go to [http://localhost:8080](http://localhost:8080) to view it in your browser. The page will reload when you make changes. +## Self-hosted Supabase + +See [first-time setup](supabase/README.md) and the [migration guide](supabase/MIGRATION.md) for a version-pinned upstream Supabase deployment. Existing MinIO or pgsty-backed installations must migrate before starting the new configuration. + ## Adding Tests This repo uses two test types: diff --git a/supabase/.env b/supabase/.env deleted file mode 100644 index 7e1170ea56..0000000000 --- a/supabase/.env +++ /dev/null @@ -1,92 +0,0 @@ -########## -# Secrets -# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION -############ - -POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password -JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long -ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE -SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated - - -############ -# Database - You can change these to any PostgreSQL database that has logical replication enabled. -############ - -POSTGRES_HOST=db -POSTGRES_DB=postgres -POSTGRES_PORT=5432 -# default user is postgres - - -############ -# API Proxy - Configuration for the Kong Reverse proxy. -############ - -KONG_HTTP_PORT=8000 -KONG_HTTPS_PORT=8443 - - -############ -# API - Configuration for PostgREST. -############ - -PGRST_DB_SCHEMAS=public,storage,graphql_public - - -############ -# Auth - Configuration for the GoTrue authentication server. -############ -ENABLE_GITHUB_OAUTH=true -GITHUB_OAUTH_CLIENT_ID=your-github-oauth-client-id -GITHUB_OAUTH_SECRET=your-github-oauth-client-secret -GITHUB_OAUTH_REDIRECT_URI=http://localhost:8000/auth/v1/callback - -## General -SITE_URL=http://localhost:3000 -ADDITIONAL_REDIRECT_URLS= -JWT_EXPIRY=3600 -DISABLE_SIGNUP=false -API_EXTERNAL_URL=http://localhost:8000 - -## Mailer Config -MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" -MAILER_URLPATHS_INVITE="/auth/v1/verify" -MAILER_URLPATHS_RECOVERY="/auth/v1/verify" -MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" - -## Email auth -ENABLE_EMAIL_SIGNUP=true -ENABLE_EMAIL_AUTOCONFIRM=false -SMTP_ADMIN_EMAIL=admin@example.com -SMTP_HOST=supabase-mail -SMTP_PORT=2500 -SMTP_USER=fake_mail_user -SMTP_PASS=fake_mail_password -SMTP_SENDER_NAME=fake_sender -ENABLE_ANONYMOUS_USERS=true - -## Phone auth -ENABLE_PHONE_SIGNUP=true -ENABLE_PHONE_AUTOCONFIRM=true - - -############ -# Studio - Configuration for the Dashboard -############ - -STUDIO_DEFAULT_ORGANIZATION=Default Organization -STUDIO_DEFAULT_PROJECT=Default Project - -# replace if you intend to use Studio outside of localhost -SUPABASE_PUBLIC_URL=http://localhost:8000 - -############ -# Logs - Configuration for Logflare -# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction -############ - -# Change vector.toml sinks to reflect this change -LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key diff --git a/supabase/.gitignore b/supabase/.gitignore new file mode 100644 index 0000000000..fb726cc7ad --- /dev/null +++ b/supabase/.gitignore @@ -0,0 +1,5 @@ +.env +.env.old +volumes/ +.supabase-version +backups/ diff --git a/supabase/MIGRATION.md b/supabase/MIGRATION.md new file mode 100644 index 0000000000..369273f872 --- /dev/null +++ b/supabase/MIGRATION.md @@ -0,0 +1,572 @@ +# Migrate the bundled MinIO or pgsty deployment + +This procedure targets the previous bundled stack (Postgres 15.8.1.060, +Supabase Storage v1.11.13, and either the original MinIO images or `main`'s +`pgsty/silo` + `pgsty/mc` replacement images) and moves it to the release in +[UPSTREAM](UPSTREAM), using Postgres 17 and Storage v1.81.0's default file backend. +Auth uses the required upstream `v2.198.0-rc.25` override to include the eMudhra +TLS fix; keep it enabled after migration and container recreation. This is a +release candidate, selected because the pinned bundle's stable Auth predates the +fix. See [README.md](README.md) for the upstream security change. +Rehearse against a restored copy of your deployment first. New installations +use [README.md](README.md); hosted Supabase users are unaffected. + +The database upgrade and storage move are separate steps in **one maintenance +window**. Storage object IDs, versions, owners, timestamps, bucket definitions, +and policies remain in Postgres. `migrate-storage.py` exports object bytes through +the old API and installs their versioned files with HTTP metadata in the new +backend. It never writes database records or re-uploads objects through the API. +Storage v1.81.0 itself applies one-way database migrations: after it starts, an +older Storage image cannot safely use that database. Rollback requires the +coordinated pre-upgrade database, object-store, encryption and configuration +backups below; changing image tags alone is insufficient. + +Requirements: a Linux server, Git, Docker Compose, Bash, curl, rsync, OpenSSL, Python 3.11+, +root access for the upstream database upgrade, and sufficient free disk space. +Allow at least **three additional object-data copies** beyond the current +object store (physical backup, API export, and new file storage), plus **two +additional database copies** (external PG15 backup and PG17 data). Allow extra +space for filesystem/per-object metadata and a safety margin; measure the +existing on-disk directories rather than relying only on logical object sizes. Pre-pull the target images before +starting the upgrade; allow at least 15 GB additional temporary space for the +upgrade images, extracted binaries, and compressed bundle (also inside the +Docker VM when using Docker Desktop). The command below puts build files next +to the backup, avoiding a small or memory-backed `/tmp`. The destination +filesystem must support Linux extended attributes. Do not reuse MinIO's on-disk directory as +Supabase file storage. Custom S3 layouts, custom object-version separators, or +multiple Supabase stacks on one Docker host require a separately rehearsed plan. + +## 1. Inventory and export while the old stack is running + +Keep the old Compose/configuration files and images available for rollback. +Save the exact cached server/client images used by the old containers, including +the pgsty images when migrating `main`. Do not replace Silo with an old MinIO +image or change object-store versions during this migration. +Record your database extensions, OAuth/SMTP settings, proxy configuration, +bucket limits, storage tenant, backend bucket, and any custom access policies. +Restrict application traffic and pause participant/designer/background writes +for the remainder of the procedure. Keep Supabase accessible to the migration +operator until export finishes. + +Obtain the new reVISit checkout separately; do not update the old deployment +in place before taking backups, because this change removes its bundled +configuration files. Copy `migrate-storage.py` from the new checkout to the +**old deployment directory**. +From that old directory, create a private backup outside the deployment: + +```sh +umask 077 +postgres_database=postgres # Set this to the existing .env POSTGRES_DB. +migration_backup="$(cd .. && pwd)/supabase-migration-backup" +mkdir "$migration_backup" +cp .env docker-compose.yml "$migration_backup/" +cp -a volumes/api "$migration_backup/api" +storage_server_image=$(docker inspect "$(docker compose ps -q minio)" \ + --format '{{.Image}}') +storage_client_image=$(docker inspect "$(docker compose ps -a -q minio-createbucket)" \ + --format '{{.Image}}') +test -n "$storage_server_image" && test -n "$storage_client_image" && +docker image save "$storage_server_image" "$storage_client_image" \ + -o "$migration_backup/storage-images.tar" && +cat > "$migration_backup/docker-compose.storage-images.yml" < "$migration_backup/database.sql" +db_config_volume=$(docker inspect supabase-db --format \ + '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}') +test -n "$db_config_volume" +printf '%s\n' "$db_config_volume" > "$migration_backup/db-config-volume" +docker inspect supabase-db --format \ + '{{index .Config.Labels "com.docker.compose.project"}}' \ + > "$migration_backup/compose-project" +docker run --rm -v "$db_config_volume:/source:ro" \ + -v "$migration_backup:/backup" \ + alpine:3.22 tar -cf /backup/db-config.tar -C /source . +``` + +Require every backup command to succeed before proceeding. The storage-image +archive and override use immutable IDs from the existing containers, so rollback +does not depend on mutable `latest` tags or registry availability. The bucket +setup container may be stopped; keep it until its image has been saved. If it +was already removed, resolve its original client image ID from your deployment +inventory and save it with the server image before continuing. The Compose +service names remain `minio` and `minio-createbucket` with the pgsty images. + +Privately set `SUPABASE_SERVICE_ROLE_KEY` to the **old** `.env`'s `SERVICE_ROLE_KEY` +(for example, use `read -rs SUPABASE_SERVICE_ROLE_KEY` in Bash, then `export +SUPABASE_SERVICE_ROLE_KEY`; paste the key when prompted). Do not put the key in +shell history or in the application's `VITE_*` variables. + +```sh +python3 migrate-storage.py export "$migration_backup/objects" \ + --url http://localhost:8000 --database "$postgres_database" && +python3 migrate-storage.py check "$migration_backup/objects" \ + --bucket stub --tenant stub +``` + +Use your preserved `GLOBAL_S3_BUCKET` and `TENANT_ID` instead of `stub` for +`check` and installation. The check validates every export checksum and rejects +S3 keys that collide as file/directory paths, before any database upgrade. Such +a layout requires retaining an S3 backend with a separately rehearsed plan. +Export also refuses to publish a manifest if object metadata changes while it +runs; keep all writers paused and repeat into a new export directory. + +Use your actual gateway URL and preserved `POSTGRES_DB` for both export and +verification. Keep `postgres_database` in the same shell session. Export includes +all live objects in every Supabase bucket, not only `revisit`, and fails on +missing objects, truncated HTTP responses, or size mismatches. +The export directory must not already exist. An interrupted export must be +repeated into a new directory; only a completed export contains `manifest.json`. +It does not copy orphaned MinIO objects absent from the database. + +Snapshot the bundled application records, bucket settings, and access policies +while writes remain paused. Keep this private alongside the other backups: + +```sh +cat > "$migration_backup/check-application.sql" <<'SQL' +SELECT json_build_object( + 'records', (SELECT json_agg(r ORDER BY "studyId", "docId") FROM public.revisit r), + 'buckets', (SELECT json_agg(b ORDER BY id) FROM ( + SELECT id, name, owner, owner_id, public, created_at, updated_at, + file_size_limit, allowed_mime_types + FROM storage.buckets + ) b), + 'policies', (SELECT json_agg(p ORDER BY schemaname, tablename, policyname) + FROM pg_policies p WHERE schemaname IN ('public', 'storage')) +); +SQL +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-before.json" +``` + +The final verification repeats this query and compares its output. Add separate +checks for any custom application tables, schemas, extensions, or encrypted data +identified in your inventory; the bundled snapshot does not certify those. + +Stop the old stack **using its old Compose file**, without deleting volumes: + +```sh +docker compose down +sudo cp -a volumes/db/data "$migration_backup/postgres15-data" +sudo cp -a volumes/storage "$migration_backup/minio-data" +``` + +Save the other deployment-specific configuration files as needed. Verify backup +completion before continuing. The physical database and encryption-volume +backups must belong to the same quiesced deployment. + +## 2. Install the new configuration; preserve existing secrets + +If this deployment is inside the old reVISit checkout, move the stopped directory +outside it after verifying the backups. Choose a destination that does not exist: + +```sh +deployment=/absolute/path/to/supabase-project +test ! -e "$deployment" +sudo mv "$PWD" "$deployment" +cd "$deployment" +``` + +Skip the move if the deployment is already separate. Keep the same shell session +so `migration_backup` continues to refer to the original absolute backup path. +Application checkout updates must not replace deployment configuration or data. + +Fetch the pinned official bundle into a separate source directory. From the +old deployment directory, replace the support path with your new reVISit checkout: + +```sh +revisit_support=/absolute/path/to/new-checkout/supabase +git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.3 \ + https://github.com/supabase/supabase.git ../supabase-upstream && +git -C ../supabase-upstream sparse-checkout set docker && +test "$(git -C ../supabase-upstream rev-parse HEAD)" = \ + bc27d2661edd09a5d0b8d76ce33da6d4071a756e && +rsync -av --exclude='.env' --exclude='.env.old' --exclude='.supabase-version' \ + --exclude='volumes/db/data*' --exclude='volumes/storage*' \ + ../supabase-upstream/docker/ ./ && +cp "$revisit_support/docker-compose.auth.yml" . && +cp "$revisit_support/docker-compose.github.yml" . +``` + +Do not add `--delete`. The command replaces files supplied by upstream; review +any local edits to those files before copying and carry them over deliberately. +Preserve the existing Compose project name and `db-config` named volume. +The upstream scripts now live in this deployment; +reVISit does not maintain copies of them. + +Merge new `.env.example` settings into your existing private `.env`. Keep the +existing `POSTGRES_DB`, database password, `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY`, and +encryption keys. **Do not run `generate-keys.sh` during migration:** rotating +these values can invalidate sessions or make encrypted data unreadable. + +- Set `COMPOSE_FILE=docker-compose.yml` and `ENABLE_ANONYMOUS_USERS=true`. +- Carry the previous `TENANT_ID` into `STORAGE_TENANT_ID`; keep `GLOBAL_S3_BUCKET` + and `REGION` consistent with the storage installation command below. +- Set the new gateway port `API_GW_HTTP_PORT` to your previous HTTP port. +- Supply newly required secrets such as `SECRET_KEY_BASE`, `REALTIME_DB_ENC_KEY`, + `VAULT_ENC_KEY`, and `PG_META_CRYPTO_KEY` if absent. Also generate fresh + `S3_PROTOCOL_ACCESS_KEY_ID` and `S3_PROTOCOL_ACCESS_KEY_SECRET` if missing; + these secure Storage’s S3 API even when its backend is local files. Generate + only missing values using the commands/lengths in `.env.example`; never use + the template’s default secrets or credentials. +- Preserve `SITE_URL`, application redirects, SMTP, signup settings, and OAuth + provider credentials. Convert `API_EXTERNAL_URL` to the full Auth endpoint: + `https://your-domain` becomes `https://your-domain/auth/v1` (do not append the + suffix twice). Keep the final `GITHUB_OAUTH_REDIRECT_URI` registered with + GitHub unchanged at `https://your-domain/auth/v1/callback`. Any custom OAuth + override derived from `${API_EXTERNAL_URL}/auth/v1/callback` must now use + `${API_EXTERNAL_URL}/callback`. SAML deployments must update their IdP's ACS + and metadata endpoints to `/auth/v1/sso/saml/acs` and + `/auth/v1/sso/saml/metadata`. If configured, convert `SAML_EXTERNAL_URL` + (mapped to `GOTRUE_SAML_EXTERNAL_URL`) to the full `/auth/v1` base too; it + overrides `API_EXTERNAL_URL`. Re-fetch provider metadata afterwards. See + [upstream Auth URL migration guidance](https://supabase.com/changelog/47093-self-hosted-supabase-api-external-url-to-include-auth-v1). + The old GitHub settings are preserved by `sh run.sh config add github`. This optional + override reads the existing `ENABLE_GITHUB_OAUTH`, `GITHUB_OAUTH_CLIENT_ID`, + `GITHUB_OAUTH_SECRET`, and `GITHUB_OAUTH_REDIRECT_URI` variables. Keep the same + callback URL and registered GitHub OAuth application. +- Keep legacy signing for this migration. Introducing asymmetric signing/key + rotation is optional later; legacy keys are supported by the pinned stack. + +Before modifying database files, pull the target images and check the two Node +services start on this host: + +```sh +# Auth is required even if GitHub OAuth is disabled. +sh run.sh config add auth +# The existing ENABLE_GITHUB_OAUTH value controls whether GitHub sign-in is enabled. +sh run.sh config add github +sh run.sh pull +docker compose run --rm --no-deps studio node --version +docker compose run --rm --no-deps meta node --version +``` + +If either crashes, resolve host/image compatibility before upgrading. + +Move the old MinIO data aside and create an empty file-storage directory: + +```sh +mv volumes/storage volumes/storage.minio-backup +mkdir volumes/storage +sudo python3 - <<'PYTHON' +import os +from pathlib import Path +import tempfile +with tempfile.TemporaryDirectory(dir="volumes/storage") as temporary: + probe = Path(temporary) / "xattr-probe" + probe.write_bytes(b"storage preflight") + os.setxattr(probe, "user.supabase.content-type", b"application/octet-stream") + assert os.getxattr(probe, "user.supabase.content-type") == b"application/octet-stream" +PYTHON +``` + +Require the filesystem probe to succeed before upgrading the database. An +unsupported or unwritable filesystem must be fixed while the original database +and object-store backups are still untouched. + +Do **not** apply `revisit.sql`: existing application schema, buckets, and policies +are retained by the database upgrade. + +## 3. Upgrade Postgres using the upstream script + +The upgrade script expects a running Postgres 15 container. Start **only the old +database** with the saved old configuration, pointing Compose at this deployment +directory so its relative paths still refer to the original data: + +```sh +docker compose --project-directory "$PWD" \ + -f "$migration_backup/docker-compose.yml" up -d --wait db +docker exec supabase-db pg_isready -U postgres +# Our previous Compose did not initialize the upstream Realtime schema. +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -v ON_ERROR_STOP=1 < volumes/db/realtime.sql +python3 "$revisit_support/prepare-pg17-upgrade.py" \ + utils/upgrade-pg17.sh "$migration_backup/upgrade-pg17.revisit.sh" && +mkdir -p "$migration_backup/upgrade-tmp" && +sudo env TMPDIR="$migration_backup/upgrade-tmp" \ + bash "$migration_backup/upgrade-pg17.revisit.sh" && +sudo rmdir "$migration_backup/upgrade-tmp" && +sh run.sh recreate auth +``` + +The preparation helper leaves the downloaded upstream script unchanged and +writes a deployment copy that selects the `db-config` volume actually mounted +by `supabase-db`, starts services using the saved Auth/GitHub overrides, and +aborts on a failed required SQL migration. It rejects an unexpected upstream +script before writing the prepared copy. Do not run the unprepared script: +it guesses the encryption volume, ignores saved overrides during startup, and +can report success after required migration errors. The final Auth recreation +and image check confirm the selected configuration. Keep traffic blocked +throughout; use `sh run.sh start` for normal operation rather than the original +script's printed commands. + +Read all output, including migration warnings. This script uses upstream's +`pg_upgrade` procedure, preserves `data.bak.pg15` and the encryption key, applies +required role/extension changes, and starts the new stack. See +[upstream database upgrade guidance](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17). +The prepared script selects the running database's actual named encryption +volume even when unrelated or leftover `db-config` volumes exist. A missing +volume mount is an error; do not substitute another volume. + +Require a successful script exit and healthy services. +**Upgrade complete is not sufficient:** the prepared copy stops on required +migration-file failures, but upstream extension reconciliation can still warn +and continue. Keep writes blocked and +**stop here** on missing migration files, SQL errors, or migration/extension +reconciliation failures. Resolve each failure and reapply only its affected SQL +from `/docker-entrypoint-initdb.d/migrations/` in the target database container +with `psql -v ON_ERROR_STOP=1`; inspect the pinned `utils/upgrade-pg17.sh` for the +required files and effects. Do not rerun the whole database upgrade: it refuses +an already upgraded database. Resolve startup failures or restore the preserved +backups to rehearse again. + +The upstream script applies its database-local SQL to `postgres`. If your +`POSTGRES_DB` differs, separately rehearse and apply the required database-local +migrations there as well; the helper's database option does not adapt the +upstream upgrade script. Before proceeding, require the following role/grant +check to succeed in the configured application database, and confirm every +required migration completed without errors: + +```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -v ON_ERROR_STOP=1 <<'SQL' +DO $$ +BEGIN + IF NOT ( + pg_has_role('postgres', 'supabase_privileged_role', 'MEMBER') + AND pg_has_role('supabase_etl_admin', 'supabase_privileged_role', 'MEMBER') + AND pg_has_role('supabase_etl_admin', 'pg_read_all_data', 'MEMBER') + AND has_database_privilege('supabase_etl_admin', current_database(), 'CREATE') + AND pg_has_role('supabase_etl_admin', 'pg_monitor', 'MEMBER') + AND pg_has_role('supabase_read_only_user', 'pg_monitor', 'MEMBER') + AND EXISTS ( + SELECT 1 FROM pg_auth_members m + JOIN pg_roles granted ON granted.oid = m.roleid + JOIN pg_roles member_role ON member_role.oid = m.member + WHERE granted.rolname = 'pg_create_subscription' + AND member_role.rolname = 'postgres' AND m.admin_option + ) + AND has_function_privilege('postgres', 'pg_catalog.pg_reload_conf()', + 'EXECUTE WITH GRANT OPTION') + ) THEN + RAISE EXCEPTION 'Required PG17 roles/grants missing; keep writes blocked'; + END IF; +END +$$; +SQL +``` + +A missing role or grant is a migration failure; resolve it before continuing. +These checks complement review of all migration output, including database-local +function/search-path, role settings, GraphQL trigger, and extension changes. + +The target image changes the libc collation version. While traffic is still +blocked, rebuild indexes in each retained database. The block includes the +configured application database once; add any other retained databases to the +`set --` list: + +```sh +( + set -- postgres template1 _supabase + case "$postgres_database" in + postgres|template1|_supabase) ;; + *) set -- "$@" "$postgres_database" ;; + esac + for database do + docker exec -i supabase-db psql -U supabase_admin -d "$database" \ + -v ON_ERROR_STOP=1 -v database="$database" <<'SQL' || exit 1 +REINDEX DATABASE :"database"; +SQL + done +) +``` + +Require this block to exit successfully for **every** database. On any failure, +stop, resolve the error, and rerun it before installing files or reopening writes. +The subshell stops at the first failed command without closing your login shell. + +The upstream script refreshes the version marker; that alone does not rebuild +indexes. See [PostgreSQL collation guidance](https://www.postgresql.org/docs/17/sql-altercollation.html). +Allow time and temporary disk space for this step. + +Keep application access blocked. Stop Storage and install exported files: + +```sh +docker compose stop storage +sudo python3 migrate-storage.py install "$migration_backup/objects" \ + --destination volumes/storage --bucket stub --tenant stub +docker compose up -d storage +sh run.sh start +``` + +Replace `stub` with your preserved backend bucket and tenant values. The helper +checks all export hashes before writing, rejects unsafe paths/conflicting files, +and preserves Content-Type and Cache-Control using Linux extended attributes. +Installed directories use mode 755 and files 644 so the separate imgproxy user +can read them; restrict server access to this storage directory. Export backups +retain their private permissions. +Each payload and its HTTP metadata are staged on the destination filesystem +before an atomic rename, so a failed copy leaves the final object untouched. +It can be rerun with the same verified export after fixing an installation error. +A forcibly terminated process can leave `.revisit-migration-*` staging +directories beside its target objects. Once no migration process is running, +remove only those incomplete staging directories if they consume needed disk +space; preserve the exported backup and final object paths. Rerun installation +and verification before reopening writes. + +## 4. Verify before reopening writes + +```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-after.json" && +diff "$migration_backup/application-before.json" "$migration_backup/application-after.json" && +python3 migrate-storage.py verify "$migration_backup/objects" \ + --url http://localhost:8000 --database "$postgres_database" && +sh run.sh status && +test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ + supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 && +docker exec supabase-db psql -U supabase_admin -d postgres \ + -c 'SHOW server_version;' && +printf 'ref=self-hosted/v0.8.3\n' > .supabase-version +``` + +Require the entire verification block to succeed. Any unexplained record, +bucket, or policy difference must be resolved before reopening writes. Run +these comparisons before the smoke tests create new participant data. If the Auth image check +fails, keep writes blocked, enable the required Auth override, run +`sh run.sh recreate auth`, and repeat verification. + +Verification streams every migrated object through the new API and checks its +SHA-256, Content-Type, and Cache-Control, plus the original database object +metadata. Keep the legacy application URL/key unchanged when possible. Check Study Designer login, +existing participants/answers/configurations, recordings, exports, access +policies, and a new participant submission. Restart the stack and confirm both +old and new data persist. Retain database/file/encryption backups and old images +until the deployment has been accepted. After verifying your external database/file/encryption backups and accepting +the migration, move `volumes/db/data.bak.pg15` and +`volumes/storage.minio-backup` outside the deployment (retain them for rollback). +Do this **before any upstream update**: its configuration archive excludes only +the exact live `data` and `storage` directories, includes these retained copies, +and can continue after archive errors. Keep their new locations in your rollback +inventory. Also move or remove the disposable `volumes/db/pg17_upgrade_bin_*.tar.gz` +cache after acceptance; it is also included in that archive and can exceed 1 GB. +Retain the encryption/key backups. Then reopen application writes. + +## Rollback + +Keep writes paused. Run the following in **Bash from the migrated deployment +directory**, using the same absolute `migration_backup` path as above. If using +a new shell, set that variable again first. The commands preserve the failed +deployment's data/configuration for diagnosis, then restore the coordinated +Postgres 15, MinIO/pgsty, and `db-config` backups. Restoring only database files +is insufficient: the upgrade also changes ownership/configuration in the +encryption volume. + +The saved Compose project name preserves the original named-volume mapping, +even if the deployment directory was renamed. The volume name comes from the +original database container, rather than guessing a name. Restore any additional +custom configuration/overrides from your inventory before restarting. + +```sh +( +set -eu +: "${migration_backup:?Set migration_backup to the original absolute backup path}" +db_config_volume=$(cat "$migration_backup/db-config-volume") +compose_project=$(cat "$migration_backup/compose-project") +test -n "$db_config_volume" && test -n "$compose_project" +test -d "$migration_backup/postgres15-data" +test -d "$migration_backup/minio-data" +test -f "$migration_backup/.env" +test -f "$migration_backup/docker-compose.yml" +test -d "$migration_backup/api" +tar -tf "$migration_backup/db-config.tar" > /dev/null +docker volume inspect "$db_config_volume" > /dev/null +docker image load -i "$migration_backup/storage-images.tar" + +# Stop the upgraded stack with its current configuration. Never add -v. +docker compose --project-name "$compose_project" down +volume_users=$(docker ps -q --filter "volume=$db_config_volume") +if [ -n "$volume_users" ]; then + echo 'db-config is still in use; stop leftover upgrade containers before restoring.' >&2 + exit 1 +fi +umask 077 +failed_upgrade="$migration_backup/failed-upgrade-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir "$failed_upgrade" +cp -a .env docker-compose.yml "$failed_upgrade/" +docker run --rm -v "$db_config_volume:/source:ro" \ + -v "$failed_upgrade:/backup" \ + alpine:3.22 tar -cf /backup/db-config.tar -C /source . +for directory in volumes/db/data volumes/storage volumes/api; do + if [ -e "$directory" ]; then + sudo mv "$directory" "$failed_upgrade/$(basename "$directory")" + fi +done + +sudo cp -a "$migration_backup/postgres15-data" volumes/db/data +sudo cp -a "$migration_backup/minio-data" volumes/storage +sudo cp -a "$migration_backup/api" volumes/api +cp "$migration_backup/.env" .env +cp "$migration_backup/docker-compose.yml" docker-compose.yml + +# Remove PG17-only files, including hidden files, before restoring PG15's +# numeric ownership, permissions, encryption keys, and configuration. +docker run --rm -v "$db_config_volume:/restore" \ + -v "$migration_backup:/backup:ro" alpine:3.22 sh -ec ' + rm -rf /restore/* /restore/.[!.]* /restore/..?* + tar -xpf /backup/db-config.tar -C /restore + ' + +docker compose --project-name "$compose_project" -f docker-compose.yml \ + -f "$migration_backup/docker-compose.storage-images.yml" up -d --wait db +docker exec supabase-db psql -U supabase_admin -d postgres \ + -v ON_ERROR_STOP=1 -c 'SHOW server_version;' +docker compose --project-name "$compose_project" -f docker-compose.yml \ + -f "$migration_backup/docker-compose.storage-images.yml" up -d +) +``` + +The subshell stops on any failed command; do not restart services after a failed +restore. It restores the same pgsty or MinIO image IDs that were running before +migration. Once the Auth and Storage APIs are ready, verify the restored objects +with the old service-role key and gateway URL, using the original `POSTGRES_DB`: + +```sh +docker exec -i supabase-db psql -U supabase_admin -d "$postgres_database" \ + -At -v ON_ERROR_STOP=1 < "$migration_backup/check-application.sql" \ + > "$migration_backup/application-restored.json" && +diff "$migration_backup/application-before.json" "$migration_backup/application-restored.json" && +python3 migrate-storage.py verify "$migration_backup/objects" \ + --url http://localhost:8000 --database "$postgres_database" +``` + +**Rollback also restores the old vulnerable Auth image.** Keep application +traffic blocked and GitHub OAuth/secure email unused until Auth is protected. +For the original bundled `v2.171.0` image, apply the certificate mitigation: + +```sh +compose_project=$(cat "$migration_backup/compose-project") +docker compose --project-name "$compose_project" -f docker-compose.yml \ + exec -T --user root auth apk add --no-cache --upgrade \ + 'ca-certificates>=20261020-r0' 'ca-certificates-bundle>=20261020-r0' && +docker compose --project-name "$compose_project" -f docker-compose.yml restart auth +``` + +Require a successful package update and healthy Auth before testing sign-in. +This mitigation is discarded if the old Auth container is recreated; complete the fixed-image migration before +normal operation, or reapply the mitigation after every recreation. If your +Auth image or certificate mounts differ, verify a deployment-specific fix first. + +Check existing participants/answers and Study Designer login before reopening +writes. Keep the failed-upgrade directory and original backups until rollback +has been accepted. These are local copies, not automatic off-host backups. + +After new writes are accepted, restoring the old backups loses those writes. +Pause and plan how to retain them rather than treating rollback as lossless. diff --git a/supabase/README.md b/supabase/README.md new file mode 100644 index 0000000000..7cb8226f65 --- /dev/null +++ b/supabase/README.md @@ -0,0 +1,206 @@ +# Self-hosted Supabase for reVISit + +This directory contains the reVISit-specific setup and migration files. Obtain +the deployment bundle directly from **Supabase self-hosted/v0.8.3** (see +[UPSTREAM](UPSTREAM)), following the upstream manual installation below. Its +default Storage backend uses persistent local files; MinIO is not required. +For an existing deployment, follow [MIGRATION.md](MIGRATION.md) **before starting +this configuration**. Hosted Supabase deployments do not need this migration. + +The required Auth override pins upstream `v2.198.0-rc.25` by digest. This release +candidate includes Supabase's [Go 1.27.3 TLS fix](https://github.com/supabase/auth/pull/2883) +for the [eMudhra intermediate CA compromise](https://bugzilla.mozilla.org/show_bug.cgi?id=2083449). +The pinned bundle's default Auth `v2.197.0` predates that fix. Keep this override +enabled, including when recreating containers or updating the upstream bundle; +review a replacement stable Auth release before removing it. GitHub OAuth +configuration remains optional; the TLS fix is required for every deployment. + +## First-time setup + +Requirements: a Linux host with Git, Docker Engine, Docker Compose v2 or newer, OpenSSL, +and Node.js 16+ (the upstream key helper can use Docker instead). For server +resources and operating-system details, follow the +[upstream installation guide](https://supabase.com/docs/guides/self-hosting/docker). +The storage filesystem must support Linux extended attributes. Mac shared +folders in Docker Desktop may reject these attributes; use a Docker-managed +Linux volume for a local rehearsal and back it up separately. Verify image +startup on your actual host before a migration. + +Keep the deployment outside the reVISit checkout so application updates do not +replace its configuration or data. The upstream stack uses fixed container +names: run only one copy per Docker host, or deliberately isolate it. + +1. From the parent directory where you want your new deployment, obtain the + pinned upstream bundle. Replace `revisit_support` with the absolute path to + this directory in your reVISit checkout: + + ```sh + revisit_support=/absolute/path/to/revisit-study/supabase + git clone --filter=blob:none --sparse --depth 1 --branch self-hosted/v0.8.3 \ + https://github.com/supabase/supabase.git supabase-upstream && + git -C supabase-upstream sparse-checkout set docker && + test "$(git -C supabase-upstream rev-parse HEAD)" = \ + bc27d2661edd09a5d0b8d76ce33da6d4071a756e && + mkdir supabase-project && + cp -a supabase-upstream/docker/. supabase-project/ && + cd supabase-project + ``` + + These are upstream's manual installation steps with a sparse checkout and + commit check. Subsequent deployment commands run from `supabase-project`. + Create your private configuration and generate fresh secrets: + + ```sh + cp .env.example .env + sh utils/generate-keys.sh --update-env + sh utils/add-new-auth-keys.sh --update-env + printf 'ref=self-hosted/v0.8.3\n' > .supabase-version + cp "$revisit_support/docker-compose.auth.yml" . + sh run.sh config add auth + ``` + + These helpers print secrets; keep their output private. Never commit `.env`. + The second helper enables asymmetric signing in `docker-compose.yml` while + retaining verification of legacy JWTs. + +2. Edit `.env`: + - Set `ENABLE_ANONYMOUS_USERS=true` for participant sessions. + - Set `SUPABASE_PUBLIC_URL` to the Supabase base URL clients can reach + (`http://localhost:8000` locally). Set `API_EXTERNAL_URL` to its Auth + endpoint (`http://localhost:8000/auth/v1` locally), following the template. + - Set `SITE_URL` and `ADDITIONAL_REDIRECT_URLS` for your reVISit application. + - Review email confirmation and SMTP settings for Study Designer accounts. + For GitHub OAuth, append `ENABLE_GITHUB_OAUTH=true`, + `GITHUB_OAUTH_CLIENT_ID`, `GITHUB_OAUTH_SECRET`, and + `GITHUB_OAUTH_REDIRECT_URI` to `.env`, then copy and enable our override: + + ```sh + cp "$revisit_support/docker-compose.github.yml" . + sh run.sh config add github + ``` + + The redirect URI is + `/auth/v1/callback`; register it in the GitHub OAuth app. + See [upstream OAuth guidance](https://supabase.com/docs/guides/self-hosting/auth). + - Keep `STORAGE_TENANT_ID=stub`, `GLOBAL_S3_BUCKET=stub`, and `REGION=stub` + unless you have an existing deployment with different values. + +3. Pull and start the stack: + + ```sh + sh run.sh pull && + sh run.sh start && + sh run.sh status && + test "$(docker inspect supabase-auth --format '{{.Config.Image}}')" = \ + supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 + ``` + + Require this block to succeed, including the exact Auth image check. + If it fails, keep the deployment closed to users until corrected. + + Open Studio at `http://localhost:8000` and use your generated + `DASHBOARD_USERNAME` and `DASHBOARD_PASSWORD`. On a server, expose the API + through HTTPS and restrict administration/database ports. The upstream + pooler publishes ports 5432 and 6543; do not allow public access to them. + Follow [upstream HTTPS guidance](https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https). + +4. Initialize the reVISit table and private bucket **on a fresh database only**: + + ```sh + docker exec -i supabase-db psql -U supabase_admin \ + -v ON_ERROR_STOP=1 < "$revisit_support/revisit.sql" + ``` + + `psql` uses the container's `PGDATABASE`, which upstream sets from + `POSTGRES_DB`; this initializes the configured application database. + This applies the same schema and access model as the existing + [reVISit setup instructions](https://revisit.dev/docs/data-and-deployment/supabase/setup/). + Those policies allow anyone holding the public application key to read and + write reVISit records and objects. A private bucket prevents unauthenticated + public URLs; it does **not** provide participant-level isolation under these + policies. Keep your existing stricter policies when migrating. This upgrade + does not redesign authorization. + +5. Configure the `.env` at the root of your **reVISit application**: + + ```dotenv + VITE_STORAGE_ENGINE="supabase" + VITE_SUPABASE_URL="http://localhost:8000" + VITE_SUPABASE_ANON_KEY="" + ``` + + The application variable retains its existing name and accepts the new + publishable key. Never put `SERVICE_ROLE_KEY` or `SUPABASE_SECRET_KEY` in + application configuration. Restart `yarn serve` or rebuild/redeploy the app + after changing these build-time variables. + +6. Complete a short study, confirm its participant data in Studio, and restart + Supabase. Confirm the original data still opens and another participant can + submit answers. Exercise your configured Study Designer sign-in as well. + +## Storage and ongoing operations + +Files persist under `volumes/storage`; Postgres persists under `volumes/db/data`. +The `db-config` named volume contains encryption material and must also be backed +up. **Never use `docker compose down -v` on a deployment you intend to keep.** +Back up files and database together with writes paused, store backups outside +the server, test restores, and monitor disk capacity. File storage is suitable +for a single-server deployment; it does not replicate data or provide failover. +For deployments requiring object storage, use +[upstream S3 configuration](https://supabase.com/docs/guides/self-hosting/self-hosted-s3). + +Use `sh run.sh stop`, `start`, `status`, and `logs` to operate the stack. +Before any update, verify external backups and move retained migration copies +`volumes/db/data.bak.pg15` and `volumes/storage.minio-backup` outside the deployment; +keep them for rollback. Also move or remove the disposable +`volumes/db/pg17_upgrade_bin_*.tar.gz` cache after accepting migration, while +retaining encryption/key backups. Upstream's configuration backup excludes only the exact +live-data paths, so these copies would be archived too, potentially filling the +disk. Archive errors only warn: check the resulting configuration archive before +relying on it, and stop if it is incomplete. + +Review future tagged updates using `sh update.sh --dry-run --to ` +and [upstream update guidance](https://supabase.com/docs/guides/self-hosting/updating). +The update tool belongs to the downloaded upstream bundle. Review its plan and +the required Auth override and your GitHub settings before applying an update; +this guide and migration helper target the pinned release above. This release includes Storage and Realtime +security fixes. Its Storage database migrations are one-way: reverting an image +alone is unsafe; restore coordinated pre-upgrade backups for rollback. Review +[upstream release notes](https://github.com/supabase/supabase/blob/self-hosted/v0.8.3/docker/CHANGELOG.md) +before upgrading. If you use Realtime Postgres Changes, the release notes also +document a connection-restart issue and the `REGION: us-east-1` workaround; +reVISit's storage engine does not subscribe to Postgres Changes. + +## Local regression checks + +Run these checks from your reVISit checkout. The migration helper requires +Python 3.11+ and no third-party Python packages: + +```sh +python3 supabase/tests/migrate-storage.spec.py +python3 supabase/tests/prepare-pg17-upgrade.spec.py +``` + +The opt-in Chromium test is `tests/supabase-live.spec.ts`. It needs an isolated +running stack, initialized `supabase/revisit.sql`, and the application's Supabase +environment variables. Set `PW_SUPABASE_LIVE=1` and +`PW_SUPABASE_SERVICE_ROLE_KEY` privately, then run the focused suite from the +repository root with `yarn exec playwright test tests/supabase-live.spec.ts --project chromium`. +Use `PW_SUPABASE_PARTICIPANT_ID` to retain a known UUID before migration and +`PW_SUPABASE_EXISTING_PARTICIPANT` to verify it afterwards. The test creates real +participant records; do not point it at production. + +For an unfinished-participant rehearsal, set `PW_SUPABASE_RESUME_PARTICIPANT` to +a fresh UUID and `PW_SUPABASE_SEED_RESUME=1` before migration. Afterwards retain +the UUID and remove `PW_SUPABASE_SEED_RESUME`; the test retrieves the earlier +answer in a fresh browser and completes the study. This consumes the unfinished +fixture, so omit that UUID on subsequent recreation runs. + +The Designer export check additionally needs private +`PW_SUPABASE_DESIGNER_EMAIL` / `PW_SUPABASE_DESIGNER_PASSWORD` values for a +confirmed test account authorized in the fixture's user-management settings, +and `PW_SUPABASE_EXISTING_PARTICIPANT` for the original completed participant. +It uses real password Auth locally to check the retained account and protected +JSON/CSV export UI. The application's GitHub sign-in button still requires a +separate real-provider smoke test after deployment; this local test does not +claim external OAuth or SMTP delivery coverage. diff --git a/supabase/UPSTREAM b/supabase/UPSTREAM new file mode 100644 index 0000000000..87844a7268 --- /dev/null +++ b/supabase/UPSTREAM @@ -0,0 +1,10 @@ +Supabase self-hosted/v0.8.3 +Commit: bc27d2661edd09a5d0b8d76ce33da6d4071a756e +Source: https://github.com/supabase/supabase/tree/self-hosted/v0.8.3/docker + +The deployment configuration and scripts are obtained directly from upstream. +This directory contains only reVISit setup, migration, and integration files. + +Required override: docker-compose.auth.yml pins supabase/gotrue:v2.198.0-rc.25 +Digest: sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 +Reason: https://github.com/supabase/auth/pull/2883 (Go 1.27.3 eMudhra TLS fix) diff --git a/supabase/docker-compose.auth.yml b/supabase/docker-compose.auth.yml new file mode 100644 index 0000000000..7b7beab284 --- /dev/null +++ b/supabase/docker-compose.auth.yml @@ -0,0 +1,4 @@ +# Required until the pinned upstream bundle includes the eMudhra TLS fix. +services: + auth: + image: supabase/gotrue:v2.198.0-rc.25@sha256:b4c104aa7abbb2bc6c0c294072f736230979acfec87c9114fce63f4224e0f357 diff --git a/supabase/docker-compose.github.yml b/supabase/docker-compose.github.yml new file mode 100644 index 0000000000..003f59dd16 --- /dev/null +++ b/supabase/docker-compose.github.yml @@ -0,0 +1,8 @@ +# Preserve the GitHub OAuth variables used by the previous reVISit stack. +services: + auth: + environment: + GOTRUE_EXTERNAL_GITHUB_ENABLED: ${ENABLE_GITHUB_OAUTH:-false} + GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: ${GITHUB_OAUTH_CLIENT_ID:-} + GOTRUE_EXTERNAL_GITHUB_SECRET: ${GITHUB_OAUTH_SECRET:-} + GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: ${GITHUB_OAUTH_REDIRECT_URI:-} diff --git a/supabase/docker-compose.yml b/supabase/docker-compose.yml deleted file mode 100644 index 0d224b6414..0000000000 --- a/supabase/docker-compose.yml +++ /dev/null @@ -1,332 +0,0 @@ -name: supabase - -services: - - studio: - container_name: supabase-studio - image: supabase/studio:2025.04.21-sha-173cc56 - restart: unless-stopped - healthcheck: - test: - [ - "CMD", - "node", - "-e", - "fetch('http://studio:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})" - ] - timeout: 10s - interval: 5s - retries: 3 - depends_on: - analytics: - condition: service_healthy - environment: - STUDIO_PG_META_URL: http://meta:8080 - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - - DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} - DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} - - SUPABASE_URL: http://kong:8000 - SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} - AUTH_JWT_SECRET: ${JWT_SECRET} - - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} - LOGFLARE_URL: http://analytics:4000 - NEXT_PUBLIC_ENABLE_LOGS: true - NEXT_ANALYTICS_BACKEND_PROVIDER: postgres - - kong: - container_name: supabase-kong - image: kong:2.8.1 - restart: unless-stopped - ports: - - ${KONG_HTTP_PORT}:8000/tcp - - ${KONG_HTTPS_PORT}:8443/tcp - volumes: - - ./volumes/api/kong.yml:/home/kong/temp.yml:ro,z - depends_on: - analytics: - condition: service_healthy - environment: - KONG_DATABASE: "off" - KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml - KONG_DNS_ORDER: LAST,A,CNAME - KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth - KONG_NGINX_PROXY_PROXY_BUFFER_SIZE: 160k - KONG_NGINX_PROXY_PROXY_BUFFERS: 64 160k - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} - DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} - DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} - entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' - - auth: - container_name: supabase-auth - image: supabase/gotrue:v2.171.0 - restart: unless-stopped - healthcheck: - test: - [ - "CMD", - "wget", - "--no-verbose", - "--tries=1", - "--spider", - "http://localhost:9999/health" - ] - timeout: 5s - interval: 5s - retries: 3 - depends_on: - db: - condition: service_healthy - analytics: - condition: service_healthy - environment: - GOTRUE_API_HOST: 0.0.0.0 - GOTRUE_API_PORT: 9999 - API_EXTERNAL_URL: ${API_EXTERNAL_URL} - - GOTRUE_DB_DRIVER: postgres - GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - - GOTRUE_SITE_URL: ${SITE_URL} - GOTRUE_URI_ALLOW_LIST: ${ADDITIONAL_REDIRECT_URLS} - GOTRUE_DISABLE_SIGNUP: ${DISABLE_SIGNUP} - - GOTRUE_JWT_ADMIN_ROLES: service_role - GOTRUE_JWT_AUD: authenticated - GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated - GOTRUE_JWT_EXP: ${JWT_EXPIRY} - GOTRUE_JWT_SECRET: ${JWT_SECRET} - - GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} - GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} - GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} - - GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} - GOTRUE_SMTP_HOST: ${SMTP_HOST} - GOTRUE_SMTP_PORT: ${SMTP_PORT} - GOTRUE_SMTP_USER: ${SMTP_USER} - GOTRUE_SMTP_PASS: ${SMTP_PASS} - GOTRUE_SMTP_SENDER_NAME: ${SMTP_SENDER_NAME} - GOTRUE_MAILER_URLPATHS_INVITE: ${MAILER_URLPATHS_INVITE} - GOTRUE_MAILER_URLPATHS_CONFIRMATION: ${MAILER_URLPATHS_CONFIRMATION} - GOTRUE_MAILER_URLPATHS_RECOVERY: ${MAILER_URLPATHS_RECOVERY} - GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: ${MAILER_URLPATHS_EMAIL_CHANGE} - - GOTRUE_EXTERNAL_PHONE_ENABLED: ${ENABLE_PHONE_SIGNUP} - GOTRUE_SMS_AUTOCONFIRM: ${ENABLE_PHONE_AUTOCONFIRM} - - GOTRUE_EXTERNAL_GITHUB_ENABLED: ${ENABLE_GITHUB_OAUTH} - GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: ${GITHUB_OAUTH_CLIENT_ID} - GOTRUE_EXTERNAL_GITHUB_SECRET: ${GITHUB_OAUTH_SECRET} - GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: ${GITHUB_OAUTH_REDIRECT_URI} - - rest: - container_name: supabase-rest - image: postgrest/postgrest:v12.2.11 - restart: unless-stopped - depends_on: - db: - condition: service_healthy - analytics: - condition: service_healthy - environment: - PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} - PGRST_DB_ANON_ROLE: anon - PGRST_JWT_SECRET: ${JWT_SECRET} - PGRST_DB_USE_LEGACY_GUCS: "false" - PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} - PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} - command: - [ - "postgrest" - ] - - meta: - container_name: supabase-meta - image: supabase/postgres-meta:v0.88.9 - restart: unless-stopped - depends_on: - db: - condition: service_healthy - analytics: - condition: service_healthy - environment: - PG_META_PORT: 8080 - PG_META_DB_HOST: ${POSTGRES_HOST} - PG_META_DB_PORT: ${POSTGRES_PORT} - PG_META_DB_NAME: ${POSTGRES_DB} - PG_META_DB_USER: supabase_admin - PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} - - analytics: - container_name: supabase-analytics - image: supabase/logflare:1.22.1 - restart: unless-stopped - ports: - - 4000:4000 - healthcheck: - test: - [ - "CMD", - "curl", - "http://localhost:4000/health" - ] - timeout: 5s - interval: 5s - retries: 10 - depends_on: - db: - condition: service_healthy - environment: - LOGFLARE_NODE_HOST: 127.0.0.1 - DB_USERNAME: supabase_admin - DB_DATABASE: _supabase - DB_HOSTNAME: ${POSTGRES_HOST} - DB_PORT: ${POSTGRES_PORT} - DB_PASSWORD: ${POSTGRES_PASSWORD} - DB_SCHEMA: _analytics - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} - LOGFLARE_SINGLE_TENANT: true - LOGFLARE_SUPABASE_MODE: true - LOGFLARE_MIN_CLUSTER_SIZE: 1 - LOGFLARE_LOG_LEVEL: warn - - POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase - POSTGRES_BACKEND_SCHEMA: _analytics - LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true - - db: - container_name: supabase-db - image: supabase/postgres:15.8.1.060 - restart: unless-stopped - volumes: - # Must be superuser to create event trigger - - ./volumes/db/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:Z - # Must be superuser to alter reserved role - - ./volumes/db/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:Z - # Initialize the database settings with JWT_SECRET and JWT_EXP - - ./volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:Z - # PGDATA directory is persisted between restarts - - ./volumes/db/data:/var/lib/postgresql/data:Z - # Changes required for internal supabase data such as _analytics - - ./volumes/db/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:Z - # Changes required for Analytics support - - ./volumes/db/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:Z - # Use named volume to persist pgsodium decryption key between restarts - - db-config:/etc/postgresql-custom - healthcheck: - test: - [ - "CMD", - "pg_isready", - "-U", - "postgres", - "-h", - "localhost" - ] - interval: 5s - timeout: 5s - retries: 10 - environment: - POSTGRES_HOST: /var/run/postgresql - PGPORT: ${POSTGRES_PORT} - POSTGRES_PORT: ${POSTGRES_PORT} - PGPASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - PGDATABASE: ${POSTGRES_DB} - POSTGRES_DB: ${POSTGRES_DB} - JWT_SECRET: ${JWT_SECRET} - JWT_EXP: ${JWT_EXPIRY} - command: - [ - "postgres", - "-c", - "config_file=/etc/postgresql/postgresql.conf", - "-c", - "log_min_messages=fatal" # prevents Realtime polling queries from appearing in logs - ] - - minio: - image: minio/minio - restart: unless-stopped - ports: - - '9000:9000' - - '9001:9001' - environment: - MINIO_ROOT_USER: supa-storage - MINIO_ROOT_PASSWORD: secret1234 - command: server --console-address ":9001" /data - healthcheck: - test: [ "CMD", "curl", "-f", "http://minio:9000/minio/health/live" ] - interval: 2s - timeout: 10s - retries: 5 - volumes: - - ./volumes/storage:/data:z - - minio-createbucket: - image: minio/mc - depends_on: - minio: - condition: service_healthy - entrypoint: > - /bin/sh -c " - /usr/bin/mc alias set supa-minio http://minio:9000 supa-storage secret1234; - /usr/bin/mc mb --ignore-existing supa-minio/stub; - exit 0; - " - storage: - container_name: supabase-storage - image: supabase/storage-api:v1.11.13 - depends_on: - db: - condition: service_healthy - rest: - condition: service_started - minio: - condition: service_healthy - healthcheck: - test: - [ - "CMD", - "wget", - "--no-verbose", - "--tries=1", - "--spider", - "http://127.0.0.1:5000/status" - ] - timeout: 5s - interval: 5s - retries: 3 - restart: unless-stopped - environment: - ANON_KEY: ${ANON_KEY} - SERVICE_KEY: ${SERVICE_ROLE_KEY} - POSTGREST_URL: http://rest:3000 - PGRST_JWT_SECRET: ${JWT_SECRET} - DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - FILE_SIZE_LIMIT: 52428800 - STORAGE_BACKEND: s3 - GLOBAL_S3_BUCKET: stub - GLOBAL_S3_ENDPOINT: http://minio:9000 - GLOBAL_S3_PROTOCOL: http - GLOBAL_S3_FORCE_PATH_STYLE: true - AWS_ACCESS_KEY_ID: supa-storage - AWS_SECRET_ACCESS_KEY: secret1234 - AWS_DEFAULT_REGION: stub - FILE_STORAGE_BACKEND_PATH: /var/lib/storage - TENANT_ID: stub - REGION: stub - ENABLE_IMAGE_TRANSFORMATION: "true" - volumes: - - ./volumes/storage:/var/lib/storage:z - -volumes: - db-config: diff --git a/supabase/migrate-storage.py b/supabase/migrate-storage.py new file mode 100644 index 0000000000..4e48d282f9 --- /dev/null +++ b/supabase/migrate-storage.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Offline MinIO/pgsty -> Supabase v1.81.0 file-backend migration. No database writes. + +Export runs with the old stack online and application writes stopped. Install +runs on Linux with the new Storage service stopped. See MIGRATION.md. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import urllib.parse +import urllib.request + + +def objects(database): + sql = "SELECT coalesce(json_agg(o ORDER BY bucket_id, name), '[]'::json) FROM storage.objects o" + result = subprocess.run( + ["docker", "exec", "supabase-db", "psql", "-U", "supabase_admin", + "-d", database, "-At", "-v", "ON_ERROR_STOP=1", "-c", sql], + check=True, capture_output=True, text=True, + ) + return json.loads(result.stdout) + + +def download(url, obj): + key = urllib.parse.quote(f"{obj['bucket_id']}/{obj['name']}", safe="/") + token = os.environ["SUPABASE_SERVICE_ROLE_KEY"] + request = urllib.request.Request( + f"{url.rstrip('/')}/storage/v1/object/authenticated/{key}", + headers={"apikey": token, "Authorization": f"Bearer {token}"}, + ) + return urllib.request.urlopen(request, timeout=120) + + +def digest(file): + with file.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def export(url, directory, database="postgres"): + directory.mkdir(parents=True, exist_ok=False) + original = objects(database) + manifest = [dict(obj) for obj in original] + for index, obj in enumerate(manifest): + obj["file"] = f"{index:08d}.bin" + target = directory / obj["file"] + with download(url, obj) as response, target.open("wb") as output: + content_length = response.headers.get("Content-Length") + obj["content_type"] = response.headers.get("Content-Type", "application/octet-stream") + obj["cache_control"] = response.headers.get("Cache-Control", "no-cache") + shutil.copyfileobj(response, output) + if content_length is not None and target.stat().st_size != int(content_length): + raise ValueError("Downloaded object size differs from HTTP Content-Length") + obj["sha256"] = digest(target) + size = (obj.get("metadata") or {}).get("size") + if size is not None and target.stat().st_size != int(size): + raise ValueError("Downloaded object size differs from database metadata") + if objects(database) != original: + raise ValueError("Storage metadata changed during export; keep writes paused and export again") + (directory / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(f"Exported {len(manifest)} objects") + + +def destination(root, bucket, tenant, obj): + # These match v1.81.0's default location and withOptionalVersion helpers. + parts = [bucket, tenant, obj["bucket_id"], *obj["name"].split("/")] + if obj.get("version"): + parts.append(obj["version"]) + if any(not part or part in (".", "..") or "/" in part or "\\" in part for part in parts): + raise ValueError("Unsafe storage object path") + target = root.joinpath(*parts) + if not target.resolve().is_relative_to(root.resolve()): + raise ValueError("Storage path escapes destination") + return target + + +def validate_export(directory, root, bucket, tenant): + manifest = json.loads((directory / "manifest.json").read_text()) + targets = set() + # Validate the entire export before changing the destination. + for index, obj in enumerate(manifest): + if obj["file"] != f"{index:08d}.bin" or digest(directory / obj["file"]) != obj["sha256"]: + raise ValueError("Export checksum mismatch") + target = destination(root, bucket, tenant, obj) + if target in targets: + raise ValueError("Storage objects map to the same file path") + targets.add(target) + if target.exists() and digest(target) != obj["sha256"]: + raise ValueError("Destination already contains different data") + if any(parent in targets for target in targets for parent in target.parents): + raise ValueError("Storage object paths conflict as files and directories; retain S3 storage") + return manifest + + +def check(directory, bucket, tenant): + with tempfile.TemporaryDirectory() as temporary: + manifest = validate_export(directory, Path(temporary) / "storage", bucket, tenant) + print(f"Checked {len(manifest)} objects for file-backend compatibility") + + +def install(directory, root, bucket, tenant): + if not hasattr(os, "setxattr"): + raise RuntimeError("Install requires Linux extended attributes; see MIGRATION.md") + manifest = validate_export(directory, root, bucket, tenant) + root.mkdir(parents=True, exist_ok=True) + root.chmod(0o755) + for obj in manifest: + target = destination(root, bucket, tenant, obj) + target.parent.mkdir(parents=True, exist_ok=True) + # imgproxy runs as a different user; private backup umasks must not + # prevent it from traversing the installed tree or reading payloads. + for directory_path in [target.parent, *target.parent.parents]: + if directory_path.is_relative_to(root): + directory_path.chmod(0o755) + # Publish only complete bytes and metadata; failed copies remain retryable. + with tempfile.TemporaryDirectory(dir=target.parent, prefix=".revisit-migration-") as staging: + payload = Path(staging) / "payload" + shutil.copyfile(directory / obj["file"], payload) + payload.chmod(0o644) + # Storage reads HTTP metadata from Linux extended attributes. + os.setxattr(payload, "user.supabase.content-type", obj["content_type"].encode()) + os.setxattr(payload, "user.supabase.cache-control", obj["cache_control"].encode()) + os.replace(payload, target) + print(f"Installed {len(manifest)} objects; database metadata unchanged") + + +def verify(url, directory, database="postgres"): + manifest = json.loads((directory / "manifest.json").read_text()) + current = objects(database) + exported_columns = set(manifest[0]) - {"file", "sha256", "content_type", "cache_control"} if manifest else set() + original = {obj["id"]: {key: obj[key] for key in exported_columns} for obj in manifest} + retained = {obj["id"]: {key: obj.get(key) for key in exported_columns} for obj in current} + if retained != original or len(current) != len(manifest): + raise ValueError("Database object metadata changed; verify before reopening writes") + for obj in manifest: + hasher = hashlib.sha256() + with download(url, obj) as response: + if response.headers.get("Content-Type") != obj["content_type"]: + raise ValueError("Object content type changed") + if response.headers.get("Cache-Control") != obj["cache_control"]: + raise ValueError("Object cache control changed") + while chunk := response.read(1024 * 1024): + hasher.update(chunk) + if hasher.hexdigest() != obj["sha256"]: + raise ValueError("Migrated object checksum mismatch") + print(f"Verified {len(manifest)} objects and unchanged database metadata") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=["export", "check", "install", "verify"]) + parser.add_argument("directory", type=Path) + parser.add_argument("--url", default="http://localhost:8000") + parser.add_argument("--database", default="postgres", help="POSTGRES_DB for export and verify") + parser.add_argument("--destination", type=Path) + parser.add_argument("--bucket", default="stub", help="GLOBAL_S3_BUCKET, not the application bucket") + parser.add_argument("--tenant", default="stub", help="STORAGE_TENANT_ID (previously TENANT_ID)") + args = parser.parse_args() + if args.operation == "check": + check(args.directory, args.bucket, args.tenant) + elif args.operation == "install": + if args.destination is None: + parser.error("install requires --destination") + install(args.directory, args.destination, args.bucket, args.tenant) + elif args.operation == "export": + export(args.url, args.directory, args.database) + else: + verify(args.url, args.directory, args.database) + + +if __name__ == "__main__": + main() diff --git a/supabase/prepare-pg17-upgrade.py b/supabase/prepare-pg17-upgrade.py new file mode 100644 index 0000000000..bbcff05b41 --- /dev/null +++ b/supabase/prepare-pg17-upgrade.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Prepare the pinned upstream upgrade script for this deployment's saved overrides.""" +import argparse +from pathlib import Path + + +VOLUME_SELECTION = '''db_config_vol=$(docker volume ls --filter "name=db-config" --format '{{.Name}}' \\ + | grep -E '^db-config$|_db-config$' | head -n 1)''' +CONTAINER_VOLUME = '''db_config_vol=$(docker inspect "$DB_CONTAINER" --format \\ + '{{range .Mounts}}{{if eq .Destination "/etc/postgresql-custom"}}{{.Name}}{{end}}{{end}}')''' +STARTUP = 'docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d' +FAILED_MIGRATION = '|| warn " $m failed (non-fatal)"' + + +def prepare(source, target): + if source.resolve() == target.resolve(): + raise ValueError("Keep the original upstream script unchanged") + text = source.read_text() + # Fail before writing if a different upstream release changes any assumption. + for old, new in ((f" {VOLUME_SELECTION}\n", f" {CONTAINER_VOLUME}\n"), + (f" {STARTUP}\n", " docker compose up -d\n"), + (FAILED_MIGRATION, '|| die "Required migration $m failed; keep writes blocked"')): + if text.count(old) != 1: + raise ValueError("Unexpected upstream upgrade script; review this release before upgrading") + text = text.replace(old, new) + target.write_text(text) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("target", type=Path) + args = parser.parse_args() + prepare(args.source, args.target) diff --git a/supabase/revisit.sql b/supabase/revisit.sql new file mode 100644 index 0000000000..635837594f --- /dev/null +++ b/supabase/revisit.sql @@ -0,0 +1,15 @@ +-- Match the existing documented access model; see README.md before applying. +BEGIN; +CREATE TABLE public.revisit ( + "createdAt" timestamp DEFAULT now(), + "studyId" varchar NOT NULL, + "docId" varchar NOT NULL, + data jsonb, + PRIMARY KEY ("studyId", "docId") +); +ALTER TABLE public.revisit ENABLE ROW LEVEL SECURITY; +GRANT SELECT, INSERT, UPDATE, DELETE ON public.revisit TO anon, authenticated, service_role; +CREATE POLICY allow_authenticated_read_write ON public.revisit FOR ALL TO anon, authenticated, service_role USING (true) WITH CHECK (true); +INSERT INTO storage.buckets (id, name, public) VALUES ('revisit', 'revisit', false); +CREATE POLICY revisit_storage_read_write ON storage.objects FOR ALL TO anon, authenticated, service_role USING (bucket_id = 'revisit') WITH CHECK (bucket_id = 'revisit'); +COMMIT; diff --git a/supabase/tests/migrate-storage.spec.py b/supabase/tests/migrate-storage.spec.py new file mode 100644 index 0000000000..42ee2c4ac2 --- /dev/null +++ b/supabase/tests/migrate-storage.spec.py @@ -0,0 +1,245 @@ +import hashlib +import json +import os +from io import BytesIO +from pathlib import Path +import runpy +import shutil +import tempfile +import unittest +from unittest.mock import Mock, patch + + +migration = runpy.run_path(str(Path(__file__).parents[1] / "migrate-storage.py")) + + +class StorageMigrationTests(unittest.TestCase): + def setUp(self): + attributes = patch("os.setxattr", create=True) + attributes.start() + self.addCleanup(attributes.stop) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.export = Path(self.temp.name) / "export" + self.export.mkdir() + self.target = Path(self.temp.name) / "storage" + data = b"existing participant data" + (self.export / "00000000.bin").write_bytes(data) + self.obj = {"bucket_id": "revisit", "name": "study/participant data.json", + "version": "existing-version", "file": "00000000.bin", + "sha256": hashlib.sha256(data).hexdigest(), + "content_type": "application/json", "cache_control": "max-age=3600"} + + def manifest(self, objects): + (self.export / "manifest.json").write_text(json.dumps(objects)) + + def test_export_and_verify_query_the_selected_database(self): + directory = self.export / "empty" + with patch("subprocess.run") as query: + query.return_value.stdout = "[]" + for operation in ("export", "verify"): + with patch("sys.argv", ["migrate-storage.py", operation, str(directory), + "--database", "study_database"]): + migration["main"]() + command = query.call_args.args[0] + self.assertEqual(command[command.index("-d") + 1], "study_database") + + def test_preserves_versioned_layout_and_http_metadata(self): + self.manifest([self.obj]) + with patch("os.setxattr", create=True) as attributes: + migration["install"](self.export, self.target, "stub", "stub") + result = self.target / "stub/stub/revisit/study/participant data.json/existing-version" + self.assertEqual(result.read_bytes(), b"existing participant data") + self.assertEqual({call.args[1]: call.args[2] for call in attributes.call_args_list}, { + "user.supabase.content-type": b"application/json", + "user.supabase.cache-control": b"max-age=3600", + }) + + def test_interrupted_copy_preserves_destination_and_can_be_retried(self): + self.manifest([self.obj]) + + def interrupted_copy(source, target): + Path(target).write_bytes(b"partial copy") + raise OSError("copy interrupted") + + for existing in (False, True): + with self.subTest(existing=existing): + root = self.target / str(existing) + target = migration["destination"](root, "stub", "stub", self.obj) + if existing: + target.parent.mkdir(parents=True) + shutil.copyfile(self.export / self.obj["file"], target) + with patch("shutil.copyfile", side_effect=interrupted_copy): + with self.assertRaisesRegex(OSError, "copy interrupted"): + migration["install"](self.export, root, "stub", "stub") + if existing: + self.assertEqual(target.read_bytes(), b"existing participant data") + else: + self.assertFalse(target.exists()) + migration["install"](self.export, root, "stub", "stub") + self.assertEqual(target.read_bytes(), b"existing participant data") + + def test_installed_files_are_readable_by_imgproxy_with_private_backup_umask(self): + self.manifest([self.obj]) + self.target.mkdir(mode=0o700) + previous = os.umask(0o077) + try: + migration["install"](self.export, self.target, "stub", "stub") + finally: + os.umask(previous) + result = migration["destination"](self.target, "stub", "stub", self.obj) + self.assertEqual(result.stat().st_mode & 0o777, 0o644) + for directory in result.parents: + if directory.is_relative_to(self.target): + self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + + def test_empty_export_prepares_readable_storage_root(self): + self.manifest([]) + self.target.mkdir(mode=0o700) + previous = os.umask(0o077) + try: + migration["install"](self.export, self.target, "stub", "stub") + finally: + os.umask(previous) + self.assertEqual(self.target.stat().st_mode & 0o777, 0o755) + + def test_corruption_is_rejected_before_installing_any_object(self): + self.manifest([self.obj, {**self.obj, "file": "00000001.bin"}]) + (self.export / "00000001.bin").write_bytes(b"corrupted") + with self.assertRaisesRegex(ValueError, "checksum"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + + def test_verify_ignores_row_order_but_rejects_changed_metadata(self): + objects = [{**self.obj, "id": "first"}, + {**self.obj, "id": "second", "name": "another object"}] + self.manifest(objects) + + def download(url, obj): + response = BytesIO(b"existing participant data") + response.headers = {"Content-Type": obj["content_type"], + "Cache-Control": obj["cache_control"]} + return response + + with patch.dict(migration["verify"].__globals__, + objects=lambda database: list(reversed(objects)), download=download): + migration["verify"]("http://example.test", self.export) + changed = [{**objects[0], "version": "changed"}, objects[1]] + with patch.dict(migration["verify"].__globals__, objects=lambda database: changed): + with self.assertRaisesRegex(ValueError, "metadata changed"): + migration["verify"]("http://example.test", self.export) + + def test_file_directory_collisions_fail_before_installing_any_payload(self): + second = {**self.obj, "file": "00000001.bin", + "name": "study/participant data.json/existing-version/child"} + (self.export / second["file"]).write_bytes(b"existing participant data") + self.manifest([self.obj, second]) + with self.assertRaisesRegex(ValueError, "conflict as files"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + with self.assertRaisesRegex(ValueError, "conflict as files"): + migration["check"](self.export, "stub", "stub") + + def test_duplicate_paths_and_missing_exports_fail_before_install(self): + for second in ({**self.obj, "file": "00000001.bin"}, + {**self.obj, "file": "00000001.bin", "name": "different"}): + with self.subTest(name=second["name"]): + self.manifest([self.obj, second]) + if second["name"] == self.obj["name"]: + (self.export / second["file"]).write_bytes(b"existing participant data") + with self.assertRaisesRegex(ValueError, "same file path"): + migration["install"](self.export, self.target, "stub", "stub") + (self.export / second["file"]).unlink() + else: + with self.assertRaises(FileNotFoundError): + migration["install"](self.export, self.target, "stub", "stub") + self.assertFalse(self.target.exists()) + + def test_symlink_escape_is_rejected_without_writing_outside_storage(self): + self.manifest([self.obj]) + self.target.mkdir() + outside = Path(self.temp.name) / "outside" + outside.mkdir() + (self.target / "stub").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "escapes destination"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(list(outside.iterdir()), []) + + def test_metadata_failure_keeps_payload_unpublished_and_allows_retry(self): + self.manifest([self.obj]) + with patch("os.setxattr", side_effect=OSError("extended attributes unavailable")): + with self.assertRaisesRegex(OSError, "extended attributes"): + migration["install"](self.export, self.target, "stub", "stub") + target = migration["destination"](self.target, "stub", "stub", self.obj) + self.assertFalse(target.exists()) + self.assertFalse(list(self.target.rglob(".revisit-migration-*"))) + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(target.read_bytes(), b"existing participant data") + + def test_concurrent_writes_and_interrupted_export_never_publish_a_manifest(self): + def response(url, obj): + stream = BytesIO(b"existing participant data") + stream.headers = {"Content-Type": "application/json", "Cache-Control": "max-age=3600"} + return stream + + original = [{"id": "original", "bucket_id": "revisit", "name": "recording", + "metadata": {"size": len(b"existing participant data")}}] + directory = self.export / "incomplete" + with patch.dict(migration["export"].__globals__, download=response, + objects=Mock(side_effect=[original, [{**original[0], "version": "new"}]])): + with self.assertRaisesRegex(ValueError, "metadata changed during export"): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + with self.assertRaises(FileExistsError): + migration["export"]("http://example.test", directory) + for failure in (OSError("connection interrupted"), ValueError("size differs")): + directory = directory.with_name(directory.name + "-retry") + with patch.dict(migration["export"].__globals__, objects=lambda database: original, + download=Mock(side_effect=failure)): + with self.assertRaises(type(failure)): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + + def test_truncated_http_response_without_database_size_never_completes_export(self): + stream = BytesIO(b"partial") + stream.headers = {"Content-Length": "100"} + directory = self.export / "truncated" + obj = {"id": "legacy", "bucket_id": "revisit", "name": "legacy", "metadata": None} + with patch.dict(migration["export"].__globals__, objects=lambda database: [obj], + download=lambda url, item: stream): + with self.assertRaisesRegex(ValueError, "Content-Length"): + migration["export"]("http://example.test", directory) + self.assertFalse((directory / "manifest.json").exists()) + + def test_verify_rejects_wrong_bytes_headers_and_added_objects(self): + obj = {**self.obj, "id": "existing"} + self.manifest([obj]) + for body, content_type, cache in ((b"corrupted", obj["content_type"], obj["cache_control"]), + (b"existing participant data", "text/plain", obj["cache_control"]), + (b"existing participant data", obj["content_type"], "no-cache")): + stream = BytesIO(body) + stream.headers = {"Content-Type": content_type, "Cache-Control": cache} + with patch.dict(migration["verify"].__globals__, objects=lambda database: [obj], + download=lambda url, item: stream): + with self.assertRaises(ValueError): + migration["verify"]("http://example.test", self.export) + with patch.dict(migration["verify"].__globals__, + objects=lambda database: [obj, {**obj, "id": "new"}]): + with self.assertRaisesRegex(ValueError, "metadata changed"): + migration["verify"]("http://example.test", self.export) + + def test_rejects_traversal_and_conflicting_destination(self): + self.manifest([{**self.obj, "name": "../outside"}]) + with self.assertRaisesRegex(ValueError, "Unsafe"): + migration["install"](self.export, self.target, "stub", "stub") + self.manifest([self.obj]) + result = migration["destination"](self.target, "stub", "stub", self.obj) + result.parent.mkdir(parents=True) + result.write_bytes(b"other data") + with self.assertRaisesRegex(ValueError, "different data"): + migration["install"](self.export, self.target, "stub", "stub") + self.assertEqual(result.read_bytes(), b"other data") + + +if __name__ == "__main__": + unittest.main() diff --git a/supabase/tests/prepare-pg17-upgrade.spec.py b/supabase/tests/prepare-pg17-upgrade.spec.py new file mode 100644 index 0000000000..58811f1229 --- /dev/null +++ b/supabase/tests/prepare-pg17-upgrade.spec.py @@ -0,0 +1,46 @@ +import runpy +from pathlib import Path +import tempfile +import unittest + + +helper = runpy.run_path(str(Path(__file__).parents[1] / "prepare-pg17-upgrade.py")) + + +class PrepareUpgradeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.source = Path(self.temp.name) / "upstream.sh" + self.target = Path(self.temp.name) / "prepared.sh" + self.original = (f" {helper['VOLUME_SELECTION']}\n" + f" {helper['STARTUP']}\n" + f" psql -f migration.sql {helper['FAILED_MIGRATION']}\n") + self.source.write_text(self.original) + + def test_preserves_source_selects_actual_volume_and_applies_saved_overrides(self): + helper["prepare"](self.source, self.target) + self.assertEqual(self.source.read_text(), self.original) + prepared = self.target.read_text() + self.assertIn(helper["CONTAINER_VOLUME"], prepared) + self.assertIn(" docker compose up -d\n", prepared) + self.assertNotIn('|| warn', prepared) + self.assertIn('|| die "Required migration $m failed; keep writes blocked"', prepared) + + def test_unknown_or_duplicated_upstream_commands_do_not_write_output(self): + for changed in (self.original.replace(helper["STARTUP"], "different command"), + self.original + f" {helper['VOLUME_SELECTION']}\n"): + with self.subTest(changed=changed): + self.source.write_text(changed) + with self.assertRaisesRegex(ValueError, "Unexpected upstream"): + helper["prepare"](self.source, self.target) + self.assertFalse(self.target.exists()) + + def test_rejects_overwriting_upstream_source(self): + with self.assertRaisesRegex(ValueError, "original upstream"): + helper["prepare"](self.source, self.source) + self.assertEqual(self.source.read_text(), self.original) + + +if __name__ == "__main__": + unittest.main() diff --git a/supabase/volumes/api/kong.yml b/supabase/volumes/api/kong.yml deleted file mode 100644 index b90c111b69..0000000000 --- a/supabase/volumes/api/kong.yml +++ /dev/null @@ -1,241 +0,0 @@ -_format_version: '2.1' -_transform: true - -### -### Consumers / Users -### -consumers: - - username: DASHBOARD - - username: anon - keyauth_credentials: - - key: $SUPABASE_ANON_KEY - - username: service_role - keyauth_credentials: - - key: $SUPABASE_SERVICE_KEY - -### -### Access Control List -### -acls: - - consumer: anon - group: anon - - consumer: service_role - group: admin - -### -### Dashboard credentials -### -basicauth_credentials: - - consumer: DASHBOARD - username: $DASHBOARD_USERNAME - password: $DASHBOARD_PASSWORD - -### -### API Routes -### -services: - ## Open Auth routes - - name: auth-v1-open - url: http://auth:9999/verify - routes: - - name: auth-v1-open - strip_path: true - paths: - - /auth/v1/verify - plugins: - - name: cors - - name: auth-v1-open-callback - url: http://auth:9999/callback - routes: - - name: auth-v1-open-callback - strip_path: true - paths: - - /auth/v1/callback - plugins: - - name: cors - - name: auth-v1-open-authorize - url: http://auth:9999/authorize - routes: - - name: auth-v1-open-authorize - strip_path: true - paths: - - /auth/v1/authorize - plugins: - - name: cors - - ## Secure Auth routes - - name: auth-v1 - _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' - url: http://auth:9999/ - routes: - - name: auth-v1-all - strip_path: true - paths: - - /auth/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure REST routes - - name: rest-v1 - _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' - url: http://rest:3000/ - routes: - - name: rest-v1-all - strip_path: true - paths: - - /rest/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure GraphQL routes - - name: graphql-v1 - _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql' - url: http://rest:3000/rpc/graphql - routes: - - name: graphql-v1-all - strip_path: true - paths: - - /graphql/v1 - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: request-transformer - config: - add: - headers: - - Content-Profile:graphql_public - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure Realtime routes - - name: realtime-v1-ws - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/socket - protocol: ws - routes: - - name: realtime-v1-ws - strip_path: true - paths: - - /realtime/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - name: realtime-v1-rest - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/api - protocol: http - routes: - - name: realtime-v1-rest - strip_path: true - paths: - - /realtime/v1/api - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - ## Storage routes: the storage server manages its own auth - - name: storage-v1 - _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' - url: http://storage:5000/ - routes: - - name: storage-v1-all - strip_path: true - paths: - - /storage/v1/ - plugins: - - name: cors - - ## Edge Functions routes - - name: functions-v1 - _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' - url: http://functions:9000/ - routes: - - name: functions-v1-all - strip_path: true - paths: - - /functions/v1/ - plugins: - - name: cors - - ## Analytics routes - - name: analytics-v1 - _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*' - url: http://analytics:4000/ - routes: - - name: analytics-v1-all - strip_path: true - paths: - - /analytics/v1/ - - ## Secure Database routes - - name: meta - _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*' - url: http://meta:8080/ - routes: - - name: meta-all - strip_path: true - paths: - - /pg/ - plugins: - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - ## Protected Dashboard - catch all remaining routes - - name: dashboard - _comment: 'Studio: /* -> http://studio:3000/*' - url: http://studio:3000/ - routes: - - name: dashboard-all - strip_path: true - paths: - - / - plugins: - - name: cors - - name: basic-auth - config: - hide_credentials: true \ No newline at end of file diff --git a/supabase/volumes/db/_supabase.sql b/supabase/volumes/db/_supabase.sql deleted file mode 100644 index 6236ae1bcc..0000000000 --- a/supabase/volumes/db/_supabase.sql +++ /dev/null @@ -1,3 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -CREATE DATABASE _supabase WITH OWNER :pguser; diff --git a/supabase/volumes/db/init/data.sql b/supabase/volumes/db/init/data.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/supabase/volumes/db/jwt.sql b/supabase/volumes/db/jwt.sql deleted file mode 100644 index cfd3b16028..0000000000 --- a/supabase/volumes/db/jwt.sql +++ /dev/null @@ -1,5 +0,0 @@ -\set jwt_secret `echo "$JWT_SECRET"` -\set jwt_exp `echo "$JWT_EXP"` - -ALTER DATABASE postgres SET "app.settings.jwt_secret" TO :'jwt_secret'; -ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; diff --git a/supabase/volumes/db/logs.sql b/supabase/volumes/db/logs.sql deleted file mode 100644 index 255c0f407b..0000000000 --- a/supabase/volumes/db/logs.sql +++ /dev/null @@ -1,6 +0,0 @@ -\set pguser `echo "$POSTGRES_USER"` - -\c _supabase -create schema if not exists _analytics; -alter schema _analytics owner to :pguser; -\c postgres diff --git a/supabase/volumes/db/roles.sql b/supabase/volumes/db/roles.sql deleted file mode 100644 index 8f7161a6db..0000000000 --- a/supabase/volumes/db/roles.sql +++ /dev/null @@ -1,8 +0,0 @@ --- NOTE: change to your own passwords for production environments -\set pgpass `echo "$POSTGRES_PASSWORD"` - -ALTER USER authenticator WITH PASSWORD :'pgpass'; -ALTER USER pgbouncer WITH PASSWORD :'pgpass'; -ALTER USER supabase_auth_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_functions_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; diff --git a/supabase/volumes/db/webhooks.sql b/supabase/volumes/db/webhooks.sql deleted file mode 100644 index 5837b86188..0000000000 --- a/supabase/volumes/db/webhooks.sql +++ /dev/null @@ -1,208 +0,0 @@ -BEGIN; - -- Create pg_net extension - CREATE EXTENSION IF NOT EXISTS pg_net SCHEMA extensions; - -- Create supabase_functions schema - CREATE SCHEMA supabase_functions AUTHORIZATION supabase_admin; - GRANT USAGE ON SCHEMA supabase_functions TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON TABLES TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON FUNCTIONS TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON SEQUENCES TO postgres, anon, authenticated, service_role; - -- supabase_functions.migrations definition - CREATE TABLE supabase_functions.migrations ( - version text PRIMARY KEY, - inserted_at timestamptz NOT NULL DEFAULT NOW() - ); - -- Initial supabase_functions migration - INSERT INTO supabase_functions.migrations (version) VALUES ('initial'); - -- supabase_functions.hooks definition - CREATE TABLE supabase_functions.hooks ( - id bigserial PRIMARY KEY, - hook_table_id integer NOT NULL, - hook_name text NOT NULL, - created_at timestamptz NOT NULL DEFAULT NOW(), - request_id bigint - ); - CREATE INDEX supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); - CREATE INDEX supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); - COMMENT ON TABLE supabase_functions.hooks IS 'Supabase Functions Hooks: Audit trail for triggered hooks.'; - CREATE FUNCTION supabase_functions.http_request() - RETURNS trigger - LANGUAGE plpgsql - AS $function$ - DECLARE - request_id bigint; - payload jsonb; - url text := TG_ARGV[0]::text; - method text := TG_ARGV[1]::text; - headers jsonb DEFAULT '{}'::jsonb; - params jsonb DEFAULT '{}'::jsonb; - timeout_ms integer DEFAULT 1000; - BEGIN - IF url IS NULL OR url = 'null' THEN - RAISE EXCEPTION 'url argument is missing'; - END IF; - - IF method IS NULL OR method = 'null' THEN - RAISE EXCEPTION 'method argument is missing'; - END IF; - - IF TG_ARGV[2] IS NULL OR TG_ARGV[2] = 'null' THEN - headers = '{"Content-Type": "application/json"}'::jsonb; - ELSE - headers = TG_ARGV[2]::jsonb; - END IF; - - IF TG_ARGV[3] IS NULL OR TG_ARGV[3] = 'null' THEN - params = '{}'::jsonb; - ELSE - params = TG_ARGV[3]::jsonb; - END IF; - - IF TG_ARGV[4] IS NULL OR TG_ARGV[4] = 'null' THEN - timeout_ms = 1000; - ELSE - timeout_ms = TG_ARGV[4]::integer; - END IF; - - CASE - WHEN method = 'GET' THEN - SELECT http_get INTO request_id FROM net.http_get( - url, - params, - headers, - timeout_ms - ); - WHEN method = 'POST' THEN - payload = jsonb_build_object( - 'old_record', OLD, - 'record', NEW, - 'type', TG_OP, - 'table', TG_TABLE_NAME, - 'schema', TG_TABLE_SCHEMA - ); - - SELECT http_post INTO request_id FROM net.http_post( - url, - payload, - params, - headers, - timeout_ms - ); - ELSE - RAISE EXCEPTION 'method argument % is invalid', method; - END CASE; - - INSERT INTO supabase_functions.hooks - (hook_table_id, hook_name, request_id) - VALUES - (TG_RELID, TG_NAME, request_id); - - RETURN NEW; - END - $function$; - -- Supabase super admin - DO - $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_roles - WHERE rolname = 'supabase_functions_admin' - ) - THEN - CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; - END IF; - END - $$; - GRANT ALL PRIVILEGES ON SCHEMA supabase_functions TO supabase_functions_admin; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA supabase_functions TO supabase_functions_admin; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA supabase_functions TO supabase_functions_admin; - ALTER USER supabase_functions_admin SET search_path = "supabase_functions"; - ALTER table "supabase_functions".migrations OWNER TO supabase_functions_admin; - ALTER table "supabase_functions".hooks OWNER TO supabase_functions_admin; - ALTER function "supabase_functions".http_request() OWNER TO supabase_functions_admin; - GRANT supabase_functions_admin TO postgres; - -- Remove unused supabase_pg_net_admin role - DO - $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_roles - WHERE rolname = 'supabase_pg_net_admin' - ) - THEN - REASSIGN OWNED BY supabase_pg_net_admin TO supabase_admin; - DROP OWNED BY supabase_pg_net_admin; - DROP ROLE supabase_pg_net_admin; - END IF; - END - $$; - -- pg_net grants when extension is already enabled - DO - $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_extension - WHERE extname = 'pg_net' - ) - THEN - GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - END IF; - END - $$; - -- Event trigger for pg_net - CREATE OR REPLACE FUNCTION extensions.grant_pg_net_access() - RETURNS event_trigger - LANGUAGE plpgsql - AS $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_event_trigger_ddl_commands() AS ev - JOIN pg_extension AS ext - ON ev.objid = ext.oid - WHERE ext.extname = 'pg_net' - ) - THEN - GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; - ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; - REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; - GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; - END IF; - END; - $$; - COMMENT ON FUNCTION extensions.grant_pg_net_access IS 'Grants access to pg_net'; - DO - $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_event_trigger - WHERE evtname = 'issue_pg_net_access' - ) THEN - CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end WHEN TAG IN ('CREATE EXTENSION') - EXECUTE PROCEDURE extensions.grant_pg_net_access(); - END IF; - END - $$; - INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants'); - ALTER function supabase_functions.http_request() SECURITY DEFINER; - ALTER function supabase_functions.http_request() SET search_path = supabase_functions; - REVOKE ALL ON FUNCTION supabase_functions.http_request() FROM PUBLIC; - GRANT EXECUTE ON FUNCTION supabase_functions.http_request() TO postgres, anon, authenticated, service_role; -COMMIT; diff --git a/tests/supabase-live.spec.ts b/tests/supabase-live.spec.ts new file mode 100644 index 0000000000..81160c9f2c --- /dev/null +++ b/tests/supabase-live.spec.ts @@ -0,0 +1,240 @@ +/* eslint-disable no-await-in-loop */ +import { randomUUID } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { createClient } from '@supabase/supabase-js'; +import { expect, Page, test } from '@playwright/test'; +import { ParticipantData } from '../src/storage/types'; +import { nextClick, waitForStudyEndMessage } from './utils'; + +// Run against an isolated local stack, with Vite configured to use the same Supabase URL. +// Keep the participants so this suite can verify them again after a migration/restart. +const enabled = process.env.PW_SUPABASE_LIVE === '1'; +const studyId = 'dev-demo-condition'; + +test.describe('Live self-hosted Supabase', () => { + test.skip(!enabled, 'Set PW_SUPABASE_LIVE=1 for the local Supabase migration rehearsal'); + + function storageClient() { + const url = process.env.VITE_SUPABASE_URL; + const key = process.env.PW_SUPABASE_SERVICE_ROLE_KEY; + if (!url || !key) throw new Error('Live Supabase requires VITE_SUPABASE_URL and PW_SUPABASE_SERVICE_ROLE_KEY'); + return createClient(url, key, { auth: { persistSession: false, autoRefreshToken: false } }); + } + + async function readParticipant(participantId: string) { + const { data, error } = await storageClient().storage.from('revisit') + .download(`${studyId}/participants/${participantId}_participantData`); + if (error || !data) return null; + return JSON.parse(await data.text()) as ParticipantData; + } + + function colorAnswers(participant: ParticipantData | null) { + return Object.values(participant?.answers ?? {}) + .filter((answer) => answer.componentName.startsWith('color-trial-')) + .map((answer) => answer.answer); + } + + async function selectParticipant(page: Page, participantId: string) { + await page.goto('/'); + await page.evaluate(({ id, study }) => new Promise((resolve, reject) => { + const request = indexedDB.open('revisit-supabase'); + request.onupgradeneeded = () => request.result.createObjectStore('keyvaluepairs'); + request.onerror = () => reject(request.error); + request.onsuccess = () => { + const database = request.result; + const transaction = database.transaction('keyvaluepairs', 'readwrite'); + transaction.objectStore('keyvaluepairs').put(id, `${study}/currentParticipantId`); + transaction.oncomplete = () => { database.close(); resolve(); }; + transaction.onerror = () => { database.close(); reject(transaction.error); }; + }; + }), { id: participantId, study: studyId }); + } + + test('persists real participant answers and resumes from Supabase in a fresh browser context', async ({ page, browser }) => { + const startedAt = Date.now(); + const participantId = process.env.PW_SUPABASE_PARTICIPANT_ID || randomUUID(); + const studyPath = '/demo-condition?condition=color'; + await selectParticipant(page, participantId); + await page.goto(studyPath); + await expect(page.getByText(/url query parameter.*condition/i)).toBeVisible(); + await nextClick(page); + + for (let trial = 0; trial < 2; trial += 1) { + await expect(page.getByText(/Which color is the (lightest|darkest)\?/)).toBeVisible(); + await page.getByPlaceholder('Select an option').click(); + await page.getByRole('option', { name: 'A', exact: true }).click(); + await nextClick(page); + } + await waitForStudyEndMessage(page); + + await expect.poll(async () => colorAnswers(await readParticipant(participantId)), { timeout: 15000 }) + .toEqual(expect.arrayContaining([ + { 'color-trial-1-response': 'A' }, + { 'color-trial-2-response': 'A' }, + ])); + const participant = await readParticipant(participantId); + expect(participant?.participantId).toBe(participantId); + expect(participant?.participantConfigHash).toBeTruthy(); + const { data: assignments, error } = await storageClient().from('revisit').select('data') + .eq('studyId', studyId) + .eq('docId', `sequenceAssignment_${participantId}`); + expect(error).toBeNull(); + expect(assignments?.[0]?.data.completed).toBeGreaterThanOrEqual(startedAt); + + // Force the reload to retrieve the participant from the backend, not localforage. + const completedUrl = page.url(); + const freshContext = await browser.newContext({ baseURL: new URL(completedUrl).origin }); + try { + const freshPage = await freshContext.newPage(); + await selectParticipant(freshPage, participantId); + await freshPage.goto(completedUrl); + await waitForStudyEndMessage(freshPage); + } finally { + await freshContext.close(); + } + expect(colorAnswers(await readParticipant(participantId))).toEqual(colorAnswers(participant)); + }); + + test('reads a participant retained from the old stack after migration', async ({ page }) => { + const participantId = process.env.PW_SUPABASE_EXISTING_PARTICIPANT; + test.skip(!participantId, 'Set PW_SUPABASE_EXISTING_PARTICIPANT to the pre-migration participant ID'); + const participant = await readParticipant(participantId!); + expect(participant?.participantId).toBe(participantId); + expect(colorAnswers(participant)).toEqual(expect.arrayContaining([ + { 'color-trial-1-response': 'A' }, + { 'color-trial-2-response': 'A' }, + ])); + await selectParticipant(page, participantId!); + await page.goto('/demo-condition?condition=color'); + await nextClick(page); + await expect(page.getByText(/Which color is the (lightest|darkest)\?/)).toBeVisible(); + await expect(page.getByPlaceholder('Select an option')).toHaveValue('A'); + expect(colorAnswers(await readParticipant(participantId!))).toEqual(colorAnswers(participant)); + }); + + test('resumes an unfinished pre-migration participant and preserves the earlier answer', async ({ page }) => { + const participantId = process.env.PW_SUPABASE_RESUME_PARTICIPANT; + test.skip(!participantId, 'Set PW_SUPABASE_RESUME_PARTICIPANT for the unfinished participant rehearsal'); + const seed = process.env.PW_SUPABASE_SEED_RESUME === '1'; + const previous = seed ? null : await readParticipant(participantId!); + const earlierAnswer = Object.values(previous?.answers ?? {}) + .find((answer) => answer.componentName.startsWith('color-trial-') && Object.values(answer.answer).includes('A')); + if (!seed) { + expect(previous?.participantId).toBe(participantId); + expect(earlierAnswer).toBeTruthy(); + expect(colorAnswers(previous).flatMap((answer) => Object.values(answer))).not.toContain('B'); + } + + await selectParticipant(page, participantId!); + await page.goto('/demo-condition?condition=color'); + await nextClick(page); + const question = page.getByText(/Which color is the (lightest|darkest)\?/); + await expect(question).toBeVisible(); + const firstQuestion = await question.textContent(); + const response = page.getByPlaceholder('Select an option'); + if (seed) { + await response.click(); + await page.getByRole('option', { name: 'A', exact: true }).click(); + } else { + await expect(response).toHaveValue('A'); + } + await nextClick(page); + await expect(question).toBeVisible(); + await expect(question).not.toHaveText(firstQuestion!); + await expect(response).toHaveValue(''); + await expect.poll(async () => colorAnswers(await readParticipant(participantId!)) + .flatMap((answer) => Object.values(answer))).toContain('A'); + + if (!seed) { + await response.click(); + await page.getByRole('option', { name: 'B', exact: true }).click(); + await nextClick(page); + await waitForStudyEndMessage(page); + await expect.poll(async () => colorAnswers(await readParticipant(participantId!)) + .flatMap((answer) => Object.values(answer))).toEqual(expect.arrayContaining(['A', 'B'])); + const current = await readParticipant(participantId!); + expect(Object.values(current!.answers).find((answer) => answer.componentName === earlierAnswer!.componentName)?.answer) + .toEqual(earlierAnswer!.answer); + } + }); + + test('authenticates a retained Designer account and exports retained answers through the UI', async ({ page }) => { + const email = process.env.PW_SUPABASE_DESIGNER_EMAIL; + const password = process.env.PW_SUPABASE_DESIGNER_PASSWORD; + const participantId = process.env.PW_SUPABASE_EXISTING_PARTICIPANT; + test.skip(!email || !password || !participantId, 'Set Designer credentials and the retained participant ID'); + + await page.goto('/analysis/stats/demo-condition/manage'); + await expect(page.getByRole('button', { name: 'Sign In With Supabase' })).toBeVisible(); + + // The app's sign-in button uses external GitHub OAuth. Use a real Auth password + // session here to exercise account retention, authorization and exports locally. + const url = process.env.VITE_SUPABASE_URL!; + const client = createClient(url, process.env.VITE_SUPABASE_ANON_KEY!, { + auth: { persistSession: false, autoRefreshToken: false }, + }); + const { data, error } = await client.auth.signInWithPassword({ email: email!, password: password! }); + expect(error).toBeNull(); + expect(data.user?.email).toBe(email); + const storageKey = `sb-${new URL(url).hostname.split('.')[0]}-auth-token`; + await page.evaluate(({ key, session }) => localStorage.setItem(key, JSON.stringify(session)), { + key: storageKey, session: data.session, + }); + await page.goto('/analysis/stats/demo-condition/table'); + await expect(page.getByRole('columnheader', { name: /^ID Sort by ID/ })).toBeVisible(); + + const jsonButton = page.locator('button').filter({ has: page.locator('svg.tabler-icon-database-export') }); + await expect(jsonButton).toBeEnabled(); + const jsonDownload = page.waitForEvent('download'); + await jsonButton.click(); + const downloaded = await jsonDownload; + expect(await downloaded.failure()).toBeNull(); + const participants = JSON.parse(await readFile((await downloaded.path())!, 'utf8')) as ParticipantData[]; + const retained = participants.find((participant) => participant.participantId === participantId); + expect(retained).toBeTruthy(); + expect(retained?.answers).toEqual((await readParticipant(participantId!))?.answers); + + const csvButton = page.locator('button').filter({ has: page.locator('svg.tabler-icon-table-export') }); + await csvButton.click(); + const exporter = page.getByRole('dialog', { name: 'Tidy CSV Exporter' }); + await expect(exporter).toBeVisible(); + const csvDownload = page.waitForEvent('download'); + await exporter.getByRole('button', { name: 'Download', exact: true }).click(); + const csv = await csvDownload; + expect(await csv.failure()).toBeNull(); + expect(await readFile((await csv.path())!, 'utf8')).toContain(participantId!); + }); + + test('supports binary files, overwrites and signed downloads with a participant session', async ({ request }) => { + const client = createClient(process.env.VITE_SUPABASE_URL!, process.env.VITE_SUPABASE_ANON_KEY!, { + auth: { persistSession: false, autoRefreshToken: false }, + }); + const { error: authError } = await client.auth.signInAnonymously(); + expect(authError).toBeNull(); + const bucket = client.storage.from('revisit'); + const path = `${studyId}/recording checks/${randomUUID()}.webm`; + const original = Uint8Array.from([0, 1, 127, 128, 255]); + const replacement = Uint8Array.from([255, 128, 0, 42]); + try { + const { error: uploadError } = await bucket.upload(path, original, { contentType: 'video/webm' }); + expect(uploadError).toBeNull(); + const { data: downloaded, error: downloadError } = await bucket.download(path); + expect(downloadError).toBeNull(); + expect(new Uint8Array(await downloaded!.arrayBuffer())).toEqual(original); + + const { error: overwriteError } = await bucket.upload(path, replacement, { upsert: true, contentType: 'video/webm' }); + expect(overwriteError).toBeNull(); + const { data: signed, error: signError } = await bucket.createSignedUrl(path, 60); + expect(signError).toBeNull(); + const response = await request.get(signed!.signedUrl); + expect(response.ok()).toBeTruthy(); + expect(response.headers()['content-type']).toContain('video/webm'); + expect(new Uint8Array(await response.body())).toEqual(replacement); + } finally { + const { error } = await bucket.remove([path]); + expect(error).toBeNull(); + } + const { error: removedError } = await bucket.download(path); + expect(removedError).not.toBeNull(); + }); +});