feat: Add is_producer_url_unstable to feed schema and expose in API

.github/workflows/catalog-update.yml

@@ -18,6 +18,11 @@ on:
           - qa
           - prod
           - dev
+      CSV_FILE_NAME:
+        description: Name of the sources CSV object to load from the mdb-csv bucket (e.g. sources_test.csv). Defaults to sources.csv.
+        required: false
+        default: sources.csv
+        type: string
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated, gbfs-systems-updated ]
 
@@ -26,8 +31,23 @@ env:
   liquibase_version: '4.33.0'
 
 jobs:
+  validate-inputs:
+    name: Validate inputs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Disallow prod/all runs with a non-official CSV
+        # Security guard: a test CSV (anything other than sources.csv) must never be loaded into
+        # prod, and 'all' includes prod. Manual (workflow_dispatch) runs only; repository_dispatch
+        # always uses the official sources.csv. This job runs on every event and only fails on the
+        # disallowed combination, so all downstream jobs can safely depend on it.
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.CSV_FILE_NAME != 'sources.csv' && (inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod') }}
+        run: |
+          echo "::error::Refusing to run: CSV_FILE_NAME='${{ inputs.CSV_FILE_NAME }}' is not the official 'sources.csv', so ENVIRONMENT='${{ inputs.ENVIRONMENT }}' (all/prod) is not allowed. Use 'qa' or 'dev' for test CSVs."
+          exit 1
+
   resolve-api-meta-qa:
     name: QA Resolve API commit/version
+    needs: [ validate-inputs ]
     if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'qa'
     runs-on: ubuntu-latest
     outputs:
@@ -57,6 +77,8 @@ jobs:
       # With workflow_dispatch we take the specified values of DRY_RUN.
       DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
       CHECKOUT_REF: ${{ needs.resolve-api-meta-qa.outputs.CHECKOUT_REF }}
+      # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+      CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
@@ -68,6 +90,7 @@ jobs:
 
   resolve-api-meta-prod:
     name: PROD Resolve API commit/version
+    needs: [ validate-inputs ]
     if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod'
     runs-on: ubuntu-latest
     outputs:
@@ -97,6 +120,8 @@ jobs:
       # With workflow_dispatch we take the specified values of DRY_RUN.
       DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
       CHECKOUT_REF: ${{ needs.resolve-api-meta-prod.outputs.CHECKOUT_REF }}
+      # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+      CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
@@ -108,6 +133,7 @@ jobs:
 
   resolve-api-meta-dev:
     name: DEV Resolve API commit/version
+    needs: [ validate-inputs ]
     if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'dev'
     runs-on: ubuntu-latest
     outputs:
@@ -140,6 +166,8 @@ jobs:
       # With workflow_dispatch we take the specified values of DRY_RUN.
       DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
       CHECKOUT_REF: ${{ needs.resolve-api-meta-dev.outputs.CHECKOUT_REF }}
+      # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+      CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
@@ -152,7 +180,7 @@ jobs:
 
   notify-slack-on-failure:
     # Run after all relevant jobs and notify if any failed
-    needs: [ resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
+    needs: [ validate-inputs, resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
     if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }}
     runs-on: ubuntu-latest
     steps:

.github/workflows/db-update-content.yml

@@ -64,6 +64,11 @@ on:
         required: false
         default: main
         type: string
+      CSV_FILE_NAME:
+        description: Name of the sources CSV object to download from the mdb-csv bucket. Defaults to sources.csv.
+        required: false
+        default: sources.csv
+        type: string
 
 env:
   python_version: '3.11'
@@ -150,7 +155,7 @@ jobs:
 
     - name: Download csv version of the database
       if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }}
-      run: wget -O sources.csv https://storage.googleapis.com/storage/v1/b/mdb-csv/o/sources.csv?alt=media
+      run: wget -O sources.csv "https://storage.googleapis.com/storage/v1/b/mdb-csv/o/${{ inputs.CSV_FILE_NAME }}?alt=media"
 
     - name: Get full path of sources.csv
       if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }}

api/.coveragerc

@@ -5,6 +5,7 @@ omit =
     src/shared/database_gen/*
     src/shared/users_database_gen/*
     src/user_service_gen/*
+    src/utils/logger.py
 
 [report]
 exclude_lines =

api/src/scripts/populate_db_gtfs.py

@@ -274,6 +274,11 @@ def populate_db(self, session: "Session", fetch_url: bool = True):
                         new_url=producer_url,
                         source="populate_db_gtfs",
                     )
+            # Three-state flag from the catalog CSV ("True"/"False"/empty). An empty cell leaves the DB
+            # value untouched (same as is_official), so a re-import never wipes an operator-set value.
+            is_producer_url_unstable_from_csv = self.get_safe_boolean_value(row, "is_producer_url_unstable", None)
+            if is_producer_url_unstable_from_csv is not None:
+                feed.is_producer_url_unstable = is_producer_url_unstable_from_csv
             feed.authentication_type = str(int(float(self.get_safe_value(row, "urls.authentication_type", "0"))))
             feed.authentication_info_url = self.get_safe_value(row, "urls.authentication_info", "")
             feed.api_key_parameter_name = self.get_safe_value(row, "urls.api_key_parameter_name", "")

api/src/shared/db_models/basic_feed_impl.py

@@ -38,6 +38,7 @@ def from_orm(cls, feed: Feed | None) -> BasicFeed | None:
             feed_contact_email=feed.feed_contact_email,
             source_info=SourceInfo(
                 producer_url=feed.producer_url,
+                is_producer_url_unstable=feed.is_producer_url_unstable,
                 authentication_type=None if feed.authentication_type is None else int(feed.authentication_type),
                 authentication_info_url=feed.authentication_info_url,
                 api_key_parameter_name=feed.api_key_parameter_name,

api/src/shared/db_models/feed_impl.py

@@ -58,6 +58,7 @@ def to_orm_from_dict(cls, feed_dict: dict | None, db_session: Session | None = N
             provider=feed_dict.get("provider"),
             feed_contact_email=feed_dict.get("feed_contact_email"),
             producer_url=feed_dict.get("producer_url"),
+            is_producer_url_unstable=feed_dict.get("is_producer_url_unstable"),
             authentication_type=(
                 None if feed_dict.get("authentication_type") is None else str(feed_dict.get("authentication_type"))
             ),

api/src/shared/db_models/search_feed_item_result_impl.py

@@ -34,6 +34,7 @@ def from_orm_gtfs(cls, feed_search_row: t_feedsearch):
             feed_contact_email=feed_search_row.feed_contact_email,
             source_info=SourceInfo(
                 producer_url=feed_search_row.producer_url,
+                is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
                 authentication_type=(
                     int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
                 ),
@@ -88,6 +89,7 @@ def from_orm_gbfs(cls, feed_search_row):
             versions=feed_search_row.versions,
             source_info=SourceInfo(
                 producer_url=feed_search_row.producer_url,
+                is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
                 authentication_type=(
                     int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
                 ),
@@ -118,6 +120,7 @@ def from_orm_gtfs_rt(cls, feed_search_row):
             feed_contact_email=feed_search_row.feed_contact_email,
             source_info=SourceInfo(
                 producer_url=feed_search_row.producer_url,
+                is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
                 authentication_type=(
                     int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
                 ),

api/tests/integration/populate_tests/test_data/sources_test.csv

@@ -3,12 +3,15 @@
 # and see the effect.
 # We want to make sure the is_official flag is handled properly in that case.
 # Normally changing to True or False should change the original value but changing to an empty cell should not
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable
 # For mdb-40, change is_official from TRUE to empty. Should retain True# For mdb-40, change is_official from TRUE to empty. Should retain True
-40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
-# For mdb-50, change is_official from FALSE to empty. Should retain False
-50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
-# For mdb-1562, change is_official from FALSE to TRUE. Should change to True
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
-# For mdb-1563, change is_official from TRUE to FALSE. Should change to False
-1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,
+40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,
+# For mdb-50, change is_official from FALSE to empty. Should retain False.
+# Also leave is_producer_url_unstable empty here: should retain the base FALSE value.
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,
+# For mdb-1562, change is_official from FALSE to TRUE. Should change to True.
+# Also change is_producer_url_unstable from base FALSE to TRUE: should change to True.
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,TRUE
+# For mdb-1563, change is_official from TRUE to FALSE. Should change to False.
+# Also change is_producer_url_unstable from base TRUE to FALSE: should change to False.
+1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,FALSE

api/tests/integration/populate_tests/test_populate.py

@@ -55,6 +55,86 @@ def test_is_official_overwrite(client: TestClient, values):
     assert json_response["official"] is expected_official, values["assert_fail_message"]
 
 
+@pytest.mark.parametrize(
+    "values",
+    [
+        {
+            "feed_id": "mdb-40",
+            "expected_is_producer_url_unstable": True,
+            "assert_fail_message": "mdb-40, is_producer_url_unstable changed from TRUE to empty. Should retain True.",
+        },
+        {
+            "feed_id": "mdb-50",
+            "expected_is_producer_url_unstable": False,
+            "assert_fail_message": "mdb-50, is_producer_url_unstable changed from FALSE to empty. Should retain False.",
+        },
+        {
+            "feed_id": "mdb-1562",
+            "expected_is_producer_url_unstable": True,
+            "assert_fail_message": "mdb-1562, is_producer_url_unstable FALSE->TRUE should change to True.",
+        },
+        {
+            "feed_id": "mdb-1563",
+            "expected_is_producer_url_unstable": False,
+            "assert_fail_message": "mdb-1563, is_producer_url_unstable TRUE->FALSE should change to False.",
+        },
+    ],
+    ids=[
+        "unstable_change_true_to_empty",
+        "unstable_change_false_to_empty",
+        "unstable_change_false_to_true",
+        "unstable_change_true_to_false",
+    ],
+)
+def test_is_producer_url_unstable_overwrite(client: TestClient, values):
+    """An empty CSV cell must retain the stored value; an explicit True/False must overwrite it."""
+    feed_id = values["feed_id"]
+    expected = values["expected_is_producer_url_unstable"]
+
+    response = client.request(
+        "GET",
+        "/v1/feeds/{id}".format(id=feed_id),
+        headers=authHeaders,
+    )
+
+    assert response.status_code == 200
+    json_response = response.json()
+    assert json_response["source_info"]["is_producer_url_unstable"] is expected, values["assert_fail_message"]
+
+
+@pytest.mark.parametrize(
+    "values",
+    [
+        {
+            "feed_id": "mdb-702",
+            "assert_fail_message": "mdb-702 has an empty is_producer_url_unstable cell; must stay null, not False.",
+        },
+        {
+            "feed_id": "mdb-1",
+            "assert_fail_message": "mdb-1's CSV has no is_producer_url_unstable column; must default to null.",
+        },
+    ],
+    ids=[
+        "unstable_empty_cell_stays_null",
+        "unstable_absent_column_stays_null",
+    ],
+)
+def test_is_producer_url_unstable_defaults_null(client: TestClient, values):
+    """An empty cell or a missing column must leave is_producer_url_unstable as null (never coerced to False)."""
+    feed_id = values["feed_id"]
+
+    response = client.request(
+        "GET",
+        "/v1/feeds/{id}".format(id=feed_id),
+        headers=authHeaders,
+    )
+
+    assert response.status_code == 200
+    json_response = response.json()
+    # Robust to either representation: key present with null, or key omitted.
+    assert json_response["source_info"].get("is_producer_url_unstable") is None, values["assert_fail_message"]
+
+
 def test_is_feed_reference_overwrite(client: TestClient):
     feed_id = "mdb-1562"
     response = client.request(