diff --git a/.github/workflows/catalog-update.yml b/.github/workflows/catalog-update.yml
index b90deba60..79582118a 100644
--- a/.github/workflows/catalog-update.yml
+++ b/.github/workflows/catalog-update.yml
@@ -18,6 +18,11 @@ on:
- qa
- prod
- dev
+ CSV_FILE_NAME:
+ description: Name of the sources CSV object to load from the mdb-csv bucket (e.g. sources_test.csv). Defaults to sources.csv.
+ required: false
+ default: sources.csv
+ type: string
repository_dispatch: # Update on mobility-database-catalog repo dispatch
types: [ catalog-sources-updated, gbfs-systems-updated ]
@@ -26,8 +31,23 @@ env:
liquibase_version: '4.33.0'
jobs:
+ validate-inputs:
+ name: Validate inputs
+ runs-on: ubuntu-latest
+ steps:
+ - name: Disallow prod/all runs with a non-official CSV
+ # Security guard: a test CSV (anything other than sources.csv) must never be loaded into
+ # prod, and 'all' includes prod. Manual (workflow_dispatch) runs only; repository_dispatch
+ # always uses the official sources.csv. This job runs on every event and only fails on the
+ # disallowed combination, so all downstream jobs can safely depend on it.
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.CSV_FILE_NAME != 'sources.csv' && (inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod') }}
+ run: |
+ echo "::error::Refusing to run: CSV_FILE_NAME='${{ inputs.CSV_FILE_NAME }}' is not the official 'sources.csv', so ENVIRONMENT='${{ inputs.ENVIRONMENT }}' (all/prod) is not allowed. Use 'qa' or 'dev' for test CSVs."
+ exit 1
+
resolve-api-meta-qa:
name: QA Resolve API commit/version
+ needs: [ validate-inputs ]
if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'qa'
runs-on: ubuntu-latest
outputs:
@@ -57,6 +77,8 @@ jobs:
# With workflow_dispatch we take the specified values of DRY_RUN.
DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
CHECKOUT_REF: ${{ needs.resolve-api-meta-qa.outputs.CHECKOUT_REF }}
+ # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+ CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
secrets:
DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
@@ -68,6 +90,7 @@ jobs:
resolve-api-meta-prod:
name: PROD Resolve API commit/version
+ needs: [ validate-inputs ]
if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod'
runs-on: ubuntu-latest
outputs:
@@ -97,6 +120,8 @@ jobs:
# With workflow_dispatch we take the specified values of DRY_RUN.
DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
CHECKOUT_REF: ${{ needs.resolve-api-meta-prod.outputs.CHECKOUT_REF }}
+ # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+ CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
secrets:
DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
@@ -108,6 +133,7 @@ jobs:
resolve-api-meta-dev:
name: DEV Resolve API commit/version
+ needs: [ validate-inputs ]
if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'dev'
runs-on: ubuntu-latest
outputs:
@@ -140,6 +166,8 @@ jobs:
# With workflow_dispatch we take the specified values of DRY_RUN.
DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }}
CHECKOUT_REF: ${{ needs.resolve-api-meta-dev.outputs.CHECKOUT_REF }}
+ # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv.
+ CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }}
secrets:
DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
@@ -152,7 +180,7 @@ jobs:
notify-slack-on-failure:
# Run after all relevant jobs and notify if any failed
- needs: [ resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
+ needs: [ validate-inputs, resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }}
runs-on: ubuntu-latest
steps:
diff --git a/.github/workflows/db-update-content.yml b/.github/workflows/db-update-content.yml
index 3eb058ec6..4b1525c51 100644
--- a/.github/workflows/db-update-content.yml
+++ b/.github/workflows/db-update-content.yml
@@ -64,6 +64,11 @@ on:
required: false
default: main
type: string
+ CSV_FILE_NAME:
+ description: Name of the sources CSV object to download from the mdb-csv bucket. Defaults to sources.csv.
+ required: false
+ default: sources.csv
+ type: string
env:
python_version: '3.11'
@@ -150,7 +155,7 @@ jobs:
- name: Download csv version of the database
if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }}
- run: wget -O sources.csv https://storage.googleapis.com/storage/v1/b/mdb-csv/o/sources.csv?alt=media
+ run: wget -O sources.csv "https://storage.googleapis.com/storage/v1/b/mdb-csv/o/${{ inputs.CSV_FILE_NAME }}?alt=media"
- name: Get full path of sources.csv
if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }}
diff --git a/api/.coveragerc b/api/.coveragerc
index 7201df45d..5fd71aba3 100644
--- a/api/.coveragerc
+++ b/api/.coveragerc
@@ -5,6 +5,7 @@ omit =
src/shared/database_gen/*
src/shared/users_database_gen/*
src/user_service_gen/*
+ src/utils/logger.py
[report]
exclude_lines =
diff --git a/api/src/scripts/populate_db_gtfs.py b/api/src/scripts/populate_db_gtfs.py
index 99f0edf44..d106f58d7 100644
--- a/api/src/scripts/populate_db_gtfs.py
+++ b/api/src/scripts/populate_db_gtfs.py
@@ -274,6 +274,11 @@ def populate_db(self, session: "Session", fetch_url: bool = True):
new_url=producer_url,
source="populate_db_gtfs",
)
+ # Three-state flag from the catalog CSV ("True"/"False"/empty). An empty cell leaves the DB
+ # value untouched (same as is_official), so a re-import never wipes an operator-set value.
+ is_producer_url_unstable_from_csv = self.get_safe_boolean_value(row, "is_producer_url_unstable", None)
+ if is_producer_url_unstable_from_csv is not None:
+ feed.is_producer_url_unstable = is_producer_url_unstable_from_csv
feed.authentication_type = str(int(float(self.get_safe_value(row, "urls.authentication_type", "0"))))
feed.authentication_info_url = self.get_safe_value(row, "urls.authentication_info", "")
feed.api_key_parameter_name = self.get_safe_value(row, "urls.api_key_parameter_name", "")
diff --git a/api/src/shared/db_models/basic_feed_impl.py b/api/src/shared/db_models/basic_feed_impl.py
index fd36155bc..1e2ba2bb8 100644
--- a/api/src/shared/db_models/basic_feed_impl.py
+++ b/api/src/shared/db_models/basic_feed_impl.py
@@ -38,6 +38,7 @@ def from_orm(cls, feed: Feed | None) -> BasicFeed | None:
feed_contact_email=feed.feed_contact_email,
source_info=SourceInfo(
producer_url=feed.producer_url,
+ is_producer_url_unstable=feed.is_producer_url_unstable,
authentication_type=None if feed.authentication_type is None else int(feed.authentication_type),
authentication_info_url=feed.authentication_info_url,
api_key_parameter_name=feed.api_key_parameter_name,
diff --git a/api/src/shared/db_models/feed_impl.py b/api/src/shared/db_models/feed_impl.py
index d3b795a7e..abcd9c3d3 100644
--- a/api/src/shared/db_models/feed_impl.py
+++ b/api/src/shared/db_models/feed_impl.py
@@ -58,6 +58,7 @@ def to_orm_from_dict(cls, feed_dict: dict | None, db_session: Session | None = N
provider=feed_dict.get("provider"),
feed_contact_email=feed_dict.get("feed_contact_email"),
producer_url=feed_dict.get("producer_url"),
+ is_producer_url_unstable=feed_dict.get("is_producer_url_unstable"),
authentication_type=(
None if feed_dict.get("authentication_type") is None else str(feed_dict.get("authentication_type"))
),
diff --git a/api/src/shared/db_models/search_feed_item_result_impl.py b/api/src/shared/db_models/search_feed_item_result_impl.py
index 2b1ba1148..d342fb46b 100644
--- a/api/src/shared/db_models/search_feed_item_result_impl.py
+++ b/api/src/shared/db_models/search_feed_item_result_impl.py
@@ -34,6 +34,7 @@ def from_orm_gtfs(cls, feed_search_row: t_feedsearch):
feed_contact_email=feed_search_row.feed_contact_email,
source_info=SourceInfo(
producer_url=feed_search_row.producer_url,
+ is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
authentication_type=(
int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
),
@@ -88,6 +89,7 @@ def from_orm_gbfs(cls, feed_search_row):
versions=feed_search_row.versions,
source_info=SourceInfo(
producer_url=feed_search_row.producer_url,
+ is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
authentication_type=(
int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
),
@@ -118,6 +120,7 @@ def from_orm_gtfs_rt(cls, feed_search_row):
feed_contact_email=feed_search_row.feed_contact_email,
source_info=SourceInfo(
producer_url=feed_search_row.producer_url,
+ is_producer_url_unstable=feed_search_row.is_producer_url_unstable,
authentication_type=(
int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None
),
diff --git a/api/tests/integration/populate_tests/test_data/sources_test.csv b/api/tests/integration/populate_tests/test_data/sources_test.csv
index c7ca1cf39..2004b7100 100644
--- a/api/tests/integration/populate_tests/test_data/sources_test.csv
+++ b/api/tests/integration/populate_tests/test_data/sources_test.csv
@@ -3,12 +3,15 @@
# and see the effect.
# We want to make sure the is_official flag is handled properly in that case.
# Normally changing to True or False should change the original value but changing to an empty cell should not
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable
# For mdb-40, change is_official from TRUE to empty. Should retain True# For mdb-40, change is_official from TRUE to empty. Should retain True
-40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
-# For mdb-50, change is_official from FALSE to empty. Should retain False
-50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
-# For mdb-1562, change is_official from FALSE to TRUE. Should change to True
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
-# For mdb-1563, change is_official from TRUE to FALSE. Should change to False
-1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,
+40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,
+# For mdb-50, change is_official from FALSE to empty. Should retain False.
+# Also leave is_producer_url_unstable empty here: should retain the base FALSE value.
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,
+# For mdb-1562, change is_official from FALSE to TRUE. Should change to True.
+# Also change is_producer_url_unstable from base FALSE to TRUE: should change to True.
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,TRUE
+# For mdb-1563, change is_official from TRUE to FALSE. Should change to False.
+# Also change is_producer_url_unstable from base TRUE to FALSE: should change to False.
+1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,FALSE
diff --git a/api/tests/integration/populate_tests/test_populate.py b/api/tests/integration/populate_tests/test_populate.py
index 0fe9558cd..35f990a6b 100644
--- a/api/tests/integration/populate_tests/test_populate.py
+++ b/api/tests/integration/populate_tests/test_populate.py
@@ -55,6 +55,86 @@ def test_is_official_overwrite(client: TestClient, values):
assert json_response["official"] is expected_official, values["assert_fail_message"]
+@pytest.mark.parametrize(
+ "values",
+ [
+ {
+ "feed_id": "mdb-40",
+ "expected_is_producer_url_unstable": True,
+ "assert_fail_message": "mdb-40, is_producer_url_unstable changed from TRUE to empty. Should retain True.",
+ },
+ {
+ "feed_id": "mdb-50",
+ "expected_is_producer_url_unstable": False,
+ "assert_fail_message": "mdb-50, is_producer_url_unstable changed from FALSE to empty. Should retain False.",
+ },
+ {
+ "feed_id": "mdb-1562",
+ "expected_is_producer_url_unstable": True,
+ "assert_fail_message": "mdb-1562, is_producer_url_unstable FALSE->TRUE should change to True.",
+ },
+ {
+ "feed_id": "mdb-1563",
+ "expected_is_producer_url_unstable": False,
+ "assert_fail_message": "mdb-1563, is_producer_url_unstable TRUE->FALSE should change to False.",
+ },
+ ],
+ ids=[
+ "unstable_change_true_to_empty",
+ "unstable_change_false_to_empty",
+ "unstable_change_false_to_true",
+ "unstable_change_true_to_false",
+ ],
+)
+def test_is_producer_url_unstable_overwrite(client: TestClient, values):
+ """An empty CSV cell must retain the stored value; an explicit True/False must overwrite it."""
+ feed_id = values["feed_id"]
+ expected = values["expected_is_producer_url_unstable"]
+
+ response = client.request(
+ "GET",
+ "/v1/feeds/{id}".format(id=feed_id),
+ headers=authHeaders,
+ )
+
+ assert response.status_code == 200
+ json_response = response.json()
+ assert json_response["source_info"]["is_producer_url_unstable"] is expected, values["assert_fail_message"]
+
+
+@pytest.mark.parametrize(
+ "values",
+ [
+ {
+ "feed_id": "mdb-702",
+ "assert_fail_message": "mdb-702 has an empty is_producer_url_unstable cell; must stay null, not False.",
+ },
+ {
+ "feed_id": "mdb-1",
+ "assert_fail_message": "mdb-1's CSV has no is_producer_url_unstable column; must default to null.",
+ },
+ ],
+ ids=[
+ "unstable_empty_cell_stays_null",
+ "unstable_absent_column_stays_null",
+ ],
+)
+def test_is_producer_url_unstable_defaults_null(client: TestClient, values):
+ """An empty cell or a missing column must leave is_producer_url_unstable as null (never coerced to False)."""
+ feed_id = values["feed_id"]
+
+ response = client.request(
+ "GET",
+ "/v1/feeds/{id}".format(id=feed_id),
+ headers=authHeaders,
+ )
+
+ assert response.status_code == 200
+ json_response = response.json()
+ # Robust to either representation: key present with null, or key omitted.
+ assert json_response["source_info"].get("is_producer_url_unstable") is None, values["assert_fail_message"]
+
+
def test_is_feed_reference_overwrite(client: TestClient):
feed_id = "mdb-1562"
response = client.request(
diff --git a/api/tests/integration/test_data/sources_test.csv b/api/tests/integration/test_data/sources_test.csv
index b6fc48f62..43e19e02c 100644
--- a/api/tests/integration/test_data/sources_test.csv
+++ b/api/tests/integration/test_data/sources_test.csv
@@ -1,6 +1,6 @@
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
-40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
-50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
-702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
-1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,
\ No newline at end of file
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable
+40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,TRUE
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,FALSE
+702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,,
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,FALSE
+1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,TRUE
\ No newline at end of file
diff --git a/api/tests/unittest/models/test_basic_feed_impl.py b/api/tests/unittest/models/test_basic_feed_impl.py
index d262f773e..7bed51fc1 100644
--- a/api/tests/unittest/models/test_basic_feed_impl.py
+++ b/api/tests/unittest/models/test_basic_feed_impl.py
@@ -31,6 +31,7 @@
feed_name="feed_name",
note="note",
producer_url="producer_url",
+ is_producer_url_unstable=True,
authentication_type="1",
authentication_info_url="authentication_info_url",
api_key_parameter_name="api_key_parameter_name",
@@ -98,6 +99,7 @@
related_links=[],
source_info=SourceInfo(
producer_url="producer_url",
+ is_producer_url_unstable=True,
authentication_type=1,
authentication_info_url="authentication_info_url",
api_key_parameter_name="api_key_parameter_name",
@@ -178,6 +180,7 @@ def test_to_orm_from_dict_full_payload(self):
"provider": "Provider A",
"feed_contact_email": "contact@example.com",
"producer_url": "https://producer.example.com",
+ "is_producer_url_unstable": True,
"authentication_type": 1, # should be converted to string
"authentication_info_url": "https://auth.example.com",
"api_key_parameter_name": "api_key",
@@ -211,6 +214,7 @@ def test_to_orm_from_dict_full_payload(self):
assert obj.provider == "Provider A"
assert obj.feed_contact_email == "contact@example.com"
assert obj.producer_url == "https://producer.example.com"
+ assert obj.is_producer_url_unstable is True
# authentication_type coerced to string per implementation
assert obj.authentication_type == "1"
assert obj.authentication_info_url == "https://auth.example.com"
diff --git a/api/tests/unittest/models/test_search_feed_item_result_impl.py b/api/tests/unittest/models/test_search_feed_item_result_impl.py
index 1d5630056..ec1c2b9db 100644
--- a/api/tests/unittest/models/test_search_feed_item_result_impl.py
+++ b/api/tests/unittest/models/test_search_feed_item_result_impl.py
@@ -29,6 +29,7 @@ def __init__(self, **kwargs):
note="note",
feed_contact_email="feed_contact_email",
producer_url="producer_url",
+ is_producer_url_unstable=True,
authentication_info_url="authentication_info_url",
authentication_type=1,
api_key_parameter_name="api_key_parameter_name",
@@ -83,6 +84,7 @@ def test_from_orm_gtfs(self):
feed_contact_email=item.feed_contact_email,
source_info=SourceInfo(
producer_url=item.producer_url,
+ is_producer_url_unstable=item.is_producer_url_unstable,
authentication_type=int(item.authentication_type) if item.authentication_type else None,
authentication_info_url=item.authentication_info_url,
api_key_parameter_name=item.api_key_parameter_name,
@@ -129,6 +131,7 @@ def test_from_orm_gtfs_rt(self):
feed_contact_email=item.feed_contact_email,
source_info=SourceInfo(
producer_url=item.producer_url,
+ is_producer_url_unstable=item.is_producer_url_unstable,
authentication_type=int(item.authentication_type) if item.authentication_type else None,
authentication_info_url=item.authentication_info_url,
api_key_parameter_name=item.api_key_parameter_name,
diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml
index 537e23b2e..f016ba5e1 100644
--- a/docs/DatabaseCatalogAPI.yaml
+++ b/docs/DatabaseCatalogAPI.yaml
@@ -1147,11 +1147,22 @@ components:
properties:
producer_url:
description: >
- URL where the producer is providing the dataset.
+ URL where the producer is providing the dataset.
Refer to the authentication information to know how to access this URL.
type: string
format: url
example: https://ladotbus.com/gtfs
+ is_producer_url_unstable:
+ description: >
+ Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time.
+ This may be because the URL contains a date/time, or because the transit provider has
+ communicated that it is not permanent (e.g. it is updated monthly).
+ * true - The producer URL is unstable and changes over time.
+ * false - The producer URL is stable and unchanging over time.
+ * null (default) - There is not enough information to determine the stability of the producer URL.
+ type: boolean
+ nullable: true
+ example: true
authentication_type:
description: >
Defines the type of authentication required to access the `producer_url`. Valid values for this field are:
diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml
index bf53bfb36..538f2cef7 100644
--- a/docs/OperationsAPI.yaml
+++ b/docs/OperationsAPI.yaml
@@ -1337,6 +1337,19 @@ components:
type: string
format: url
example: https://ladotbus.com/gtfs
+ is_producer_url_unstable:
+ description: >
+ Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time.
+ This may be because the URL contains a date/time, or because the transit provider has
+ communicated that it is not permanent (e.g. it is updated monthly).
+
+ * true - The producer URL is unstable and changes over time.
+ * false - The producer URL is stable and unchanging over time.
+ * null (default) - There is not enough information to determine the stability of the producer URL.
+
+ type: boolean
+ nullable: true
+ example: true
authentication_type:
description: >
Defines the type of authentication required to access the `producer_url`. Valid values for this field are:
diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py
index 5d2d36098..b7c2ad55d 100644
--- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py
+++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py
@@ -52,6 +52,7 @@ def from_orm(cls, obj: Gtfsfeed | None) -> UpdateRequestGtfsFeed | None:
feed_contact_email=obj.feed_contact_email,
source_info=SourceInfo(
producer_url=obj.producer_url,
+ is_producer_url_unstable=obj.is_producer_url_unstable,
authentication_type=(
None
if obj.authentication_type is None
@@ -96,6 +97,12 @@ def to_orm(
)
else update_request.source_info.producer_url
)
+ # Nullable three-state boolean: preserve False and None (unknown), so guard only on source_info.
+ entity.is_producer_url_unstable = (
+ None
+ if update_request.source_info is None
+ else update_request.source_info.is_producer_url_unstable
+ )
entity.authentication_type = (
None
if (
diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py
index 782e129c4..f276d8404 100644
--- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py
+++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py
@@ -58,6 +58,7 @@ def from_orm(cls, obj: Gtfsrealtimefeed | None) -> UpdateRequestGtfsRtFeed | Non
feed_contact_email=obj.feed_contact_email,
source_info=SourceInfo(
producer_url=obj.producer_url,
+ is_producer_url_unstable=obj.is_producer_url_unstable,
authentication_type=(
None
if obj.authentication_type is None
@@ -108,6 +109,12 @@ def to_orm(
)
else update_request.source_info.producer_url
)
+ # Nullable three-state boolean: preserve False and None (unknown), so guard only on source_info.
+ entity.is_producer_url_unstable = (
+ None
+ if update_request.source_info is None
+ else update_request.source_info.is_producer_url_unstable
+ )
entity.authentication_type = (
None
if (
diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py
index 965729c1d..9e5ad714e 100644
--- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py
+++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py
@@ -22,6 +22,7 @@ def test_from_orm():
note="note",
feed_contact_email="email@example.com",
producer_url="http://producer.url",
+ is_producer_url_unstable=True,
authentication_type=1,
authentication_info_url="http://auth.info.url",
api_key_parameter_name="api_key",
@@ -38,6 +39,7 @@ def test_from_orm():
assert result.note == "note"
assert result.feed_contact_email == "email@example.com"
assert result.source_info.producer_url == "http://producer.url"
+ assert result.source_info.is_producer_url_unstable is True
assert result.source_info.authentication_type == 1
assert result.source_info.authentication_info_url == "http://auth.info.url"
assert result.source_info.api_key_parameter_name == "api_key"
@@ -63,6 +65,7 @@ def test_to_orm():
feed_contact_email="email@example.com",
source_info=SourceInfo(
producer_url="http://producer.url",
+ is_producer_url_unstable=True,
authentication_type=1,
authentication_info_url="http://auth.info.url",
api_key_parameter_name="api_key",
@@ -85,6 +88,7 @@ def test_to_orm():
assert result.note == "note"
assert result.feed_contact_email == "email@example.com"
assert result.producer_url == "http://producer.url"
+ assert result.is_producer_url_unstable is True
assert result.authentication_type == "1"
assert result.authentication_info_url == "http://auth.info.url"
assert result.api_key_parameter_name == "api_key"
@@ -114,6 +118,7 @@ def test_to_orm_invalid_source_info():
result = UpdateRequestGtfsFeedImpl.to_orm(update_request, entity, session)
assert result.producer_url is None
+ assert result.is_producer_url_unstable is None
assert result.authentication_type is None
assert result.authentication_info_url is None
assert result.api_key_parameter_name is None
diff --git a/liquibase/changelog.xml b/liquibase/changelog.xml
index 13822937d..ae02dc518 100644
--- a/liquibase/changelog.xml
+++ b/liquibase/changelog.xml
@@ -119,6 +119,8 @@
+
+
diff --git a/liquibase/changes/feat_1766.sql b/liquibase/changes/feat_1766.sql
new file mode 100644
index 000000000..fbb06a804
--- /dev/null
+++ b/liquibase/changes/feat_1766.sql
@@ -0,0 +1,5 @@
+-- Issue #1766: Add is_producer_url_unstable to the feed table.
+-- Marks whether a feed's producer URL is known to be unstable/non-permanent
+-- (e.g. it contains a date/time or the transit provider updates it periodically).
+-- Nullable three-state boolean: TRUE = unstable, FALSE = stable, NULL = unknown (default).
+ALTER TABLE feed ADD COLUMN IF NOT EXISTS is_producer_url_unstable BOOLEAN;
diff --git a/liquibase/materialized_views/feed_search.sql b/liquibase/materialized_views/feed_search.sql
index f59df5d18..81f058981 100644
--- a/liquibase/materialized_views/feed_search.sql
+++ b/liquibase/materialized_views/feed_search.sql
@@ -11,6 +11,7 @@ SELECT
-- source
Feed.producer_url,
+ Feed.is_producer_url_unstable,
Feed.authentication_info_url,
Feed.authentication_type,
Feed.api_key_parameter_name,