diff --git a/.github/workflows/catalog-update.yml b/.github/workflows/catalog-update.yml index b90deba60..79582118a 100644 --- a/.github/workflows/catalog-update.yml +++ b/.github/workflows/catalog-update.yml @@ -18,6 +18,11 @@ on: - qa - prod - dev + CSV_FILE_NAME: + description: Name of the sources CSV object to load from the mdb-csv bucket (e.g. sources_test.csv). Defaults to sources.csv. + required: false + default: sources.csv + type: string repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -26,8 +31,23 @@ env: liquibase_version: '4.33.0' jobs: + validate-inputs: + name: Validate inputs + runs-on: ubuntu-latest + steps: + - name: Disallow prod/all runs with a non-official CSV + # Security guard: a test CSV (anything other than sources.csv) must never be loaded into + # prod, and 'all' includes prod. Manual (workflow_dispatch) runs only; repository_dispatch + # always uses the official sources.csv. This job runs on every event and only fails on the + # disallowed combination, so all downstream jobs can safely depend on it. + if: ${{ github.event_name == 'workflow_dispatch' && inputs.CSV_FILE_NAME != 'sources.csv' && (inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod') }} + run: | + echo "::error::Refusing to run: CSV_FILE_NAME='${{ inputs.CSV_FILE_NAME }}' is not the official 'sources.csv', so ENVIRONMENT='${{ inputs.ENVIRONMENT }}' (all/prod) is not allowed. Use 'qa' or 'dev' for test CSVs." + exit 1 + resolve-api-meta-qa: name: QA Resolve API commit/version + needs: [ validate-inputs ] if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'qa' runs-on: ubuntu-latest outputs: @@ -57,6 +77,8 @@ jobs: # With workflow_dispatch we take the specified values of DRY_RUN. DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }} CHECKOUT_REF: ${{ needs.resolve-api-meta-qa.outputs.CHECKOUT_REF }} + # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv. + CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} @@ -68,6 +90,7 @@ jobs: resolve-api-meta-prod: name: PROD Resolve API commit/version + needs: [ validate-inputs ] if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'prod' runs-on: ubuntu-latest outputs: @@ -97,6 +120,8 @@ jobs: # With workflow_dispatch we take the specified values of DRY_RUN. DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }} CHECKOUT_REF: ${{ needs.resolve-api-meta-prod.outputs.CHECKOUT_REF }} + # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv. + CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }} secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} @@ -108,6 +133,7 @@ jobs: resolve-api-meta-dev: name: DEV Resolve API commit/version + needs: [ validate-inputs ] if: github.event_name == 'repository_dispatch' || inputs.ENVIRONMENT == 'all' || inputs.ENVIRONMENT == 'dev' runs-on: ubuntu-latest outputs: @@ -140,6 +166,8 @@ jobs: # With workflow_dispatch we take the specified values of DRY_RUN. DRY_RUN: ${{ (github.event_name != 'repository_dispatch' && inputs.DRY_RUN) }} CHECKOUT_REF: ${{ needs.resolve-api-meta-dev.outputs.CHECKOUT_REF }} + # repository_dispatch leaves inputs empty, so this falls back to the official sources.csv. + CSV_FILE_NAME: ${{ inputs.CSV_FILE_NAME || 'sources.csv' }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} @@ -152,7 +180,7 @@ jobs: notify-slack-on-failure: # Run after all relevant jobs and notify if any failed - needs: [ resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ] + needs: [ validate-inputs, resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ] if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/db-update-content.yml b/.github/workflows/db-update-content.yml index 3eb058ec6..4b1525c51 100644 --- a/.github/workflows/db-update-content.yml +++ b/.github/workflows/db-update-content.yml @@ -64,6 +64,11 @@ on: required: false default: main type: string + CSV_FILE_NAME: + description: Name of the sources CSV object to download from the mdb-csv bucket. Defaults to sources.csv. + required: false + default: sources.csv + type: string env: python_version: '3.11' @@ -150,7 +155,7 @@ jobs: - name: Download csv version of the database if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }} - run: wget -O sources.csv https://storage.googleapis.com/storage/v1/b/mdb-csv/o/sources.csv?alt=media + run: wget -O sources.csv "https://storage.googleapis.com/storage/v1/b/mdb-csv/o/${{ inputs.CSV_FILE_NAME }}?alt=media" - name: Get full path of sources.csv if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }} diff --git a/api/.coveragerc b/api/.coveragerc index 7201df45d..5fd71aba3 100644 --- a/api/.coveragerc +++ b/api/.coveragerc @@ -5,6 +5,7 @@ omit = src/shared/database_gen/* src/shared/users_database_gen/* src/user_service_gen/* + src/utils/logger.py [report] exclude_lines = diff --git a/api/src/scripts/populate_db_gtfs.py b/api/src/scripts/populate_db_gtfs.py index 99f0edf44..d106f58d7 100644 --- a/api/src/scripts/populate_db_gtfs.py +++ b/api/src/scripts/populate_db_gtfs.py @@ -274,6 +274,11 @@ def populate_db(self, session: "Session", fetch_url: bool = True): new_url=producer_url, source="populate_db_gtfs", ) + # Three-state flag from the catalog CSV ("True"/"False"/empty). An empty cell leaves the DB + # value untouched (same as is_official), so a re-import never wipes an operator-set value. + is_producer_url_unstable_from_csv = self.get_safe_boolean_value(row, "is_producer_url_unstable", None) + if is_producer_url_unstable_from_csv is not None: + feed.is_producer_url_unstable = is_producer_url_unstable_from_csv feed.authentication_type = str(int(float(self.get_safe_value(row, "urls.authentication_type", "0")))) feed.authentication_info_url = self.get_safe_value(row, "urls.authentication_info", "") feed.api_key_parameter_name = self.get_safe_value(row, "urls.api_key_parameter_name", "") diff --git a/api/src/shared/db_models/basic_feed_impl.py b/api/src/shared/db_models/basic_feed_impl.py index fd36155bc..1e2ba2bb8 100644 --- a/api/src/shared/db_models/basic_feed_impl.py +++ b/api/src/shared/db_models/basic_feed_impl.py @@ -38,6 +38,7 @@ def from_orm(cls, feed: Feed | None) -> BasicFeed | None: feed_contact_email=feed.feed_contact_email, source_info=SourceInfo( producer_url=feed.producer_url, + is_producer_url_unstable=feed.is_producer_url_unstable, authentication_type=None if feed.authentication_type is None else int(feed.authentication_type), authentication_info_url=feed.authentication_info_url, api_key_parameter_name=feed.api_key_parameter_name, diff --git a/api/src/shared/db_models/feed_impl.py b/api/src/shared/db_models/feed_impl.py index d3b795a7e..abcd9c3d3 100644 --- a/api/src/shared/db_models/feed_impl.py +++ b/api/src/shared/db_models/feed_impl.py @@ -58,6 +58,7 @@ def to_orm_from_dict(cls, feed_dict: dict | None, db_session: Session | None = N provider=feed_dict.get("provider"), feed_contact_email=feed_dict.get("feed_contact_email"), producer_url=feed_dict.get("producer_url"), + is_producer_url_unstable=feed_dict.get("is_producer_url_unstable"), authentication_type=( None if feed_dict.get("authentication_type") is None else str(feed_dict.get("authentication_type")) ), diff --git a/api/src/shared/db_models/search_feed_item_result_impl.py b/api/src/shared/db_models/search_feed_item_result_impl.py index 2b1ba1148..d342fb46b 100644 --- a/api/src/shared/db_models/search_feed_item_result_impl.py +++ b/api/src/shared/db_models/search_feed_item_result_impl.py @@ -34,6 +34,7 @@ def from_orm_gtfs(cls, feed_search_row: t_feedsearch): feed_contact_email=feed_search_row.feed_contact_email, source_info=SourceInfo( producer_url=feed_search_row.producer_url, + is_producer_url_unstable=feed_search_row.is_producer_url_unstable, authentication_type=( int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None ), @@ -88,6 +89,7 @@ def from_orm_gbfs(cls, feed_search_row): versions=feed_search_row.versions, source_info=SourceInfo( producer_url=feed_search_row.producer_url, + is_producer_url_unstable=feed_search_row.is_producer_url_unstable, authentication_type=( int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None ), @@ -118,6 +120,7 @@ def from_orm_gtfs_rt(cls, feed_search_row): feed_contact_email=feed_search_row.feed_contact_email, source_info=SourceInfo( producer_url=feed_search_row.producer_url, + is_producer_url_unstable=feed_search_row.is_producer_url_unstable, authentication_type=( int(feed_search_row.authentication_type) if feed_search_row.authentication_type else None ), diff --git a/api/tests/integration/populate_tests/test_data/sources_test.csv b/api/tests/integration/populate_tests/test_data/sources_test.csv index c7ca1cf39..2004b7100 100644 --- a/api/tests/integration/populate_tests/test_data/sources_test.csv +++ b/api/tests/integration/populate_tests/test_data/sources_test.csv @@ -3,12 +3,15 @@ # and see the effect. # We want to make sure the is_official flag is handled properly in that case. # Normally changing to True or False should change the original value but changing to an empty cell should not -mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment +mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable # For mdb-40, change is_official from TRUE to empty. Should retain True# For mdb-40, change is_official from TRUE to empty. Should retain True -40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,, -# For mdb-50, change is_official from FALSE to empty. Should retain False -50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment -# For mdb-1562, change is_official from FALSE to TRUE. Should change to True -1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10, -# For mdb-1563, change is_official from TRUE to FALSE. Should change to False -1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10, +40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,, +# For mdb-50, change is_official from FALSE to empty. Should retain False. +# Also leave is_producer_url_unstable empty here: should retain the base FALSE value. +50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment, +# For mdb-1562, change is_official from FALSE to TRUE. Should change to True. +# Also change is_producer_url_unstable from base FALSE to TRUE: should change to True. +1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,TRUE +# For mdb-1563, change is_official from TRUE to FALSE. Should change to False. +# Also change is_producer_url_unstable from base TRUE to FALSE: should change to False. +1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,FALSE diff --git a/api/tests/integration/populate_tests/test_populate.py b/api/tests/integration/populate_tests/test_populate.py index 0fe9558cd..35f990a6b 100644 --- a/api/tests/integration/populate_tests/test_populate.py +++ b/api/tests/integration/populate_tests/test_populate.py @@ -55,6 +55,86 @@ def test_is_official_overwrite(client: TestClient, values): assert json_response["official"] is expected_official, values["assert_fail_message"] +@pytest.mark.parametrize( + "values", + [ + { + "feed_id": "mdb-40", + "expected_is_producer_url_unstable": True, + "assert_fail_message": "mdb-40, is_producer_url_unstable changed from TRUE to empty. Should retain True.", + }, + { + "feed_id": "mdb-50", + "expected_is_producer_url_unstable": False, + "assert_fail_message": "mdb-50, is_producer_url_unstable changed from FALSE to empty. Should retain False.", + }, + { + "feed_id": "mdb-1562", + "expected_is_producer_url_unstable": True, + "assert_fail_message": "mdb-1562, is_producer_url_unstable FALSE->TRUE should change to True.", + }, + { + "feed_id": "mdb-1563", + "expected_is_producer_url_unstable": False, + "assert_fail_message": "mdb-1563, is_producer_url_unstable TRUE->FALSE should change to False.", + }, + ], + ids=[ + "unstable_change_true_to_empty", + "unstable_change_false_to_empty", + "unstable_change_false_to_true", + "unstable_change_true_to_false", + ], +) +def test_is_producer_url_unstable_overwrite(client: TestClient, values): + """An empty CSV cell must retain the stored value; an explicit True/False must overwrite it.""" + feed_id = values["feed_id"] + expected = values["expected_is_producer_url_unstable"] + + response = client.request( + "GET", + "/v1/feeds/{id}".format(id=feed_id), + headers=authHeaders, + ) + + assert response.status_code == 200 + json_response = response.json() + assert json_response["source_info"]["is_producer_url_unstable"] is expected, values["assert_fail_message"] + + +@pytest.mark.parametrize( + "values", + [ + { + "feed_id": "mdb-702", + "assert_fail_message": "mdb-702 has an empty is_producer_url_unstable cell; must stay null, not False.", + }, + { + "feed_id": "mdb-1", + "assert_fail_message": "mdb-1's CSV has no is_producer_url_unstable column; must default to null.", + }, + ], + ids=[ + "unstable_empty_cell_stays_null", + "unstable_absent_column_stays_null", + ], +) +def test_is_producer_url_unstable_defaults_null(client: TestClient, values): + """An empty cell or a missing column must leave is_producer_url_unstable as null (never coerced to False).""" + feed_id = values["feed_id"] + + response = client.request( + "GET", + "/v1/feeds/{id}".format(id=feed_id), + headers=authHeaders, + ) + + assert response.status_code == 200 + json_response = response.json() + # Robust to either representation: key present with null, or key omitted. + assert json_response["source_info"].get("is_producer_url_unstable") is None, values["assert_fail_message"] + + def test_is_feed_reference_overwrite(client: TestClient): feed_id = "mdb-1562" response = client.request( diff --git a/api/tests/integration/test_data/sources_test.csv b/api/tests/integration/test_data/sources_test.csv index b6fc48f62..43e19e02c 100644 --- a/api/tests/integration/test_data/sources_test.csv +++ b/api/tests/integration/test_data/sources_test.csv @@ -1,6 +1,6 @@ -mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment -40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,, -50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment -702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,, -1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10, -1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10, \ No newline at end of file +mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable +40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,TRUE +50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,FALSE +702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,, +1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,FALSE +1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,TRUE \ No newline at end of file diff --git a/api/tests/unittest/models/test_basic_feed_impl.py b/api/tests/unittest/models/test_basic_feed_impl.py index d262f773e..7bed51fc1 100644 --- a/api/tests/unittest/models/test_basic_feed_impl.py +++ b/api/tests/unittest/models/test_basic_feed_impl.py @@ -31,6 +31,7 @@ feed_name="feed_name", note="note", producer_url="producer_url", + is_producer_url_unstable=True, authentication_type="1", authentication_info_url="authentication_info_url", api_key_parameter_name="api_key_parameter_name", @@ -98,6 +99,7 @@ related_links=[], source_info=SourceInfo( producer_url="producer_url", + is_producer_url_unstable=True, authentication_type=1, authentication_info_url="authentication_info_url", api_key_parameter_name="api_key_parameter_name", @@ -178,6 +180,7 @@ def test_to_orm_from_dict_full_payload(self): "provider": "Provider A", "feed_contact_email": "contact@example.com", "producer_url": "https://producer.example.com", + "is_producer_url_unstable": True, "authentication_type": 1, # should be converted to string "authentication_info_url": "https://auth.example.com", "api_key_parameter_name": "api_key", @@ -211,6 +214,7 @@ def test_to_orm_from_dict_full_payload(self): assert obj.provider == "Provider A" assert obj.feed_contact_email == "contact@example.com" assert obj.producer_url == "https://producer.example.com" + assert obj.is_producer_url_unstable is True # authentication_type coerced to string per implementation assert obj.authentication_type == "1" assert obj.authentication_info_url == "https://auth.example.com" diff --git a/api/tests/unittest/models/test_search_feed_item_result_impl.py b/api/tests/unittest/models/test_search_feed_item_result_impl.py index 1d5630056..ec1c2b9db 100644 --- a/api/tests/unittest/models/test_search_feed_item_result_impl.py +++ b/api/tests/unittest/models/test_search_feed_item_result_impl.py @@ -29,6 +29,7 @@ def __init__(self, **kwargs): note="note", feed_contact_email="feed_contact_email", producer_url="producer_url", + is_producer_url_unstable=True, authentication_info_url="authentication_info_url", authentication_type=1, api_key_parameter_name="api_key_parameter_name", @@ -83,6 +84,7 @@ def test_from_orm_gtfs(self): feed_contact_email=item.feed_contact_email, source_info=SourceInfo( producer_url=item.producer_url, + is_producer_url_unstable=item.is_producer_url_unstable, authentication_type=int(item.authentication_type) if item.authentication_type else None, authentication_info_url=item.authentication_info_url, api_key_parameter_name=item.api_key_parameter_name, @@ -129,6 +131,7 @@ def test_from_orm_gtfs_rt(self): feed_contact_email=item.feed_contact_email, source_info=SourceInfo( producer_url=item.producer_url, + is_producer_url_unstable=item.is_producer_url_unstable, authentication_type=int(item.authentication_type) if item.authentication_type else None, authentication_info_url=item.authentication_info_url, api_key_parameter_name=item.api_key_parameter_name, diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 537e23b2e..f016ba5e1 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -1147,11 +1147,22 @@ components: properties: producer_url: description: > - URL where the producer is providing the dataset. + URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. type: string format: url example: https://ladotbus.com/gtfs + is_producer_url_unstable: + description: > + Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. + This may be because the URL contains a date/time, or because the transit provider has + communicated that it is not permanent (e.g. it is updated monthly). + * true - The producer URL is unstable and changes over time. + * false - The producer URL is stable and unchanging over time. + * null (default) - There is not enough information to determine the stability of the producer URL. + type: boolean + nullable: true + example: true authentication_type: description: > Defines the type of authentication required to access the `producer_url`. Valid values for this field are: diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index bf53bfb36..538f2cef7 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -1337,6 +1337,19 @@ components: type: string format: url example: https://ladotbus.com/gtfs + is_producer_url_unstable: + description: > + Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. + This may be because the URL contains a date/time, or because the transit provider has + communicated that it is not permanent (e.g. it is updated monthly). + + * true - The producer URL is unstable and changes over time. + * false - The producer URL is stable and unchanging over time. + * null (default) - There is not enough information to determine the stability of the producer URL. + + type: boolean + nullable: true + example: true authentication_type: description: > Defines the type of authentication required to access the `producer_url`. Valid values for this field are: diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py index 5d2d36098..b7c2ad55d 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py @@ -52,6 +52,7 @@ def from_orm(cls, obj: Gtfsfeed | None) -> UpdateRequestGtfsFeed | None: feed_contact_email=obj.feed_contact_email, source_info=SourceInfo( producer_url=obj.producer_url, + is_producer_url_unstable=obj.is_producer_url_unstable, authentication_type=( None if obj.authentication_type is None @@ -96,6 +97,12 @@ def to_orm( ) else update_request.source_info.producer_url ) + # Nullable three-state boolean: preserve False and None (unknown), so guard only on source_info. + entity.is_producer_url_unstable = ( + None + if update_request.source_info is None + else update_request.source_info.is_producer_url_unstable + ) entity.authentication_type = ( None if ( diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py index 782e129c4..f276d8404 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py @@ -58,6 +58,7 @@ def from_orm(cls, obj: Gtfsrealtimefeed | None) -> UpdateRequestGtfsRtFeed | Non feed_contact_email=obj.feed_contact_email, source_info=SourceInfo( producer_url=obj.producer_url, + is_producer_url_unstable=obj.is_producer_url_unstable, authentication_type=( None if obj.authentication_type is None @@ -108,6 +109,12 @@ def to_orm( ) else update_request.source_info.producer_url ) + # Nullable three-state boolean: preserve False and None (unknown), so guard only on source_info. + entity.is_producer_url_unstable = ( + None + if update_request.source_info is None + else update_request.source_info.is_producer_url_unstable + ) entity.authentication_type = ( None if ( diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py index 965729c1d..9e5ad714e 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py @@ -22,6 +22,7 @@ def test_from_orm(): note="note", feed_contact_email="email@example.com", producer_url="http://producer.url", + is_producer_url_unstable=True, authentication_type=1, authentication_info_url="http://auth.info.url", api_key_parameter_name="api_key", @@ -38,6 +39,7 @@ def test_from_orm(): assert result.note == "note" assert result.feed_contact_email == "email@example.com" assert result.source_info.producer_url == "http://producer.url" + assert result.source_info.is_producer_url_unstable is True assert result.source_info.authentication_type == 1 assert result.source_info.authentication_info_url == "http://auth.info.url" assert result.source_info.api_key_parameter_name == "api_key" @@ -63,6 +65,7 @@ def test_to_orm(): feed_contact_email="email@example.com", source_info=SourceInfo( producer_url="http://producer.url", + is_producer_url_unstable=True, authentication_type=1, authentication_info_url="http://auth.info.url", api_key_parameter_name="api_key", @@ -85,6 +88,7 @@ def test_to_orm(): assert result.note == "note" assert result.feed_contact_email == "email@example.com" assert result.producer_url == "http://producer.url" + assert result.is_producer_url_unstable is True assert result.authentication_type == "1" assert result.authentication_info_url == "http://auth.info.url" assert result.api_key_parameter_name == "api_key" @@ -114,6 +118,7 @@ def test_to_orm_invalid_source_info(): result = UpdateRequestGtfsFeedImpl.to_orm(update_request, entity, session) assert result.producer_url is None + assert result.is_producer_url_unstable is None assert result.authentication_type is None assert result.authentication_info_url is None assert result.api_key_parameter_name is None diff --git a/liquibase/changelog.xml b/liquibase/changelog.xml index 13822937d..ae02dc518 100644 --- a/liquibase/changelog.xml +++ b/liquibase/changelog.xml @@ -119,6 +119,8 @@ + + diff --git a/liquibase/changes/feat_1766.sql b/liquibase/changes/feat_1766.sql new file mode 100644 index 000000000..fbb06a804 --- /dev/null +++ b/liquibase/changes/feat_1766.sql @@ -0,0 +1,5 @@ +-- Issue #1766: Add is_producer_url_unstable to the feed table. +-- Marks whether a feed's producer URL is known to be unstable/non-permanent +-- (e.g. it contains a date/time or the transit provider updates it periodically). +-- Nullable three-state boolean: TRUE = unstable, FALSE = stable, NULL = unknown (default). +ALTER TABLE feed ADD COLUMN IF NOT EXISTS is_producer_url_unstable BOOLEAN; diff --git a/liquibase/materialized_views/feed_search.sql b/liquibase/materialized_views/feed_search.sql index f59df5d18..81f058981 100644 --- a/liquibase/materialized_views/feed_search.sql +++ b/liquibase/materialized_views/feed_search.sql @@ -11,6 +11,7 @@ SELECT -- source Feed.producer_url, + Feed.is_producer_url_unstable, Feed.authentication_info_url, Feed.authentication_type, Feed.api_key_parameter_name,