diff --git a/functions-python/gtfs_datasets_comparer/README.md b/functions-python/gtfs_datasets_comparer/README.md index cb7f46293..45c140165 100644 --- a/functions-python/gtfs_datasets_comparer/README.md +++ b/functions-python/gtfs_datasets_comparer/README.md @@ -9,11 +9,12 @@ The function reads pre-extracted GTFS files from a GCS-mounted bucket (uploaded The function receives the following request: ``` { - "feed_stable_id": str, – stable_id of the GTFS feed - "base_dataset_stable_id": str, – stable_id of the base (older) dataset - "new_dataset_stable_id": str, – stable_id of the new (recent) dataset - "disallow_overwrite": bool (optional), – skip if changelog already exists (default: false) - "dry_run": bool (optional) – compute diff but skip GCS upload and DB write (default: false) + "feed_stable_id": str, – stable_id of the GTFS feed + "base_dataset_stable_id": str, – stable_id of the base (older) dataset + "new_dataset_stable_id": str, – stable_id of the new (recent) dataset + "disallow_overwrite": bool (optional), – skip if changelog already exists (default: false) + "dry_run": bool (optional), – compute diff but skip GCS upload and DB write (default: false) + "row_changes_cap_per_file": int (optional) – max row-level changes included per file; 0 omits row-level detail entirely; omit for no cap (default: 10000) } ``` @@ -44,6 +45,9 @@ By default the function will overwrite an existing changelog for the same datase ### `dry_run` When `dry_run: true`, the diff is computed and a summary is returned in the response, but nothing is written to GCS or the database. Useful for validating that the extracted files are present and the diff engine runs correctly. +### `row_changes_cap_per_file` +Limits the number of row-level change entries included in the changelog per GTFS file. Useful for controlling output size on large feeds. Set to `0` to omit all row-level detail (only file-level stats are included). Omit the parameter (or set to `null`) for no cap. Defaults to `10000`. + ## Response Success: diff --git a/functions-python/gtfs_datasets_comparer/src/main.py b/functions-python/gtfs_datasets_comparer/src/main.py index 902b759ae..9b8bbb036 100644 --- a/functions-python/gtfs_datasets_comparer/src/main.py +++ b/functions-python/gtfs_datasets_comparer/src/main.py @@ -52,6 +52,8 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict: new_dataset_stable_id – stable_id of the new Gtfsdataset disallow_overwrite – (optional, default false) skip if changelog already exists dry_run – (optional, default false) compute diff but skip GCS upload and DB write + row_changes_cap_per_file – (optional, default 10000) max row-level changes to include per file; + 0 omits row-level detail entirely; null means no cap Always returns HTTP 200 — errors are reported in the response body. This prevents GCP from retrying failures: we cannot distinguish transient from @@ -64,6 +66,20 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict: new_dataset_stable_id = payload.get("new_dataset_stable_id") disallow_overwrite = bool(payload.get("disallow_overwrite", False)) dry_run = bool(payload.get("dry_run", False)) + raw_cap = payload.get("row_changes_cap_per_file", 10000) + if raw_cap is None: + row_changes_cap_per_file = None + else: + try: + row_changes_cap_per_file = int(raw_cap) + except (ValueError, TypeError): + return flask.make_response( + { + "status": "error", + "error": f"row_changes_cap_per_file must be an integer, got: {raw_cap!r}", + }, + 200, + ) if not (feed_stable_id and base_dataset_stable_id and new_dataset_stable_id): return flask.make_response( @@ -94,6 +110,7 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict: bucket_mount=bucket_mount, disallow_overwrite=disallow_overwrite, dry_run=dry_run, + row_changes_cap_per_file=row_changes_cap_per_file, ) result = tracker.run() return flask.make_response({"status": "success", **result}, 200) @@ -139,6 +156,7 @@ def __init__( bucket_mount: str, disallow_overwrite: bool = False, dry_run: bool = False, + row_changes_cap_per_file: int | None = 10000, ): self.feed_stable_id = feed_stable_id self.base_dataset_stable_id = base_dataset_stable_id @@ -147,6 +165,7 @@ def __init__( self.bucket_mount = bucket_mount self.disallow_overwrite = disallow_overwrite self.dry_run = dry_run + self.row_changes_cap_per_file = row_changes_cap_per_file self.logger = get_logger(GtfsDatasetsComparer.__name__, new_dataset_stable_id) @track_metrics(metrics=("time", "memory", "cpu")) @@ -179,7 +198,9 @@ def run(self) -> dict: base_dir = self._extracted_dir(self.feed_stable_id, self.base_dataset_stable_id) new_dir = self._extracted_dir(self.feed_stable_id, self.new_dataset_stable_id) - diff_result = diff_feeds(base_dir, new_dir) + diff_result = diff_feeds( + base_dir, new_dir, row_changes_cap_per_file=self.row_changes_cap_per_file + ) if self.dry_run: self.logger.info("Dry run — skipping GCS upload and DB write.") diff --git a/functions-python/gtfs_datasets_comparer/tests/test_main.py b/functions-python/gtfs_datasets_comparer/tests/test_main.py index e7fe4b7ee..501e05440 100644 --- a/functions-python/gtfs_datasets_comparer/tests/test_main.py +++ b/functions-python/gtfs_datasets_comparer/tests/test_main.py @@ -100,6 +100,7 @@ def test_success(self, mock_tracker_cls): bucket_mount="/mobilitydata-datasets", disallow_overwrite=False, dry_run=False, + row_changes_cap_per_file=10000, ) @patch("main.GtfsDatasetsComparer") @@ -127,6 +128,7 @@ def test_disallow_overwrite_and_dry_run_passed(self, mock_tracker_cls): bucket_mount="/mobilitydata-datasets", disallow_overwrite=True, dry_run=True, + row_changes_cap_per_file=10000, ) @patch("main.GtfsDatasetsComparer") @@ -148,6 +150,21 @@ def test_exception_returns_200_with_error(self, mock_tracker_cls): self.assertEqual(body["status"], "error") self.assertIn("something went wrong", body["error"]) + def test_invalid_row_changes_cap_returns_error(self): + """Non-numeric row_changes_cap_per_file returns a clear validation error.""" + os.environ["DATASETS_BUCKET_NAME"] = "test-bucket" + status, body = self._request( + { + "feed_stable_id": "mdb-1", + "base_dataset_stable_id": "mdb-1-20240101", + "new_dataset_stable_id": "mdb-1-20240201", + "row_changes_cap_per_file": "not-a-number", + } + ) + self.assertEqual(status, 200) + self.assertEqual(body["status"], "error") + self.assertIn("row_changes_cap_per_file", body["error"]) + class TestGtfsDatasetsComparerRun(unittest.TestCase): """Tests for GtfsDatasetsComparer.run() with mocked collaborators."""