feat: Add row_changes_cap_per_file parameter to datasets comparer

functions-python/gtfs_datasets_comparer/README.md

@@ -9,11 +9,12 @@ The function reads pre-extracted GTFS files from a GCS-mounted bucket (uploaded
 The function receives the following request:
 ```
 {
-  "feed_stable_id": str,                  – stable_id of the GTFS feed
-  "base_dataset_stable_id": str,          – stable_id of the base (older) dataset
-  "new_dataset_stable_id": str,           – stable_id of the new (recent) dataset
-  "disallow_overwrite": bool (optional),  – skip if changelog already exists (default: false)
-  "dry_run": bool (optional)              – compute diff but skip GCS upload and DB write (default: false)
+  "feed_stable_id": str,                       – stable_id of the GTFS feed
+  "base_dataset_stable_id": str,               – stable_id of the base (older) dataset
+  "new_dataset_stable_id": str,                – stable_id of the new (recent) dataset
+  "disallow_overwrite": bool (optional),       – skip if changelog already exists (default: false)
+  "dry_run": bool (optional),                  – compute diff but skip GCS upload and DB write (default: false)
+  "row_changes_cap_per_file": int (optional)   – max row-level changes included per file; 0 omits row-level detail entirely; omit for no cap (default: 10000)
 }
 ```
 
@@ -44,6 +45,9 @@ By default the function will overwrite an existing changelog for the same datase
 ### `dry_run`
 When `dry_run: true`, the diff is computed and a summary is returned in the response, but nothing is written to GCS or the database. Useful for validating that the extracted files are present and the diff engine runs correctly.
 
+### `row_changes_cap_per_file`
+Limits the number of row-level change entries included in the changelog per GTFS file. Useful for controlling output size on large feeds. Set to `0` to omit all row-level detail (only file-level stats are included). Omit the parameter (or set to `null`) for no cap. Defaults to `10000`.
+
 ## Response
 
 Success:

functions-python/gtfs_datasets_comparer/src/main.py

@@ -52,6 +52,8 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict:
         new_dataset_stable_id     – stable_id of the new Gtfsdataset
         disallow_overwrite        – (optional, default false) skip if changelog already exists
         dry_run                   – (optional, default false) compute diff but skip GCS upload and DB write
+        row_changes_cap_per_file  – (optional, default 10000) max row-level changes to include per file;
+                                    0 omits row-level detail entirely; null means no cap
 
     Always returns HTTP 200 — errors are reported in the response body.
     This prevents GCP from retrying failures: we cannot distinguish transient from
@@ -64,6 +66,20 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict:
     new_dataset_stable_id = payload.get("new_dataset_stable_id")
     disallow_overwrite = bool(payload.get("disallow_overwrite", False))
     dry_run = bool(payload.get("dry_run", False))
+    raw_cap = payload.get("row_changes_cap_per_file", 10000)
+    if raw_cap is None:
+        row_changes_cap_per_file = None
+    else:
+        try:
+            row_changes_cap_per_file = int(raw_cap)
+        except (ValueError, TypeError):
+            return flask.make_response(
+                {
+                    "status": "error",
+                    "error": f"row_changes_cap_per_file must be an integer, got: {raw_cap!r}",
+                },
+                200,
+            )
 
     if not (feed_stable_id and base_dataset_stable_id and new_dataset_stable_id):
         return flask.make_response(
@@ -94,6 +110,7 @@ def gtfs_datasets_comparer(request: flask.Request) -> dict:
             bucket_mount=bucket_mount,
             disallow_overwrite=disallow_overwrite,
             dry_run=dry_run,
+            row_changes_cap_per_file=row_changes_cap_per_file,
         )
         result = tracker.run()
         return flask.make_response({"status": "success", **result}, 200)
@@ -139,6 +156,7 @@ def __init__(
         bucket_mount: str,
         disallow_overwrite: bool = False,
         dry_run: bool = False,
+        row_changes_cap_per_file: int | None = 10000,
     ):
         self.feed_stable_id = feed_stable_id
         self.base_dataset_stable_id = base_dataset_stable_id
@@ -147,6 +165,7 @@ def __init__(
         self.bucket_mount = bucket_mount
         self.disallow_overwrite = disallow_overwrite
         self.dry_run = dry_run
+        self.row_changes_cap_per_file = row_changes_cap_per_file
         self.logger = get_logger(GtfsDatasetsComparer.__name__, new_dataset_stable_id)
 
     @track_metrics(metrics=("time", "memory", "cpu"))
@@ -179,7 +198,9 @@ def run(self) -> dict:
         base_dir = self._extracted_dir(self.feed_stable_id, self.base_dataset_stable_id)
         new_dir = self._extracted_dir(self.feed_stable_id, self.new_dataset_stable_id)
 
-        diff_result = diff_feeds(base_dir, new_dir)
+        diff_result = diff_feeds(
+            base_dir, new_dir, row_changes_cap_per_file=self.row_changes_cap_per_file
+        )
 
         if self.dry_run:
             self.logger.info("Dry run — skipping GCS upload and DB write.")

functions-python/gtfs_datasets_comparer/tests/test_main.py

@@ -100,6 +100,7 @@ def test_success(self, mock_tracker_cls):
             bucket_mount="/mobilitydata-datasets",
             disallow_overwrite=False,
             dry_run=False,
+            row_changes_cap_per_file=10000,
         )
 
     @patch("main.GtfsDatasetsComparer")
@@ -127,6 +128,7 @@ def test_disallow_overwrite_and_dry_run_passed(self, mock_tracker_cls):
             bucket_mount="/mobilitydata-datasets",
             disallow_overwrite=True,
             dry_run=True,
+            row_changes_cap_per_file=10000,
         )
 
     @patch("main.GtfsDatasetsComparer")
@@ -148,6 +150,21 @@ def test_exception_returns_200_with_error(self, mock_tracker_cls):
         self.assertEqual(body["status"], "error")
         self.assertIn("something went wrong", body["error"])
 
+    def test_invalid_row_changes_cap_returns_error(self):
+        """Non-numeric row_changes_cap_per_file returns a clear validation error."""
+        os.environ["DATASETS_BUCKET_NAME"] = "test-bucket"
+        status, body = self._request(
+            {
+                "feed_stable_id": "mdb-1",
+                "base_dataset_stable_id": "mdb-1-20240101",
+                "new_dataset_stable_id": "mdb-1-20240201",
+                "row_changes_cap_per_file": "not-a-number",
+            }
+        )
+        self.assertEqual(status, 200)
+        self.assertEqual(body["status"], "error")
+        self.assertIn("row_changes_cap_per_file", body["error"])
+
 
 class TestGtfsDatasetsComparerRun(unittest.TestCase):
     """Tests for GtfsDatasetsComparer.run() with mocked collaborators."""