From ca049fde8fd0a5f0a708d1b8e3e2f72dbb401000 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:14:30 +0000 Subject: [PATCH] BigQuery Loader 2.2.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump bqLoader to 2.2.1 and document configuration for the release. Common config (pre-existing gaps found by auditing reference.conf against the docs — unchanged since 2.2.0, so no `(since)` annotation): * retries.alterTableWait.delay * retries.tooManyColumns.delay * monitoring.healthProbe.port * monitoring.healthProbe.unhealthyLatency Pub/Sub config: remove `input.maxMessagesPerPull` and `input.debounceRequests`, and add an MDX comment covering those plus `input.parallelPullFactor`. Per the common-streams 0.24.2 `PubsubSourceConfig` scaladoc these options are "Only used when streamingPull is false", and BigQuery Loader uses the streaming-pull default (it does not override `streamingPull` in modules/pubsub application.conf), so they are no-ops for this loader. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../configuration-reference/_common_config.md | 22 +++++++++++++++++++ .../configuration-reference/_pubsub_config.md | 16 ++++---------- src/componentVersions.js | 2 +- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_common_config.md b/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_common_config.md index 1c4c84f83..fc03f5ebb 100644 --- a/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_common_config.md +++ b/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_common_config.md @@ -51,6 +51,20 @@ import Link from '@docusaurus/Link'; retries.transientErrors.attempts Optional. Default value 5. Maximum number of attempts to make before giving up on a transient error. + + retries.alterTableWait.delay + +

Optional. Default value 1 second.

+

Configures backoff when waiting for the BigQuery Writer to detect that the loader has altered the table by adding new columns.

+ + + + retries.tooManyColumns.delay + +

Optional. Default value 300 seconds.

+

Relevant when the BigQuery table is close to exceeding the limit on max allowed columns in a single table. The loader will ignore a failure to alter the table due to too many columns, and it will continue to run. Some events will inevitably go to the failed events output topic until new columns have been added. This parameter configures how often the loader will retry to alter the table after an earlier failure.

+ + skipSchemas @@ -135,6 +149,14 @@ import Link from '@docusaurus/Link'; monitoring.webhook.heartbeat.* Optional. Default value 5.minutes. How often to send a heartbeat event to the webhook when healthy. + + monitoring.healthProbe.port + Optional. Default value 8000. Open a HTTP server that returns OK only if the app is healthy. + + + monitoring.healthProbe.unhealthyLatency + Optional. Default value 5 minutes. Health probe becomes unhealthy if any received event is still not fully processed before this cutoff time. + monitoring.sentry.dsn Optional. Set to a Sentry URI to report unexpected runtime exceptions. diff --git a/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_pubsub_config.md b/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_pubsub_config.md index 7fe160968..78922df63 100644 --- a/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_pubsub_config.md +++ b/docs/api-reference/loaders-storage-targets/bigquery-loader/configuration-reference/_pubsub_config.md @@ -2,6 +2,10 @@ input.subscription Required, e.g. projects/myproject/subscriptions/snowplow-enriched. Name of the Pub/Sub subscription with the enriched events +{/* Do not document `input.parallelPullFactor`, `input.maxMessagesPerPull` or `input.debounceRequests` here. + Those options are only relevant for unary pull, and BigQuery Loader (as of 2.2.1) uses streaming pull + (the common-streams default), so they have no effect for this loader. + If the loader ever switches to unary pull, they become relevant and should be documented. */} input.durationPerAckExtension Optional. Default value 15 seconds. Pub/Sub ack deadlines are extended for this duration when needed. @@ -15,18 +19,6 @@ will wait until there is 1.5 seconds left of the remaining deadline, before re-extending the message deadline. - - input.maxMessagesPerPull - Optional. Default value 1000. How many Pub/Sub messages to pull from the server in a single request. - - - input.debounceRequests - - Optional. Default value 100 millis. - Adds an artifical delay between consecutive requests to Pub/Sub for more messages. - Under some circumstances, this was found to slightly alleviate a problem in which Pub/Sub might re-deliver the same messages multiple times. - - input.retries.transientErrors.delay (since 2.1.0) diff --git a/src/componentVersions.js b/src/componentVersions.js index 65c93fb0d..a84eb72b9 100644 --- a/src/componentVersions.js +++ b/src/componentVersions.js @@ -28,7 +28,7 @@ export const versions = { snowbridge: '5.1.1', // Loaders - bqLoader: '2.2.0', + bqLoader: '2.2.1', bqLoader1x: '1.7.2', esLoader: '3.0.1', gcsLoader: '0.5.7',