feat: read BigLake Lakehouse tables through tables.get instead of pyiceberg

Parse 4-part Lakehouse table IDs (project.catalog.namespace.table) with
TableReference.from_string() and load them through tables.get, the same
as regular tables, instead of the pyiceberg REST catalog workaround.
4-part support in from_string() requires google-cloud-bigquery>=3.42.0.

The BigQuery Storage Read API accepts snapshot_time for Lakehouse tables
but returns current data, while SQL FOR SYSTEM_TIME AS OF is honored.
Reads pinned to a snapshot time are therefore left to the SQL path so a
DataFrame keeps returning the same data after the table changes. Bug filed with the BigQuery team - b/568786565.

System tests will be added in a follow-up change.

BUG=567983672

Change-Id: I7f1b23d584e0e3e3dc29f1ff63e8654870c00b0e
Reviewed-on: https://bigframes-internal-review.git.corp.google.com/c/bigframes/+/4520
Reviewed-by: Tim Swena <swast@google.com>
Kokoro-Docs: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Kokoro-E2E: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Kokoro-Lint: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Kokoro-Doctest: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Kokoro-Unit: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Kokoro-Prerelease: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>
Reviewed-by: Shenyang Cai <sycai@google.com>
Kokoro-System: Kokoro <noreply+kokoro-dedicatedkokoro-dedicated Kokorogoogle.com>

Author: sindhuvy

.pre-commit-config.yaml

@@ -61,7 +61,7 @@ repos:
           - geopandas>=0.12.2
           - google-cloud-bigquery-connection>=1.18.2
           - google-cloud-bigquery-storage>=2.30.0,<3.0.0
-          - google-cloud-bigquery[bqstorage,pandas]>=3.36.0
+          - google-cloud-bigquery[bqstorage,pandas]>=3.42.0
           - google-cloud-functions>=1.20.2
           - google-cloud-resource-manager>=1.14.2
           - google-cloud-storage>=2.0.0

bigframes/core/bq_data.py

@@ -17,6 +17,7 @@
 import concurrent.futures
 import dataclasses
 import datetime
+import enum
 import functools
 import os
 import queue
@@ -103,6 +104,22 @@ class TableMetadata:
     modified_time: Optional[datetime.datetime] = None
 
 
+class TableKind(enum.Enum):
+    """Where a table comes from.
+
+    This is separate from TableMetadata.type, which is the BigQuery table type,
+    such as TABLE or VIEW.
+    """
+
+    # A table or view in a BigQuery dataset: project.dataset.table.
+    NATIVE = enum.auto()
+    # A BigLake Lakehouse table: project.catalog.namespace.table.
+    LAKEHOUSE = enum.auto()
+    # An INFORMATION_SCHEMA view, such as
+    # project.region-us.INFORMATION_SCHEMA.SCHEMATA.
+    INFORMATION_SCHEMA = enum.auto()
+
+
 @dataclasses.dataclass(frozen=True)
 class GbqNativeTable:
     project_id: str = dataclasses.field()
@@ -170,6 +187,17 @@ def from_ref_and_schema(
     def is_physically_stored(self) -> bool:
         return self.metadata.type in ["TABLE", "MATERIALIZED_VIEW"]
 
+    @property
+    def kind(self) -> TableKind:
+        # INFORMATION_SCHEMA views are loaded with INFORMATION_SCHEMA as the
+        # dataset ID.
+        if self.dataset_id.casefold() == "INFORMATION_SCHEMA".casefold():
+            return TableKind.INFORMATION_SCHEMA
+        # Lakehouse tables have catalog.namespace in place of a dataset ID.
+        if "." in self.dataset_id:
+            return TableKind.LAKEHOUSE
+        return TableKind.NATIVE
+
     def get_table_ref(self) -> bq.TableReference:
         return bq.TableReference(
             bq.DatasetReference(self.project_id, self.dataset_id), self.table_id

bigframes/core/pyformat.py

@@ -115,7 +115,10 @@ def is_biglake(
             node: nodes.BigFrameNode, child_results: Tuple[bool, ...]
         ) -> bool:
             if isinstance(node, nodes.ReadTableNode):
-                return isinstance(node.source.table, bq_data.BiglakeIcebergTable)
+                return isinstance(node.source.table, bq_data.BiglakeIcebergTable) or (
+                    isinstance(node.source.table, bq_data.GbqNativeTable)
+                    and node.source.table.kind == bq_data.TableKind.LAKEHOUSE
+                )
             return any(child_results)
 
         contains_biglake = value._block.expr.node.reduce_up(is_biglake)

bigframes/pandas/io/api.py

@@ -58,9 +58,7 @@
 import bigframes.session
 import bigframes.session._io.bigquery
 import bigframes.session.clients
-import bigframes.session.iceberg
 import bigframes.session.metrics
-from bigframes.core import bq_data
 from bigframes.session import dry_runs
 
 # Note: the following methods are duplicated from Session. This duplication
@@ -728,13 +726,6 @@ def _set_default_session_location_if_possible_deferred_query(create_query):
                 default_project=default_project,
             )
             config.options.bigquery.location = table.location
-        elif bq_data.is_irc_table(query):
-            irc_table = bigframes.session.iceberg.get_table(
-                default_project, query, bqclient._credentials
-            )
-            config.options.bigquery.location = bq_data.get_default_bq_region(
-                irc_table.metadata.location
-            )
         else:
             table = bqclient.get_table(query)
             config.options.bigquery.location = table.location

bigframes/session/loader.py

@@ -73,7 +73,6 @@
 import bigframes.session._io.bigquery as bf_io_bigquery
 import bigframes.session._io.bigquery.read_gbq_query as bf_read_gbq_query
 import bigframes.session._io.bigquery.read_gbq_table as bf_read_gbq_table
-import bigframes.session.iceberg
 import bigframes.session.metrics
 import bigframes.session.temporary_storage
 import bigframes.session.time as session_time
@@ -1084,10 +1083,6 @@ def _get_table_metadata(
                 default_project=default_project,
             )
             table = bq_data.GbqNativeTable.from_table(client_table)
-        elif bq_data.is_irc_table(table_id):
-            table = bigframes.session.iceberg.get_table(
-                self._bqclient.project, table_id, self._bqclient._credentials
-            )
         else:
             table_ref = google.cloud.bigquery.table.TableReference.from_string(
                 table_id, default_project=default_project

bigframes/session/read_api_execution.py

@@ -55,6 +55,15 @@ async def execute(
         if not node.source.table.is_physically_stored:
             return None
 
+        # TODO(b/568786565): Remove this branch once the Read API honors
+        # snapshot_time for Lakehouse tables. Until then, time-travel reads of
+        # these tables must use SQL, since the Read API returns current data.
+        if (
+            node.source.at_time is not None
+            and node.source.table.kind == bq_data.TableKind.LAKEHOUSE
+        ):
+            return None
+
         peek = execution_spec.peek
         if limit is not None:
             if peek is None or limit < peek:

setup.py

@@ -39,7 +39,7 @@
     "gcsfs >=2023.3.0, !=2025.5.0, !=2026.2.0, !=2026.3.0",
     "geopandas >=0.12.2",
     "google-auth[pyopenssl] >=2.15.0,<3.0",
-    "google-cloud-bigquery[bqstorage,pandas] >=3.36.0",
+    "google-cloud-bigquery[bqstorage,pandas] >=3.42.0",
     # 2.30 needed for arrow support.
     "google-cloud-bigquery-storage >= 2.30.0, < 3.0.0",
     "google-cloud-functions >=1.20.2",

testing/constraints-3.10.txt

@@ -6,7 +6,7 @@ geopandas==0.12.2
 google-auth==2.15.0
 google-cloud-bigtable==2.30.0
 google-cloud-pubsub==2.29.0
-google-cloud-bigquery==3.36.0
+google-cloud-bigquery==3.42.0
 google-cloud-functions==1.20.2
 google-cloud-bigquery-connection==1.18.2
 google-cloud-iam==2.18.2

testing/constraints-3.11.txt

@@ -152,7 +152,7 @@ google-auth==2.38.0
 google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.2.2
 google-cloud-aiplatform==1.106.0
-google-cloud-bigquery==3.36.0
+google-cloud-bigquery==3.42.0
 google-cloud-bigquery-connection==1.18.3
 google-cloud-bigquery-storage==2.32.0
 google-cloud-core==2.4.3

tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_lakehouse_w_system_time/out.sql

@@ -0,0 +1,3 @@
+SELECT
+  *
+FROM `bigframes-dev`.`my_catalog.my_namespace`.`scalar_types` AS `bft_0` FOR SYSTEM_TIME AS OF '2025-11-09T03:04:05.678901+00:00'