chore: scaffold Ibis 9.0.0 tiered modernization stubs and implementat…

data_validation/clients.py

@@ -28,11 +28,12 @@
 from data_validation import client_info, consts, exceptions
 from data_validation.secret_manager import SecretManagerBuilder
 
-from third_party.ibis.ibis_bigquery.api import bigquery_connect
+# TODO: Tier 1 Upgrade: Remove custom wrapper imports. Use native backends (ibis.bigquery.connect, ibis.postgres.connect, etc.) directly.
+# from third_party.ibis.ibis_bigquery.api import bigquery_connect
 from third_party.ibis.ibis_cloud_spanner.api import spanner_connect
-from third_party.ibis.ibis_impala.api import impala_connect
-from third_party.ibis.ibis_mssql.api import mssql_connect
-from third_party.ibis.ibis_redshift.api import redshift_connect
+# from third_party.ibis.ibis_impala.api import impala_connect
+# from third_party.ibis.ibis_mssql.api import mssql_connect
+# from third_party.ibis.ibis_redshift.api import redshift_connect
 
 if TYPE_CHECKING:
     import ibis.expr.schema as sch
@@ -168,12 +169,12 @@ def get_bigquery_client(
             quota_project_id=client_project_id,
         )
 
-    return bigquery_connect(
+    # print(f"Connecting to BigQuery with native Ibis: {project_id or client_project_id}, {dataset_id}")
+    return ibis.bigquery.connect(
         project_id=project_id or client_project_id,
         dataset_id=dataset_id,
-        credentials=credentials,
-        bigquery_client=google_client,
-        bqstorage_client=bqstorage_client,
+        client=google_client,
+        storage_client=bqstorage_client,
     )
 
 
@@ -195,6 +196,13 @@ def get_pandas_client(table_name, file_path, file_type):
     else:
         raise ValueError(f"Unknown Pandas File Type: {file_type}")
 
+    # TODO: Tier 1 Upgrade (Issue #1702): The ibis.pandas backend is deprecated/removed in Ibis 9.0+.
+    # We must migrate to the DuckDB backend to load and access local dataframes/files:
+    #
+    #     pandas_client = ibis.duckdb.connect()
+    #     pandas_client.register(df, name=table_name)
+    #
+    # This resolves both the deprecation and preserves catalog-based table lookup compatibility.
     pandas_client = ibis.pandas.connect({table_name: df})
 
     return pandas_client
@@ -418,15 +426,15 @@ def get_max_in_list_size(client, in_list_over_expressions=False):
 
 CLIENT_LOOKUP = {
     consts.SOURCE_TYPE_BIGQUERY: get_bigquery_client,
-    consts.SOURCE_TYPE_IMPALA: impala_connect,
+    consts.SOURCE_TYPE_IMPALA: ibis.impala.connect,
     consts.SOURCE_TYPE_MYSQL: ibis.mysql.connect,
-    consts.SOURCE_TYPE_ORACLE: oracle_connect,
+    consts.SOURCE_TYPE_ORACLE: ibis.oracle.connect,
     consts.SOURCE_TYPE_FILESYSTEM: get_pandas_client,
     consts.SOURCE_TYPE_POSTGRES: ibis.postgres.connect,
-    consts.SOURCE_TYPE_REDSHIFT: redshift_connect,
+    consts.SOURCE_TYPE_REDSHIFT: ibis.redshift.connect,
     consts.SOURCE_TYPE_TERADATA: teradata_connect,
-    consts.SOURCE_TYPE_MSSQL: mssql_connect,
-    consts.SOURCE_TYPE_SNOWFLAKE: snowflake_connect,
+    consts.SOURCE_TYPE_MSSQL: ibis.mssql.connect,
+    consts.SOURCE_TYPE_SNOWFLAKE: ibis.snowflake.connect,
     consts.SOURCE_TYPE_SPANNER: spanner_connect,
     consts.SOURCE_TYPE_SYBASE: sybase_connect,
     consts.SOURCE_TYPE_DB2: db2_connect,

setup.py

@@ -33,7 +33,8 @@
     "google-cloud-secret-manager>=2.20.2",
     "google-cloud-spanner>=3.49.1",
     "google-cloud-storage>=2.18.2",
-    "ibis-framework==5.1.0",  # Pinned to 5.1.0, significant work to bump to 7.1.0
+    # TODO: Tier 1 Modernization: Upgrade Ibis to 9.0.0 (minimum target)
+    "ibis-framework==9.0.0",
     "impyla>=0.19.0",
     "jellyfish>=1.1.0",
     "pandas",
@@ -85,12 +86,11 @@
         release_status,
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Operating System :: OS Independent",
     ],
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     install_requires=dependencies,
     extras_require=extras_require,
     entry_points={

third_party/ibis/ibis_addon/operations.py

@@ -477,6 +477,14 @@ def _sa_whitespace_rstrip(t, op):
 # so we can piggy back Ibis code rather than writing metadata queries for all engines.
 BaseAlchemyBackend.dvt_list_tables = _dvt_list_tables
 
+# TODO: Tier 1 Upgrade: In Ibis 9.0.0 (SQLGlot), older ExprTranslator._registry mappings must be migrated to
+# visit_<OperationName> methods dynamically injected onto SQLGlotCompiler and backend-specific compilers:
+#
+# from ibis.backends.sql.compilers.base import SQLGlotCompiler
+# def visit_RawSQL(self, op, *, arg, sql):
+#     return sqlglot.parse_one(sql.this)
+# SQLGlotCompiler.visit_RawSQL = visit_RawSQL
+
 BigQueryExprTranslator._registry[ops.HashBytes] = bigquery_registry.format_hashbytes
 BigQueryExprTranslator._registry[RawSQL] = format_raw_sql
 BigQueryExprTranslator._registry[ops.Strftime] = bigquery_registry.strftime

third_party/ibis/ibis_cloud_spanner/__init__.py

@@ -18,7 +18,7 @@
 import ibis.expr.schema as sch
 import ibis.expr.types as ir
 from google.cloud import spanner
-from ibis.backends.base.sql import BaseSQLBackend
+from ibis.backends.sql import SQLBackend
 
 from third_party.ibis.ibis_cloud_spanner.datatypes import (
     dtype_from_spanner_field,
@@ -29,7 +29,7 @@
 from third_party.ibis.ibis_cloud_spanner.to_pandas import pandas_df
 
 
-class Backend(BaseSQLBackend):
+class Backend(SQLBackend):
     name = "spanner"
     compiler = SpannerCompiler
 

third_party/ibis/ibis_cloud_spanner/compiler.py

@@ -14,9 +14,10 @@
 import re
 
 import ibis.expr.operations as ops
-from ibis.backends.base.sql import compiler as sql_compiler
-from ibis.backends.bigquery import compiler as bigquery_compiler
-from ibis.backends.bigquery import rewrites
+# TODO: Tier 1 Upgrade: Modernize SpannerCompiler to inherit from SQLGlotCompiler.
+# The older ExprTranslator and Select classes are removed.
+# Define visit_RStrip, visit_RawSQL, etc., as visit_<OperationName> methods directly on SpannerCompiler.
+from ibis.backends.sql.compiler import SQLGlotCompiler
 from third_party.ibis.ibis_cloud_spanner import registry
 
 _NAME_REGEX = re.compile(r"[_A-Za-z][A-Za-z_0-9]*")

third_party/ibis/ibis_db2/compiler.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from ibis.backends.base.sql.alchemy import (
-    AlchemyCompiler,
-    AlchemyExprTranslator,
-)
+# TODO: Tier 1 Upgrade: Modernize Db2Compiler to inherit from SQLGlotCompiler.
+# The older AlchemyCompiler and AlchemyExprTranslator classes are removed.
+# Define visit_RStrip, visit_RawSQL, etc. as visit_<OperationName> methods directly on Db2Compiler.
+from ibis.backends.sql.compiler import SQLGlotCompiler
 from third_party.ibis.ibis_db2.registry import operation_registry
 
 

third_party/ibis/ibis_sybase/compiler.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ibis.backends.base.sql.alchemy import AlchemyCompiler, AlchemyExprTranslator
-from ibis.backends.base.sql.alchemy.query_builder import AlchemySelect
+# TODO: Tier 1 Upgrade: Modernize SybaseCompiler to inherit from SQLGlotCompiler.
+# The older AlchemyCompiler, AlchemyExprTranslator and AlchemySelect classes are removed.
+# Define visit_RStrip, visit_RawSQL, etc. as visit_<OperationName> methods directly on SybaseCompiler.
+from ibis.backends.sql.compiler import SQLGlotCompiler
 from sqlalchemy_sybase import DATETIME
 from sqlalchemy_sybase.base import (
     SybaseSQLCompiler,

third_party/ibis/ibis_teradata/__init__.py

@@ -22,7 +22,7 @@
 import ibis.expr.types as ir
 from typing import Mapping, Any
 import ibis.expr.schema as sch
-from ibis.backends.base.sql import BaseSQLBackend
+from ibis.backends.sql import SQLBackend
 
 from third_party.ibis.ibis_teradata.compiler import TeradataCompiler
 from third_party.ibis.ibis_teradata.datatypes import (
@@ -31,7 +31,7 @@
 )
 
 
-class Backend(BaseSQLBackend):
+class Backend(SQLBackend):
     name = "teradata"
     compiler = TeradataCompiler
     NO_LOCK_SQL = "LOCKING ROW FOR ACCESS "

third_party/ibis/ibis_teradata/compiler.py

@@ -11,13 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from ibis.backends.base.sql.compiler import (
-    Compiler,
-    ExprTranslator,
-    TableSetFormatter,
-    Select,
-)
-from ibis.backends.base.sql.registry import identifiers
+# TODO: Tier 1 Upgrade: Modernize TeradataCompiler to inherit from SQLGlotCompiler.
+# The older ExprTranslator, TableSetFormatter and Select classes are removed.
+# Define visit_RStrip, visit_RawSQL, etc. as visit_<OperationName> methods directly on TeradataCompiler.
+from ibis.backends.sql.compiler import SQLGlotCompiler
 from third_party.ibis.ibis_teradata.registry import _operation_registry