From 989751dfe5d61fd5e31bd8bc32f4ba103618917e Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 20 Oct 2026 19:31:01 +0000 Subject: [PATCH 1/5] Update xgboost dependency, execute in CI --- .../beam_PostCommit_Python_Dependency.json | 2 +- .../beam_PreCommit_Python_ML.json | 2 +- CHANGES.md | 1 + .../ml/inference/xgboost_inference_test.py | 4 +++- .../inference/xgboost_tests_requirements.txt | 4 ++-- sdks/python/setup.py | 24 ++++++++++++++----- .../python/test-suites/tox/py311/build.gradle | 9 +++++++ sdks/python/tox.ini | 12 ++++------ 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json b/.github/trigger_files/beam_PostCommit_Python_Dependency.json index 16209484727a..9bc6afbb0969 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json +++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 4 + "modification": 5 } diff --git a/.github/trigger_files/beam_PreCommit_Python_ML.json b/.github/trigger_files/beam_PreCommit_Python_ML.json index 7df0105a8112..2b9b913580a2 100644 --- a/.github/trigger_files/beam_PreCommit_Python_ML.json +++ b/.github/trigger_files/beam_PreCommit_Python_ML.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "revision": 22 + "revision": 23 } diff --git a/CHANGES.md b/CHANGES.md index d06940d12cf4..8d10689c6ac2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -93,6 +93,7 @@ * (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, matching the `group=0` behavior ([#40399](https://github.com/apache/beam/pull/40399)). * (Python) `Regex.all_matches` now returns every group of the regex, instead of failing with a `TypeError` when the regex has no groups and dropping nested or unmatched trailing groups ([#40562](https://github.com/apache/beam/issues/40562)). * (Python) `--dataflow_endpoint` now defaults to `''` instead of `https://dataflow.googleapis.com`. When unset, the Google Cloud client library resolves the Dataflow endpoint and honors `GOOGLE_CLOUD_UNIVERSE_DOMAIN`, so Dataflow jobs can be submitted in non-default universes. Code that reads `GoogleCloudOptions.dataflow_endpoint` should handle an empty value ([#40569](https://github.com/apache/beam/issues/40569)). +* (Python) The `xgboost` extras now install `xgboost>=2.0.0,<4` and `datatables>=1.1.0<2`, moving both the lower and upper bounds for both. The previous minimum `xgboost` and `datatables` versions (`1.6.0` and `1.0.0` respectively) are no longer allowed ([X]()). ## Deprecations diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference_test.py b/sdks/python/apache_beam/ml/inference/xgboost_inference_test.py index 8e3fb77615cc..e686ac5e6f25 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference_test.py @@ -69,7 +69,9 @@ def _compare_keyed_prediction_result( def predict_fn(self, data): - self.inference_calls += 1 + # Models loaded from disk inside the pipeline do not carry the counter set by + # build_monkeypatched_xgboost_classifier(). + self.inference_calls = getattr(self, 'inference_calls', 0) + 1 if isinstance(data, pandas.DataFrame): data = data.to_numpy() if isinstance(data, datatable.Frame): diff --git a/sdks/python/apache_beam/ml/inference/xgboost_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/xgboost_tests_requirements.txt index e85f4131701b..b1936b4149e4 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_tests_requirements.txt +++ b/sdks/python/apache_beam/ml/inference/xgboost_tests_requirements.txt @@ -15,5 +15,5 @@ # limitations under the License. # -datatable==1.0.0 -xgboost>=1.6.0 \ No newline at end of file +datatable>=1.1.0,<2 +xgboost>=2.0.0,<4 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a983db5395ed..ddad0e678dbf 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -166,6 +166,19 @@ def cythonize(*args, **kwargs): milvus_dependency = ['pymilvus>=2.5.10,<3.0.0'] qdrant_dependency = ['qdrant-client>=1.15.0'] +# NOTE: When updating these bounds, also update the bounds in +# apache_beam/ml/inference/xgboost_tests_requirements.txt, which is used by the +# XGBoost RunInference integration tests, and the py311-xgboost-* environments +# in tox.ini. +xgboost_dependency = [ + # xgboost<2.0 wheels fail `pip check` on recent pip ("not supported on this + # platform"), see https://github.com/apache/beam/issues/31285. + 'xgboost>=2.0.0,<4', + # apache_beam.ml.inference.xgboost_inference imports datatable directly. + # 1.1.0 is the first release with Python 3.10+ wheels; no release has + # wheels for Python 3.13+. + 'datatable>=1.1.0,<2', +] # google-adk / OpenTelemetry require protobuf>=5; tensorflow-transform in # ml_test is pinned to versions that require protobuf<5 on Python 3.10. Those @@ -608,13 +621,12 @@ def get_portability_package_data(): 'p310_ml_test': [ 'datatable', ] + ml_base + qdrant_dependency, - 'p311_ml_test': [ - 'datatable', - ] + ml_base + qdrant_dependency, + # xgboost_dependency includes datatable, which only has wheels for + # Python <=3.12, so it is limited to the 3.11 and 3.12 test extras. + 'p311_ml_test': ml_base + xgboost_dependency + qdrant_dependency, 'p312_ml_test': [ - 'datatable', 'typesafe-sdk>=0.7.1,<0.8', - ] + ml_base + qdrant_dependency, + ] + ml_base + xgboost_dependency + qdrant_dependency, # maintainer: milvus tests only run with this extension. Make sure it # is covered by docker-in-docker test when changing py version 'p313_ml_test': ml_base_core + milvus_dependency + qdrant_dependency, @@ -682,7 +694,7 @@ def get_portability_package_data(): 'transformers==4.25.1', # tensorflow transient dep 'absl-py>=0.12.0' ], - 'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'], + 'xgboost': xgboost_dependency, 'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'], 'milvus': milvus_dependency, 'qdrant': qdrant_dependency, diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index a2b0f0d5e235..433b0052b08b 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -158,6 +158,15 @@ toxTask "testPy311transformers-latest", "py311-transformers-latest", "${posargs} test.dependsOn "testPy311transformers-latest" postCommitPyDep.dependsOn "testPy311transformers-latest" +// Create a test task for each major version of xgboost +toxTask "testPy311xgboost-200", "py311-xgboost-200", "${posargs}" +test.dependsOn "testPy311xgboost-200" +postCommitPyDep.dependsOn "testPy311xgboost-200" + +toxTask "testPy311xgboost-300", "py311-xgboost-300", "${posargs}" +test.dependsOn "testPy311xgboost-300" +postCommitPyDep.dependsOn "testPy311xgboost-300" + toxTask "testPy311embeddingsMLTransform", "py311-embeddings", "${posargs}" test.dependsOn "testPy311embeddingsMLTransform" postCommitPyDep.dependsOn "testPy311embeddingsMLTransform" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index c88e5e2fea6a..a1bc7ba3322d 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -489,14 +489,12 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py311-xgboost-{160,170}] +[testenv:py311-xgboost-{200,300}] +# Keep these bounds in line with xgboost_dependency in setup.py. deps = - 160: - xgboost>=1.6.0,<1.7.0 - datatable==1.0.0 - 170: - xgboost>=1.7.0 - datatable==1.0.0 + 200: xgboost>=2.0.0,<3.0.0 + 300: xgboost>=3.0.0,<4 + datatable>=1.1.0,<2 extras = test,gcp commands = # Log XGBoost version for debugging From e4dd145f9cfc4f80193a5804f8951c827b599eba Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 20 Oct 2026 19:36:02 +0000 Subject: [PATCH 2/5] Apply correct link to entry in CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8d10689c6ac2..357c0dd46e3d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -93,7 +93,7 @@ * (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, matching the `group=0` behavior ([#40399](https://github.com/apache/beam/pull/40399)). * (Python) `Regex.all_matches` now returns every group of the regex, instead of failing with a `TypeError` when the regex has no groups and dropping nested or unmatched trailing groups ([#40562](https://github.com/apache/beam/issues/40562)). * (Python) `--dataflow_endpoint` now defaults to `''` instead of `https://dataflow.googleapis.com`. When unset, the Google Cloud client library resolves the Dataflow endpoint and honors `GOOGLE_CLOUD_UNIVERSE_DOMAIN`, so Dataflow jobs can be submitted in non-default universes. Code that reads `GoogleCloudOptions.dataflow_endpoint` should handle an empty value ([#40569](https://github.com/apache/beam/issues/40569)). -* (Python) The `xgboost` extras now install `xgboost>=2.0.0,<4` and `datatables>=1.1.0<2`, moving both the lower and upper bounds for both. The previous minimum `xgboost` and `datatables` versions (`1.6.0` and `1.0.0` respectively) are no longer allowed ([X]()). +* (Python) The `xgboost` extras now install `xgboost>=2.0.0,<4` and `datatables>=1.1.0<2`, moving both the lower and upper bounds for both. The previous minimum `xgboost` and `datatables` versions (`1.6.0` and `1.0.0` respectively) are no longer allowed ([#40694](https://github.com/apache/beam/issues/40694)). ## Deprecations From e5d68349e8cdb47417ef6b523bcea35a6e635166 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 20 Oct 2026 19:40:03 +0000 Subject: [PATCH 3/5] Fix typos --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 357c0dd46e3d..ef125f22678f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -93,7 +93,7 @@ * (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, matching the `group=0` behavior ([#40399](https://github.com/apache/beam/pull/40399)). * (Python) `Regex.all_matches` now returns every group of the regex, instead of failing with a `TypeError` when the regex has no groups and dropping nested or unmatched trailing groups ([#40562](https://github.com/apache/beam/issues/40562)). * (Python) `--dataflow_endpoint` now defaults to `''` instead of `https://dataflow.googleapis.com`. When unset, the Google Cloud client library resolves the Dataflow endpoint and honors `GOOGLE_CLOUD_UNIVERSE_DOMAIN`, so Dataflow jobs can be submitted in non-default universes. Code that reads `GoogleCloudOptions.dataflow_endpoint` should handle an empty value ([#40569](https://github.com/apache/beam/issues/40569)). -* (Python) The `xgboost` extras now install `xgboost>=2.0.0,<4` and `datatables>=1.1.0<2`, moving both the lower and upper bounds for both. The previous minimum `xgboost` and `datatables` versions (`1.6.0` and `1.0.0` respectively) are no longer allowed ([#40694](https://github.com/apache/beam/issues/40694)). +* (Python) The `xgboost` extras now install `xgboost>=2.0.0,<4` and `datatable>=1.1.0,<2`, moving both the lower and upper bounds for both. The previous minimum `xgboost` and `datatable` versions (`1.6.0` and `1.0.0` respectively) are no longer allowed ([#40694](https://github.com/apache/beam/issues/40694)). ## Deprecations From c081dda59755e4a3c312b546f19310fd1a448d87 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 21 Oct 2026 14:10:17 +0000 Subject: [PATCH 4/5] Comment cleanup --- sdks/python/apache_beam/ml/inference/xgboost_inference.py | 2 +- sdks/python/setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference.py b/sdks/python/apache_beam/ml/inference/xgboost_inference.py index f11b892062e8..82215b00edfb 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference.py @@ -119,7 +119,7 @@ def __init__( before loading the model. **Supported Versions:** RunInference APIs in Apache Beam have been tested - with XGBoost 1.6.0 and 1.7.0 + with XGBoost 2.x and 3.x XGBoost 1.0.0 introduced support for using JSON to save and load XGBoost models. XGBoost 1.6.0, additional support for Universal Binary JSON. diff --git a/sdks/python/setup.py b/sdks/python/setup.py index ddad0e678dbf..3279282fc6b9 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -171,8 +171,6 @@ def cythonize(*args, **kwargs): # XGBoost RunInference integration tests, and the py311-xgboost-* environments # in tox.ini. xgboost_dependency = [ - # xgboost<2.0 wheels fail `pip check` on recent pip ("not supported on this - # platform"), see https://github.com/apache/beam/issues/31285. 'xgboost>=2.0.0,<4', # apache_beam.ml.inference.xgboost_inference imports datatable directly. # 1.1.0 is the first release with Python 3.10+ wheels; no release has From d7e44b1de85601c6c4929838c588d65321c1a2d2 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 21 Oct 2026 15:18:32 +0000 Subject: [PATCH 5/5] Mark datatable handler as deprecated --- CHANGES.md | 1 + sdks/python/apache_beam/ml/inference/xgboost_inference.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ef125f22678f..389c656c512e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -98,6 +98,7 @@ ## Deprecations * Support for Python 3.10 was removed following its end-of-life ([#40250](https://github.com/apache/beam/issues/40250)). +* (Python) `XGBoostModelHandlerDatatable` is deprecated and will be removed in Beam 2.79.0, along with the `datatable` dependency of the `xgboost` extra. `datatable` has not had a release since 2023 and has no wheels for Python 3.13+. Use `XGBoostModelHandlerPandas` (or the NumPy/SciPy handlers) instead ([#40705](https://github.com/apache/beam/issues/40705)). * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). ## Bugfixes diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference.py b/sdks/python/apache_beam/ml/inference/xgboost_inference.py index 82215b00edfb..b27db8b2046b 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference.py @@ -36,6 +36,7 @@ from apache_beam.ml.inference.base import ModelT from apache_beam.ml.inference.base import PredictionResult from apache_beam.ml.inference.base import PredictionT +from apache_beam.utils.annotations import deprecated __all__ = [ 'XGBoostModelHandler', @@ -313,6 +314,12 @@ def get_num_bytes(self, batch: Sequence[scipy.sparse.csr_matrix]) -> int: return sum(sys.getsizeof(element) for element in batch) +@deprecated( + since='2.78.0', + extra_message=( + 'datatable has not had a release since 2023 and has no wheels for ' + 'Python 3.13+. XGBoostModelHandlerDatatable and the datatable ' + 'dependency will be removed in Beam 2.79.0.')) class XGBoostModelHandlerDatatable(XGBoostModelHandler[datatable.Frame, PredictionResult, Union[xgboost.Booster,