From 86a272a5a8c5de9efb5fd4d4254820ae20971f8e Mon Sep 17 00:00:00 2001 From: Matt Menefee Date: Tue, 17 Mar 2026 13:46:14 -0400 Subject: [PATCH] Add Elasticsearch 9.x support Elasticsearch 9.x introduced breaking changes to the elasticsearch-ruby client gem that prevent Chewy from working with ES 9 clusters: 1. The scroll API now requires scroll_id nested under a body: key ({scroll: '1m', body: {scroll_id: '...'}}) rather than as a top-level parameter. Without this, scroll requests raise an ArgumentError in elasticsearch-ruby 9.x. 2. API responses are now wrapped in Elasticsearch::API::Response objects instead of returning raw hashes/arrays. Code that calls .map or other Enumerable methods directly on responses (e.g. cat.indices) must now call .body first to access the underlying data. 3. The elasticsearch Ruby gem at version 9.x sends a compatible-with=9 header that ES 8.x servers reject. CI matrix entries that test against ES 8.x must pin the gem to ~> 8.14. Changes: - Widen elasticsearch gem dependency from < 9.0 to < 10.0 - Restructure perform_scroll to nest scroll_id under body:, which is compatible with both elasticsearch-ruby 8.x and 9.x - Access .body on cat.indices response in the drop_indices test helper - Add elasticsearch version matrix to CI: 9.4.3 default with three 8.19.3 spot-check entries (Ruby 3.3/3.4/4.0), using dynamic service container images - Pin elasticsearch gem to ~> 8.14 in CI for ES 8.x matrix entries - Add scroll body format assertion in scrolling specs - Add a Chewy 8->9 migration guide section with code examples, gem pinning guidance, and breaking-change notes - Add Chewy 9.0.x (ES 8.x + 9.x) row to the README compatibility table --- .github/workflows/ruby.yml | 21 +++++++++++++-- CHANGELOG.md | 3 +++ README.md | 1 + chewy.gemspec | 2 +- docker-compose.yml | 2 +- lib/chewy/search/scrolling.rb | 7 ++--- migration_guide.md | 40 +++++++++++++++++++++++++++++ spec/chewy/runtime_spec.rb | 2 +- spec/chewy/search/scrolling_spec.rb | 8 ++++-- spec/spec_helper.rb | 5 ++-- 10 files changed, 79 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 8cecf5f74..926507fc0 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -27,7 +27,21 @@ jobs: matrix: ruby: [ '3.2', '3.3', '3.4', '4.0' ] gemfile: [rails.7.2.activerecord, rails.8.0.activerecord, rails.8.1.activerecord] - name: ${{ matrix.ruby }}-${{ matrix.gemfile }} + elasticsearch: ['9.4.3'] + include: + - ruby: '3.3' + gemfile: rails.7.2.activerecord + elasticsearch: '8.19.3' + es_gem_constraint: '~> 8.14' + - ruby: '3.4' + gemfile: rails.8.0.activerecord + elasticsearch: '8.19.3' + es_gem_constraint: '~> 8.14' + - ruby: '4.0' + gemfile: rails.8.1.activerecord + elasticsearch: '8.19.3' + es_gem_constraint: '~> 8.14' + name: ${{ matrix.ruby }}-${{ matrix.gemfile }}-es${{ matrix.elasticsearch }} env: BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile @@ -45,7 +59,7 @@ jobs: --health-timeout 5s --health-retries 5 elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.13 + image: docker.elastic.co/elasticsearch/elasticsearch:${{ matrix.elasticsearch }} ports: - '9250:9200' env: @@ -61,6 +75,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Pin elasticsearch gem for ES 8.x + if: matrix.es_gem_constraint + run: echo "gem 'elasticsearch', '${{ matrix.es_gem_constraint }}'" >> $BUNDLE_GEMFILE - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 with: ruby-version: ${{ matrix.ruby }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 85a38efa8..2f2f0a2df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ ### Changes +* [#1009](https://github.com/toptal/chewy/pull/1009): **(Breaking)** Add Elasticsearch 9.x support while retaining ES 8.x compatibility. The `elasticsearch` gem dependency now allows `>= 8.14, < 10.0`. ([@mattmenefee][]) + * The `search_query.chewy` notification payload for scroll requests now nests `scroll_id` under `body:` (i.e. `{scroll: '1m', body: {scroll_id: '...'}}` instead of `{scroll: '1m', scroll_id: '...'}`). Update any application code that subscribes to scroll notification payloads. + ## 8.4.1 (2026-06-19) ### New Features diff --git a/README.md b/README.md index 3686eb7dd..215f7afb1 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Chewy is compatible with MRI 3.2-4.0. | Chewy version | Elasticsearch version | | ------------- | ---------------------------------- | +| 9.0.x | 8.x, 9.x | | 8.0.x | 8.x | | 7.2.x | 7.x | | 7.1.x | 7.x | diff --git a/chewy.gemspec b/chewy.gemspec index 57aba4036..bf5385485 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activesupport', '>= 7.2' - spec.add_dependency 'elasticsearch', '>= 8.14', '< 9.0' + spec.add_dependency 'elasticsearch', '>= 8.14', '< 10.0' spec.add_dependency 'elasticsearch-dsl' spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/docker-compose.yml b/docker-compose.yml index 690cc53c8..64403c79b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch_test: - image: "docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION:-8.19.13}" + image: "docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION:-9.4.3}" environment: - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false} - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}" diff --git a/lib/chewy/search/scrolling.rb b/lib/chewy/search/scrolling.rb index de505d115..d906eddc3 100644 --- a/lib/chewy/search/scrolling.rb +++ b/lib/chewy/search/scrolling.rb @@ -136,9 +136,10 @@ def scroll_objects(**options, &block) private - def perform_scroll(body) - ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: body) do - Chewy.client.scroll(body) + def perform_scroll(scroll:, scroll_id:) + request = {scroll: scroll, body: {scroll_id: scroll_id}} + ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: request) do + Chewy.client.scroll(request) end end end diff --git a/migration_guide.md b/migration_guide.md index 5f53359e0..91951b0ed 100644 --- a/migration_guide.md +++ b/migration_guide.md @@ -4,6 +4,46 @@ This document outlines the steps you need to take when migrating between major v Chewy and Elasticsearch. For simplicity's sake the guide will assume that you're using Chewy alongside a matching Elasticsearch version. +## Chewy 8/Elasticsearch 8 to Chewy 9/Elasticsearch 9 + +The Chewy 8 to 9 upgrade is relatively straightforward compared to previous major versions. +The primary changes concern the `elasticsearch` Ruby gem version and the scroll notification +payload format; no index mapping or query DSL changes are required. + +In order to upgrade Chewy 8/Elasticsearch 8 to Chewy 9/Elasticsearch 9 in the most seamless manner you have to: + +* Upgrade to the latest 8.x stable releases (Chewy 8.0.1, Elasticsearch 8.19.x). Your ES cluster must be on 8.19.x before upgrading to 9.x. +* Study carefully [Breaking changes in 9.0](https://www.elastic.co/docs/release-notes/elasticsearch/breaking-changes), make sure your application conforms. Notable changes that may affect Chewy users: + * Stricter bulk API request parsing — malformed action lines that were previously tolerated are now rejected. + * The legacy `_knn_search` endpoint has been removed — use the `knn` clause in `_search` instead. + * Date histogram aggregations on boolean fields are no longer supported. + * Elasticsearch now returns HTTP 429 (instead of 5xx) for certain timeout conditions — update any retry logic that keys on 5xx status codes. +* Update the `elasticsearch` gem — Chewy 9 requires `elasticsearch >= 8.14`. + * **Important:** The `elasticsearch` Ruby gem at version 9.x sends a `compatible-with=9` HTTP header that Elasticsearch 8.x servers reject with HTTP 400. Keep the gem pinned to `~> 8.14` (i.e. gem version 8.x) until after your Elasticsearch cluster has been upgraded to 9.x. You may optionally upgrade the gem to version 9.x after the cluster upgrade is complete. +* Update scroll notification subscribers: + * The `search_query.chewy` notification payload for scroll requests now nests `scroll_id` under `body:`: + * Old format: `{scroll: '1m', scroll_id: '...'}` + * New format: `{scroll: '1m', body: {scroll_id: '...'}}` + * Update any application code that subscribes to scroll notification payloads: + + ```ruby + # Before (Chewy < 9) + ActiveSupport::Notifications.subscribe('search_query.chewy') do |*, payload| + scroll_id = payload[:request][:scroll_id] + end + + # After (Chewy 9+) + ActiveSupport::Notifications.subscribe('search_query.chewy') do |*, payload| + scroll_id = payload[:request].dig(:body, :scroll_id) + end + ``` + +* Run your test suite on Chewy 9 / Elasticsearch 9. +* Run manual tests on Chewy 9 / Elasticsearch 9. +* Deploy Chewy 9 (with the `elasticsearch` gem still pinned to `~> 8.14`). +* Perform a [rolling upgrade](https://www.elastic.co/docs/deploy-manage/upgrade/deployment-or-cluster) of Elasticsearch to 9.x. +* (Optional) Remove the `elasticsearch` gem version pin, allowing the gem to resolve to 9.x. + ## Chewy 7/Elasticsearch 7 to Chewy 8/Elasticsearch 8 In order to upgrade Chewy 7/Elasticsearch 7 to Chewy 8/Elasticsearch 8 in the most seamless manner you have to: diff --git a/spec/chewy/runtime_spec.rb b/spec/chewy/runtime_spec.rb index e8cc457d3..782d01ba7 100644 --- a/spec/chewy/runtime_spec.rb +++ b/spec/chewy/runtime_spec.rb @@ -4,6 +4,6 @@ describe '.version' do specify { expect(described_class.version).to be_a(described_class::Version) } specify { expect(described_class.version).to be >= '8.0' } - specify { expect(described_class.version).to be < '9.0' } + specify { expect(described_class.version).to be < '10.0' } end end diff --git a/spec/chewy/search/scrolling_spec.rb b/spec/chewy/search/scrolling_spec.rb index ec9cefb7a..bd4eac297 100644 --- a/spec/chewy/search/scrolling_spec.rb +++ b/spec/chewy/search/scrolling_spec.rb @@ -84,7 +84,11 @@ end context do - before { expect(Chewy.client).to receive(:scroll).twice.and_call_original } + before do + expect(Chewy.client).to receive(:scroll) + .with(hash_including(scroll: '1m', body: hash_including(:scroll_id))) + .twice.and_call_original + end specify do expect(request.scroll_batches(batch_size: 2).map do |batch| batch.map { |hit| hit['_source']['rating'] } @@ -181,7 +185,7 @@ hash_including( index: [CitiesIndex, CountriesIndex], indexes: [CitiesIndex, CountriesIndex], - request: {scroll: '1m', scroll_id: an_instance_of(String)} + request: {scroll: '1m', body: {scroll_id: an_instance_of(String)}} ) ]) end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 69115031b..adfab6f58 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -49,11 +49,12 @@ # ) # Low-level substitute for now-obsolete drop_indices. -# Uses format: 'json' for version-portable parsing. +# Uses format: 'json' for version-portable parsing and `.body` to unwrap +# the Elasticsearch::API::Response (required for ES 9.x gem). # System indices (prefixed with '.') are excluded to avoid deleting # ES-internal indices like .security or .kibana. def drop_indices - indices = Chewy.client.cat.indices(format: 'json') + indices = Chewy.client.cat.indices(format: 'json').body .map { |entry| entry['index'] } .reject { |name| name.start_with?('.') } return if indices.blank?