From be832cdf2f3ab81efb16e1ef7175f5f531e152c8 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Sat, 16 Dec 2023 11:32:47 +0300 Subject: [PATCH 1/9] ES 8.x upgrade --- chewy.gemspec | 5 ++-- docker-compose.yml | 16 +++++++++++ lib/chewy/index/actions.rb | 10 +++---- lib/chewy/index/aliases.rb | 2 +- lib/chewy/index/syncer.rb | 2 +- lib/chewy/search/request.rb | 4 +-- spec/chewy/index/actions_spec.rb | 16 +++++------ spec/chewy/index/import/bulk_builder_spec.rb | 2 +- spec/chewy/index/import_spec.rb | 28 ++++++++++---------- spec/chewy/index/specification_spec.rb | 3 --- spec/chewy/runtime_spec.rb | 4 +-- spec/chewy/search/response_spec.rb | 2 +- spec/chewy_spec.rb | 2 +- 13 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 docker-compose.yml diff --git a/chewy.gemspec b/chewy.gemspec index 675081dde..482a4d9a1 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -11,13 +11,14 @@ Gem::Specification.new do |spec| spec.description = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL' spec.homepage = 'https://github.com/toptal/chewy' spec.license = 'MIT' + spec.required_ruby_version = '~> 3.0' spec.files = `git ls-files`.split($RS) spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x - spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8' + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'elasticsearch', '>= 8.11', '< 9.0' spec.add_dependency 'elasticsearch-dsl' spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..41042bf1d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.4" +services: + elasticsearch_test: + image: "elasticsearch:8.11.3" + environment: + - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false} + - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}" + - discovery.type=single-node + - xpack.security.enabled=false + - action.destructive_requires_name=false + ports: + - "127.0.0.1:9250:9200" + ulimits: + nofile: + soft: 65536 + hard: 65536 diff --git a/lib/chewy/index/actions.rb b/lib/chewy/index/actions.rb index afc7debcc..a146f47cc 100644 --- a/lib/chewy/index/actions.rb +++ b/lib/chewy/index/actions.rb @@ -32,7 +32,7 @@ def exists? # def create(*args, **kwargs) create!(*args, **kwargs) - rescue Elasticsearch::Transport::Transport::Errors::BadRequest + rescue Elastic::Transport::Transport::Errors::BadRequest false end @@ -83,9 +83,9 @@ def delete(suffix = nil) result = client.indices.delete index: index_names.join(',') Chewy.wait_for_status if result result - # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound + # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound # by itself, rescue is for previous versions - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound false end @@ -99,9 +99,9 @@ def delete(suffix = nil) # UsersIndex.delete '01-2014' # deletes `users_01-2014` index # def delete!(suffix = nil) - # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound + # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound # by itself, so it is raised here - delete(suffix) or raise Elasticsearch::Transport::Transport::Errors::NotFound + delete(suffix) or raise Elastic::Transport::Transport::Errors::NotFound end # Deletes and recreates index. Supports suffixes. diff --git a/lib/chewy/index/aliases.rb b/lib/chewy/index/aliases.rb index cfcf74732..9b4a2caef 100644 --- a/lib/chewy/index/aliases.rb +++ b/lib/chewy/index/aliases.rb @@ -22,7 +22,7 @@ def aliases def empty_if_not_found yield - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound [] end end diff --git a/lib/chewy/index/syncer.rb b/lib/chewy/index/syncer.rb index b8365bb50..47408c9af 100644 --- a/lib/chewy/index/syncer.rb +++ b/lib/chewy/index/syncer.rb @@ -213,7 +213,7 @@ def outdated_sync_field_type @outdated_sync_field_type = mappings .fetch('properties', {}) .fetch(@index.outdated_sync_field.to_s, {})['type'] - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound nil end end diff --git a/lib/chewy/search/request.rb b/lib/chewy/search/request.rb index f3b1031b4..ffb93cdb0 100644 --- a/lib/chewy/search/request.rb +++ b/lib/chewy/search/request.rb @@ -854,7 +854,7 @@ def count else Chewy.client.count(only(WHERE_STORAGES).render)['count'] end - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound 0 end @@ -1035,7 +1035,7 @@ def perform(additional = {}) request_body = render.merge(additional) ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: request_body) do Chewy.client.search(request_body) - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound {} end end diff --git a/spec/chewy/index/actions_spec.rb b/spec/chewy/index/actions_spec.rb index 4a0d69893..1f2456475 100644 --- a/spec/chewy/index/actions_spec.rb +++ b/spec/chewy/index/actions_spec.rb @@ -78,12 +78,12 @@ specify do expect do DummiesIndex.create! - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/) end specify do expect do DummiesIndex.create!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/) end end @@ -100,7 +100,7 @@ specify do expect do DummiesIndex.create!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/) end specify { expect(DummiesIndex.create!('2014')['acknowledged']).to eq(true) } @@ -190,11 +190,11 @@ end describe '.delete!' do - specify { expect { DummiesIndex.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) } + specify { expect { DummiesIndex.delete! }.to raise_error(Elastic::Transport::Transport::Errors::NotFound) } specify do expect do DummiesIndex.delete!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end.to raise_error(Elastic::Transport::Transport::Errors::NotFound) end context do @@ -768,7 +768,7 @@ .to receive(:clear_cache) .and_call_original expect { CitiesIndex.clear_cache({index: unexisted_index_name}) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end @@ -820,7 +820,7 @@ .to receive(:reindex) .and_call_original expect { CitiesIndex.reindex(source: unexisting_index, dest: dest_index_with_prefix) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end @@ -883,7 +883,7 @@ context 'index name' do specify do expect { CitiesIndex.update_mapping(unexisting_index, body_hash) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end diff --git a/spec/chewy/index/import/bulk_builder_spec.rb b/spec/chewy/index/import/bulk_builder_spec.rb index 1b2eb3a82..6ac4ab1df 100644 --- a/spec/chewy/index/import/bulk_builder_spec.rb +++ b/spec/chewy/index/import/bulk_builder_spec.rb @@ -216,7 +216,7 @@ def derived end def do_raw_index_comment(options:, data:) - CommentsIndex.client.index(options.merge(index: 'comments', type: '_doc', refresh: true, body: data)) + CommentsIndex.client.index(options.merge(index: 'comments', refresh: true, body: data)) end def raw_index_comment(comment) diff --git a/spec/chewy/index/import_spec.rb b/spec/chewy/index/import_spec.rb index 16593113a..60482a9fc 100644 --- a/spec/chewy/index/import_spec.rb +++ b/spec/chewy/index/import_spec.rb @@ -204,10 +204,10 @@ def subscribe_notification end end - let(:mapper_parsing_exception) do + let(:document_parsing_exception) do { - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [name] tried to parse field [name] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:9] object mapping for [name] tried to parse field [name] as object, but found a concrete value' } end @@ -215,7 +215,7 @@ def subscribe_notification payload = subscribe_notification import dummy_cities, batch_size: 2 expect(payload).to eq(index: CitiesIndex, - errors: {index: {mapper_parsing_exception => %w[1 2 3]}}, + errors: {index: {document_parsing_exception => %w[1 2 3]}}, import: {index: 3}) end end @@ -270,8 +270,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -293,8 +293,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -319,8 +319,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -383,8 +383,8 @@ def subscribe_notification # Full match doesn't work here. expect(payload[:errors][:update].keys).to match([ - hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][1]: document missing'), - hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][3]: document missing') + hash_including('type' => 'document_missing_exception', 'reason' => '[1]: document missing'), + hash_including('type' => 'document_missing_exception', 'reason' => '[3]: document missing') ]) expect(payload[:errors][:update].values).to eq([['1'], ['3']]) expect(imported_cities).to match_array([ @@ -431,8 +431,8 @@ def subscribe_notification expect(payload).to eq( errors: { update: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:26] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, diff --git a/spec/chewy/index/specification_spec.rb b/spec/chewy/index/specification_spec.rb index 68a34b8d5..92457301d 100644 --- a/spec/chewy/index/specification_spec.rb +++ b/spec/chewy/index/specification_spec.rb @@ -46,7 +46,6 @@ specify do expect { specification1.lock! }.to change { Chewy::Stash::Specification.all.hits }.from([]).to([{ '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'places', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ @@ -62,7 +61,6 @@ specify do expect { specification5.lock! }.to change { Chewy::Stash::Specification.all.hits }.to([{ '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'places', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ @@ -71,7 +69,6 @@ }.to_json)} }, { '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'namespace/cities', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ diff --git a/spec/chewy/runtime_spec.rb b/spec/chewy/runtime_spec.rb index edc85231b..e8cc457d3 100644 --- a/spec/chewy/runtime_spec.rb +++ b/spec/chewy/runtime_spec.rb @@ -3,7 +3,7 @@ describe Chewy::Runtime do describe '.version' do specify { expect(described_class.version).to be_a(described_class::Version) } - specify { expect(described_class.version).to be >= '7.0' } - specify { expect(described_class.version).to be < '8.0' } + specify { expect(described_class.version).to be >= '8.0' } + specify { expect(described_class.version).to be < '9.0' } end end diff --git a/spec/chewy/search/response_spec.rb b/spec/chewy/search/response_spec.rb index 7b291288d..2e8cd29d8 100644 --- a/spec/chewy/search/response_spec.rb +++ b/spec/chewy/search/response_spec.rb @@ -39,7 +39,7 @@ specify { expect(subject.hits).to all be_a(Hash) } specify do expect(subject.hits.flat_map(&:keys).uniq) - .to match_array(%w[_id _index _type _score _source sort]) + .to match_array(%w[_id _index _score _source sort]) end context do diff --git a/spec/chewy_spec.rb b/spec/chewy_spec.rb index 0c3e0d0ec..c4c72aa73 100644 --- a/spec/chewy_spec.rb +++ b/spec/chewy_spec.rb @@ -111,7 +111,7 @@ expect(CitiesIndex.exists?).to eq true expect(PlacesIndex.exists?).to eq true - expect { Chewy.create_indices! }.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest) + expect { Chewy.create_indices! }.to raise_error(Elastic::Transport::Transport::Errors::BadRequest) end end end From 9dcf571f667f77bb532fe18bf826a429d9497679 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 18 Dec 2023 09:58:16 +0300 Subject: [PATCH 2/9] Fix termnite after changes --- lib/chewy/search/request.rb | 4 ++-- lib/chewy/search/response.rb | 7 +++++++ lib/chewy/search/scrolling.rb | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/chewy/search/request.rb b/lib/chewy/search/request.rb index ffb93cdb0..27d5cdefe 100644 --- a/lib/chewy/search/request.rb +++ b/lib/chewy/search/request.rb @@ -46,7 +46,7 @@ class Request delegate :hits, :wrappers, :objects, :records, :documents, :object_hash, :record_hash, :document_hash, - :total, :max_score, :took, :timed_out?, to: :response + :total, :max_score, :took, :timed_out?, :terminated_early?, to: :response delegate :each, :size, :to_a, :[], to: :wrappers alias_method :to_ary, :to_a alias_method :total_count, :total @@ -891,7 +891,7 @@ def exists? def first(limit = UNDEFINED) request_limit = limit == UNDEFINED ? 1 : limit - if performed? && (request_limit <= size || size == total) + if performed? && (terminated_early? || request_limit <= size || size == total) limit == UNDEFINED ? wrappers.first : wrappers.first(limit) else result = except(EXTRA_STORAGES).limit(request_limit).to_a diff --git a/lib/chewy/search/response.rb b/lib/chewy/search/response.rb index 0a5becb24..5409a113d 100644 --- a/lib/chewy/search/response.rb +++ b/lib/chewy/search/response.rb @@ -47,6 +47,13 @@ def timed_out? @timed_out ||= @body['timed_out'] end + # Has the request been terminated early? + # + # @return [true, false] + def terminated_early? + @terminated_early ||= @body['terminated_early'] + end + # The `suggest` response part. Returns empty hash if suggests # were not requested. # diff --git a/lib/chewy/search/scrolling.rb b/lib/chewy/search/scrolling.rb index 6074b3252..f0c738374 100644 --- a/lib/chewy/search/scrolling.rb +++ b/lib/chewy/search/scrolling.rb @@ -39,7 +39,8 @@ def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEF hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total yield(hits) if hits.present? scroll_id = result['_scroll_id'] - break if fetched >= total + + break if result['terminated_early'] || fetched >= total result = perform_scroll(scroll: scroll, scroll_id: scroll_id) end From f3f9512d30e2a69959a1519a0a176c5c0d2bc071 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 18 Dec 2023 10:03:55 +0300 Subject: [PATCH 3/9] fixup es --- .github/workflows/ruby.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 825a8d4a8..62e3ee293 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -41,11 +41,11 @@ jobs: with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - name: Run Elasticsearch - uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759 - with: - stack-version: 7.10.1 - port: 9250 + - name: Start containers + run: | + docker compose up elasticsearch_test -d + sleep 10 + - name: Tests run: bundle exec rspec From 1f7b8f8cd9baebb6a6fe24ce6bafaac1487aa598 Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Mon, 19 Aug 2024 15:29:57 +0300 Subject: [PATCH 4/9] Elastic security on: configuration example --- spec/spec_helper.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index be66332f8..0e818798c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,6 +28,22 @@ } } +# To work with security enabled: +# +# user = ENV['ES_USER'] || 'elastic' +# password = ENV['ES_PASSWORD'] || '' +# ca_cert = ENV['ES_CA_CERT'] || './tmp/http_ca.crt' +# +# Chewy.settings.merge!( +# user: user, +# password: password, +# transport_options: { +# ssl: { +# ca_file: ca_cert +# } +# } +# ) + # Chewy.transport_logger = Logger.new(STDERR) RSpec.configure do |config| From 94e71988354acbcb3ee114874e277cfbc662c9bb Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Fri, 16 Aug 2024 18:30:07 +0300 Subject: [PATCH 5/9] Mass index deletion disabled by default in Elastic --- docker-compose.yml | 1 - lib/chewy.rb | 4 ++++ lib/chewy/errors.rb | 3 +++ lib/chewy/minitest/helpers.rb | 2 +- spec/chewy/elastic_client_spec.rb | 2 +- spec/chewy/fields/time_fields_spec.rb | 2 +- spec/chewy/index/actions_spec.rb | 2 +- spec/chewy/index/aliases_spec.rb | 2 +- spec/chewy/index/import/bulk_builder_spec.rb | 2 +- spec/chewy/index/import/bulk_request_spec.rb | 2 +- spec/chewy/index/import/routine_spec.rb | 2 +- spec/chewy/index/import_spec.rb | 2 +- spec/chewy/index/specification_spec.rb | 2 +- spec/chewy/index/syncer_spec.rb | 2 +- spec/chewy/index_spec.rb | 2 +- spec/chewy/journal_spec.rb | 4 ++-- spec/chewy/minitest/helpers_spec.rb | 2 +- spec/chewy/multi_search_spec.rb | 2 +- spec/chewy/rake_helper_spec.rb | 2 +- spec/chewy/rspec/update_index_spec.rb | 2 +- spec/chewy/search/loader_spec.rb | 2 +- spec/chewy/search/pagination/kaminari_examples.rb | 2 +- spec/chewy/search/request_spec.rb | 2 +- spec/chewy/search/response_spec.rb | 2 +- spec/chewy/search/scrolling_spec.rb | 2 +- spec/chewy/search_spec.rb | 2 +- spec/chewy/stash_spec.rb | 2 +- spec/chewy/strategy_spec.rb | 2 +- spec/chewy_spec.rb | 9 +++++---- spec/spec_helper.rb | 10 ++++++++++ 30 files changed, 48 insertions(+), 31 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 41042bf1d..3f77b8c1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,6 @@ services: - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}" - discovery.type=single-node - xpack.security.enabled=false - - action.destructive_requires_name=false ports: - "127.0.0.1:9250:9200" ulimits: diff --git a/lib/chewy.rb b/lib/chewy.rb index 919f9bac8..dfeed3416 100644 --- a/lib/chewy.rb +++ b/lib/chewy.rb @@ -116,6 +116,10 @@ def wait_for_status # Be careful, if current prefix is blank, this will destroy all the indexes. # def massacre + unless Chewy.settings[:delete_all_enabled] + raise FeatureDisabled, 'Feature disabled by default in ES 8. You can enable it in the cluster and set `delete_all_enabled` option in settings' + end + Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_')) Chewy.wait_for_status end diff --git a/lib/chewy/errors.rb b/lib/chewy/errors.rb index d470bd94f..e16848794 100644 --- a/lib/chewy/errors.rb +++ b/lib/chewy/errors.rb @@ -39,4 +39,7 @@ def initialize(join_field_type, join_field_name, relations) class ImportScopeCleanupError < Error end + + class FeatureDisabled < Error + end end diff --git a/lib/chewy/minitest/helpers.rb b/lib/chewy/minitest/helpers.rb index e9fbd1362..0254c8f97 100644 --- a/lib/chewy/minitest/helpers.rb +++ b/lib/chewy/minitest/helpers.rb @@ -142,7 +142,7 @@ def index_everything! teardown do # always destroy indexes between tests # Prevent croll pollution of test cases due to indexing - Chewy.massacre + drop_indices end end end diff --git a/spec/chewy/elastic_client_spec.rb b/spec/chewy/elastic_client_spec.rb index 79d3946ba..58cdc0cc7 100644 --- a/spec/chewy/elastic_client_spec.rb +++ b/spec/chewy/elastic_client_spec.rb @@ -6,7 +6,7 @@ let!(:filter_previous_value) { Chewy.before_es_request_filter } before do - Chewy.massacre + drop_indices stub_index(:products) do field :id, type: :integer end diff --git a/spec/chewy/fields/time_fields_spec.rb b/spec/chewy/fields/time_fields_spec.rb index 31aef9777..a9ddc8c45 100644 --- a/spec/chewy/fields/time_fields_spec.rb +++ b/spec/chewy/fields/time_fields_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe 'Time fields' do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:posts) do diff --git a/spec/chewy/index/actions_spec.rb b/spec/chewy/index/actions_spec.rb index 1f2456475..27d5682b4 100644 --- a/spec/chewy/index/actions_spec.rb +++ b/spec/chewy/index/actions_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Actions do - before { Chewy.massacre } + before { drop_indices } before do stub_index :dummies diff --git a/spec/chewy/index/aliases_spec.rb b/spec/chewy/index/aliases_spec.rb index dc29e1189..ad5419c7d 100644 --- a/spec/chewy/index/aliases_spec.rb +++ b/spec/chewy/index/aliases_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Aliases do - before { Chewy.massacre } + before { drop_indices } before { stub_index :dummies } diff --git a/spec/chewy/index/import/bulk_builder_spec.rb b/spec/chewy/index/import/bulk_builder_spec.rb index 6ac4ab1df..5fe598bed 100644 --- a/spec/chewy/index/import/bulk_builder_spec.rb +++ b/spec/chewy/index/import/bulk_builder_spec.rb @@ -17,7 +17,7 @@ def derived end describe Chewy::Index::Import::BulkBuilder do - before { Chewy.massacre } + before { drop_indices } subject { described_class.new(index, to_index: to_index, delete: delete, fields: fields) } let(:index) { CitiesIndex } diff --git a/spec/chewy/index/import/bulk_request_spec.rb b/spec/chewy/index/import/bulk_request_spec.rb index 0869804a1..bf33d2b16 100644 --- a/spec/chewy/index/import/bulk_request_spec.rb +++ b/spec/chewy/index/import/bulk_request_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Import::BulkRequest do - before { Chewy.massacre } + before { drop_indices } subject { described_class.new(index, suffix: suffix, bulk_size: bulk_size, **bulk_options) } let(:suffix) {} diff --git a/spec/chewy/index/import/routine_spec.rb b/spec/chewy/index/import/routine_spec.rb index 5d1064b7b..55588e314 100644 --- a/spec/chewy/index/import/routine_spec.rb +++ b/spec/chewy/index/import/routine_spec.rb @@ -2,7 +2,7 @@ # TODO: add more specs here later describe Chewy::Index::Import::Routine do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:cities) do field :name diff --git a/spec/chewy/index/import_spec.rb b/spec/chewy/index/import_spec.rb index 60482a9fc..b200a4599 100644 --- a/spec/chewy/index/import_spec.rb +++ b/spec/chewy/index/import_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Import do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/index/specification_spec.rb b/spec/chewy/index/specification_spec.rb index 92457301d..cd1b3bd7b 100644 --- a/spec/chewy/index/specification_spec.rb +++ b/spec/chewy/index/specification_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Specification do - before { Chewy.massacre } + before { drop_indices } let(:index1) do stub_index(:places) do diff --git a/spec/chewy/index/syncer_spec.rb b/spec/chewy/index/syncer_spec.rb index e71617f04..176cf047a 100644 --- a/spec/chewy/index/syncer_spec.rb +++ b/spec/chewy/index/syncer_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Syncer, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) stub_index(:cities) do diff --git a/spec/chewy/index_spec.rb b/spec/chewy/index_spec.rb index 57565861b..d96d1bf6f 100644 --- a/spec/chewy/index_spec.rb +++ b/spec/chewy/index_spec.rb @@ -177,7 +177,7 @@ def self.by_id; end context do before do - Chewy.massacre + drop_indices PlacesIndex.import!( double(colors: ['red']), double(colors: %w[red green]), diff --git a/spec/chewy/journal_spec.rb b/spec/chewy/journal_spec.rb index 3d518be09..e392f8aba 100644 --- a/spec/chewy/journal_spec.rb +++ b/spec/chewy/journal_spec.rb @@ -21,7 +21,7 @@ default_import_options journal: true end - Chewy.massacre + drop_indices Chewy.settings[:prefix] = 'some_prefix' Timecop.freeze(time) end @@ -145,7 +145,7 @@ def timestamp(time) end context do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) do update_index 'cities', :self diff --git a/spec/chewy/minitest/helpers_spec.rb b/spec/chewy/minitest/helpers_spec.rb index 4d6f2d30e..c411a14e4 100644 --- a/spec/chewy/minitest/helpers_spec.rb +++ b/spec/chewy/minitest/helpers_spec.rb @@ -17,7 +17,7 @@ def assert_equal(expected, actual, message) end before do - Chewy.massacre + drop_indices end before do diff --git a/spec/chewy/multi_search_spec.rb b/spec/chewy/multi_search_spec.rb index 8ca1e2cd2..2e7afaefb 100644 --- a/spec/chewy/multi_search_spec.rb +++ b/spec/chewy/multi_search_spec.rb @@ -2,7 +2,7 @@ require 'chewy/multi_search' describe Chewy::MultiSearch do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index f84ad129a..353164217 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -2,7 +2,7 @@ require 'rake' describe Chewy::RakeHelper, :orm do - before { Chewy.massacre } + before { drop_indices } before do described_class.instance_variable_set(:@journal_exists, journal_exists) diff --git a/spec/chewy/rspec/update_index_spec.rb b/spec/chewy/rspec/update_index_spec.rb index 5982f4c51..8054bcce6 100644 --- a/spec/chewy/rspec/update_index_spec.rb +++ b/spec/chewy/rspec/update_index_spec.rb @@ -9,7 +9,7 @@ end before do - Chewy.massacre + drop_indices DummiesIndex.create! end diff --git a/spec/chewy/search/loader_spec.rb b/spec/chewy/search/loader_spec.rb index 1bfde065a..31dfff1b5 100644 --- a/spec/chewy/search/loader_spec.rb +++ b/spec/chewy/search/loader_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Loader do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/search/pagination/kaminari_examples.rb b/spec/chewy/search/pagination/kaminari_examples.rb index 08a285716..aa86c9700 100644 --- a/spec/chewy/search/pagination/kaminari_examples.rb +++ b/spec/chewy/search/pagination/kaminari_examples.rb @@ -1,7 +1,7 @@ require 'spec_helper' shared_examples :kaminari do |request_base_class| - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) do diff --git a/spec/chewy/search/request_spec.rb b/spec/chewy/search/request_spec.rb index 414de49a7..b73b99161 100644 --- a/spec/chewy/search/request_spec.rb +++ b/spec/chewy/search/request_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Request do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) do diff --git a/spec/chewy/search/response_spec.rb b/spec/chewy/search/response_spec.rb index 2e8cd29d8..3cf5830d2 100644 --- a/spec/chewy/search/response_spec.rb +++ b/spec/chewy/search/response_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Response, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/search/scrolling_spec.rb b/spec/chewy/search/scrolling_spec.rb index 4dfe68941..003d899c7 100644 --- a/spec/chewy/search/scrolling_spec.rb +++ b/spec/chewy/search/scrolling_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Scrolling, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/search_spec.rb b/spec/chewy/search_spec.rb index d7cafa40d..b8ead283e 100644 --- a/spec/chewy/search_spec.rb +++ b/spec/chewy/search_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) diff --git a/spec/chewy/stash_spec.rb b/spec/chewy/stash_spec.rb index 2d2774ff5..fc0df7487 100644 --- a/spec/chewy/stash_spec.rb +++ b/spec/chewy/stash_spec.rb @@ -5,7 +5,7 @@ def fetch_deleted_number(response) response['deleted'] || response['_indices']['_all']['deleted'] end - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/strategy_spec.rb b/spec/chewy/strategy_spec.rb index 25a2344bd..817e2dfc3 100644 --- a/spec/chewy/strategy_spec.rb +++ b/spec/chewy/strategy_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Strategy do - before { Chewy.massacre } + before { drop_indices } subject(:strategy) { Chewy::Strategy.new } describe '#current' do diff --git a/spec/chewy_spec.rb b/spec/chewy_spec.rb index c4c72aa73..e946aecca 100644 --- a/spec/chewy_spec.rb +++ b/spec/chewy_spec.rb @@ -31,8 +31,8 @@ end end - describe '.massacre' do - before { Chewy.massacre } + xdescribe '.massacre' do + before { drop_indices } before do allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1')) @@ -40,7 +40,7 @@ allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix2')) stub_index(:developers).create! - Chewy.massacre + drop_indices allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1')) end @@ -84,7 +84,8 @@ # To avoid flaky issues when previous specs were run allow(Chewy::Index).to receive(:descendants).and_return([CitiesIndex, PlacesIndex]) - Chewy.massacre + CitiesIndex.delete + PlacesIndex.delete end specify do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0e818798c..c67ff0f88 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -44,6 +44,16 @@ # } # ) +# Low-level substitute for now-obsolete drop_indices +def drop_indices + response = Chewy.client.cat.indices + indices = response.body.lines.map { |line| line.split[2] } + return if indices.blank? + + Chewy.client.indices.delete(index: indices) + Chewy.wait_for_status +end + # Chewy.transport_logger = Logger.new(STDERR) RSpec.configure do |config| From c9774a092c2331ce4a7ba9900cbec50416c8b571 Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Mon, 19 Aug 2024 14:19:22 +0300 Subject: [PATCH 6/9] Elastic docker file: increase sleep, bump image --- .github/workflows/ruby.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 62e3ee293..c9293a9aa 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -44,7 +44,7 @@ jobs: - name: Start containers run: | docker compose up elasticsearch_test -d - sleep 10 + sleep 15 - name: Tests run: bundle exec rspec diff --git a/docker-compose.yml b/docker-compose.yml index 3f77b8c1b..0c1309de2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.4" services: elasticsearch_test: - image: "elasticsearch:8.11.3" + image: "elasticsearch:8.15.0" environment: - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false} - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}" From d8fd9aacfb48c5cbdac5d21add74a039dcf36495 Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Mon, 19 Aug 2024 13:38:22 +0300 Subject: [PATCH 7/9] Spec fixes --- lib/chewy/config.rb | 4 ++-- spec/chewy/config_spec.rb | 4 ++-- spec/chewy/strategy/delayed_sidekiq_spec.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index 1ec687e1b..fdae4ae45 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -70,12 +70,12 @@ def initialize end def transport_logger=(logger) - Chewy.client.transport.transport.logger = logger + Chewy.client.transport.logger = logger @transport_logger = logger end def transport_tracer=(tracer) - Chewy.client.transport.transport.tracer = tracer + Chewy.client.transport.tracer = tracer @transport_tracer = tracer end diff --git a/spec/chewy/config_spec.rb b/spec/chewy/config_spec.rb index 1d0321d80..f9d311342 100644 --- a/spec/chewy/config_spec.rb +++ b/spec/chewy/config_spec.rb @@ -22,7 +22,7 @@ specify do expect { subject.transport_logger = logger } - .to change { Chewy.client.transport.transport.logger }.to(logger) + .to change { Chewy.client.transport.logger }.to(logger) end specify do expect { subject.transport_logger = logger } @@ -40,7 +40,7 @@ specify do expect { subject.transport_tracer = tracer } - .to change { Chewy.client.transport.transport.tracer }.to(tracer) + .to change { Chewy.client.transport.tracer }.to(tracer) end specify do expect { subject.transport_tracer = tracer } diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index 78a1219b8..a4a8fdd09 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -134,7 +134,7 @@ def expected_at_time context 'both calls with different update fields' do it 'deos reindex with union of fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: %w[name description]).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: match_array(%w[name description])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) From d2129a8cf35f07cdcb651a669e130fd9417b3aab Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Mon, 19 Aug 2024 13:48:52 +0300 Subject: [PATCH 8/9] bump dependency --- chewy.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chewy.gemspec b/chewy.gemspec index 482a4d9a1..decbe7355 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activesupport', '>= 6.1' - spec.add_dependency 'elasticsearch', '>= 8.11', '< 9.0' + spec.add_dependency 'elasticsearch', '>= 8.14', '< 9.0' spec.add_dependency 'elasticsearch-dsl' spec.metadata['rubygems_mfa_required'] = 'true' end From 169fdb992622483fa379fe9c6bccb82116299da0 Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Tue, 27 Aug 2024 15:09:57 +0300 Subject: [PATCH 9/9] Bump version, update docs --- CHANGELOG.md | 12 ++++++++++++ README.md | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25291ca22..0c27c85fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ ### Bugs Fixed +## 8.0.0-beta (2024-08-27) + +### New Features + +* [#962](https://github.com/toptal/chewy/pull/962): ElasticSearch v.8 support added + +* `delete_all_enabled` setting introduced to align Chewy.massacre with wildcard indices deletion disabled in ES 8 by default + +### Changes + +### Bugs Fixed + ## 7.6.0 (2024-05-03) ### Changes diff --git a/README.md b/README.md index 23c13993e..da00cef6c 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Chewy is compatible with MRI 3.0-3.2ยน. | Chewy version | Elasticsearch version | | ------------- | ---------------------------------- | +| 8.0.0 | 8.x | | 7.2.x | 7.x | | 7.1.x | 7.x | | 7.0.x | 6.8, 7.x | @@ -97,7 +98,36 @@ development: Make sure you have Elasticsearch up and running. You can [install](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html) it locally, but the easiest way is to use [Docker](https://www.docker.com/get-started): ```shell -$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.11.1 +$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:8.15.0 +``` + +### Security + +Please note that starting from version 8 ElasticSearch has security features enabled by default. +Docker command above has it disabled for local testing convenience. If you want to enable it, omit +`"xpack.security.enabled=false"` part from Docker command, and run these command after starting container (container name `es8` assumed): + +Reset password for `elastic` user: +``` +docker container exec es8 '/usr/share/elasticsearch/bin/elasticsearch-reset-password' -u elastic +``` + +Extract CA certificate generated by ElasticSearch on first run: +``` +docker container cp es8:/usr/share/elasticsearch/config/certs/http_ca.crt tmp/ +``` + +And then add them to settings: + +```yaml +# config/chewy.yml +development: + host: 'localhost:9200' + user: 'elastic' + password: 'SomeLongPassword' + transport_options: + ssl: + ca_file: './tmp/http_ca.crt' ``` ### Index @@ -941,6 +971,8 @@ Controller actions are wrapped with the configurable value of `Chewy.request_str It is also a good idea to set up the `:bypass` strategy inside your test suite and import objects manually only when needed, and use `Chewy.massacre` when needed to flush test ES indices before every example. This will allow you to minimize unnecessary ES requests and reduce overhead. +Deprecation note: since version 8 wildcard removing of indices is disabled by default. You can enable it for a cluster with setting `action.destructive_requires_name` to false. + ```ruby RSpec.configure do |config| config.before(:suite) do