From cd71c83aa2a6679af87ebd83ba764cc7d1417e17 Mon Sep 17 00:00:00 2001 From: Dalton Pinto Date: Wed, 10 Feb 2021 11:44:17 +0100 Subject: [PATCH] * Add support for direct_import parameter to skip loading default scope on import (#753) * Skimpier direct_import * Fix linter error in orm.rb * Add spec and fixed an issue related to ids * Update lib/chewy/type/import.rb * Allow direct_import as default import options * Update CHANGELOG.md * Add a note about default `direct_import` value * Improve error messages in db_queries helpers Co-authored-by: Tikhon Botchkarev Co-authored-by: Ivan Rabotyaga --- CHANGELOG.md | 2 ++ lib/chewy/type.rb | 5 ++++- lib/chewy/type/adapter/orm.rb | 9 ++++++--- lib/chewy/type/import.rb | 1 + spec/chewy/type/adapter/active_record_spec.rb | 14 ++++++++++++- spec/support/active_record.rb | 20 +++++++++++++++++++ 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 276484bb5..9d454d369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## Changes + * [#753](https://github.com/toptal/chewy/pull/753): Add support for direct_import parameter to skip objects reloading ([@TikiTDO][][@dalthon][]) * [#739](https://github.com/toptal/chewy/pull/739): Remove explicit `main` branch dependencies on rspec* gems after `rspec-mocks` 3.10.2 is released ([@rabotyaga][]) # Version 5.2.0 (2021-01-28) @@ -572,6 +573,7 @@ [@sharkzp]: https://github.com/@sharkzp [@socialchorus]: https://github.com/socialchorus [@taylor-au]: https://github.com/taylor-au +[@TikiTDO]: https://github.com/TikiTDO [@undr]: https://github.com/@undr [@webgago]: https://github.com/@webgago [@yahooguntu]: https://github.com/yahooguntu diff --git a/lib/chewy/type.rb b/lib/chewy/type.rb index cb8b18865..4d21da1cf 100644 --- a/lib/chewy/type.rb +++ b/lib/chewy/type.rb @@ -14,7 +14,10 @@ module Chewy class Type - IMPORT_OPTIONS_KEYS = %i[batch_size bulk_size refresh consistency replication raw_import journal pipeline].freeze + IMPORT_OPTIONS_KEYS = %i[ + batch_size bulk_size consistency direct_import journal + pipeline raw_import refresh replication + ].freeze include Search include Mapping diff --git a/lib/chewy/type/adapter/orm.rb b/lib/chewy/type/adapter/orm.rb index 584c6aabd..d9a663432 100644 --- a/lib/chewy/type/adapter/orm.rb +++ b/lib/chewy/type/adapter/orm.rb @@ -45,6 +45,7 @@ def identify(collection) # Import options: # # :batch_size - import batch size, 1000 objects by default + # :direct_import - import objects without reloading # # Method handles destroyed objects as well. In case of objects ORM scope # or array passed, objects, responding with true to `destroyed?` method will be deleted @@ -75,10 +76,10 @@ def identify(collection) def import(*args, &block) collection, options = import_args(*args) - if collection.is_a?(relation_class) - import_scope(collection, options, &block) - else + if !collection.is_a?(relation_class) || options[:direct_import] import_objects(collection, options, &block) + else + import_scope(collection, options, &block) end end @@ -119,6 +120,8 @@ def import_objects(collection, options) indexed = collection_ids.each_slice(options[:batch_size]).map do |ids| batch = if options[:raw_import] raw_default_scope_where_ids_in(ids, options[:raw_import]) + elsif options[:direct_import] + hash.values_at(*ids.map(&:to_s)) else default_scope_where_ids_in(ids) end diff --git a/lib/chewy/type/import.rb b/lib/chewy/type/import.rb index ed42d19f4..310c8d8da 100644 --- a/lib/chewy/type/import.rb +++ b/lib/chewy/type/import.rb @@ -67,6 +67,7 @@ module ClassMethods # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default + # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default # @option options [true, false] journal enables imported objects journaling, false by default # @option options [Array] update_fields list of fields for the partial import, empty by default # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default diff --git a/spec/chewy/type/adapter/active_record_spec.rb b/spec/chewy/type/adapter/active_record_spec.rb index ed0de9e00..17cf4a8c4 100644 --- a/spec/chewy/type/adapter/active_record_spec.rb +++ b/spec/chewy/type/adapter/active_record_spec.rb @@ -86,7 +86,19 @@ def import(*args) .to eq([{index: cities.first(2)}, {index: cities.last(1)}]) end - specify { expect(import(cities)).to eq([{index: cities}]) } + specify do + cities + expects_db_queries do + expect(import(cities, direct_import: false)).to eq([{index: cities}]) + end + end + specify do + cities + expects_no_query do + expect(import(cities, direct_import: true)).to eq([{index: cities}]) + end + end + specify do expect(import(cities, batch_size: 2)) .to eq([{index: cities.first(2)}, {index: cities.last(1)}]) diff --git a/spec/support/active_record.rb b/spec/support/active_record.rb index 6727ff01b..8a79b91c1 100644 --- a/spec/support/active_record.rb +++ b/spec/support/active_record.rb @@ -29,6 +29,26 @@ def adapter :active_record end + def expects_db_queries(&block) + have_queries = false + ActiveSupport::Notifications.subscribed( + ->(*_) { have_queries = true }, + 'sql.active_record', + &block + ) + raise 'Expected some db queries, but none were made' unless have_queries + end + + def expects_no_query(&block) + queries = [] + ActiveSupport::Notifications.subscribed( + ->(*args) { queries << args[4][:sql] }, + 'sql.active_record', + &block + ) + raise "Expected no DB queries, but the following ones were made: #{queries.join(', ')}" if queries.present? + end + def stub_model(name, superclass = nil, &block) stub_class(name, superclass || ActiveRecord::Base, &block) end