diff --git a/CHANGELOG.md b/CHANGELOG.md index d276f2b8..ead2ec9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added +* Read view support for select and pairs(#343). + ## [1.2.0] - 07-06-23 ### Added -* Add `noreturn` option for operations: - `insert`, `insert_object`, `insert_many`, `insert_object_many`, - `replace`, `replace_object`, `replace_many`, `insert_object_many`, +* Add `noreturn` option for operations: + `insert`, `insert_object`, `insert_many`, `insert_object_many`, + `replace`, `replace_object`, `replace_many`, `insert_object_many`, `upsert`, `upsert_object`, `upsert_many`, `upsert_object_many`, `update`, `delete` (#267). @@ -39,7 +44,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [1.0.0] - 02-02-23 ### Added -* Add timeout condition for the validation of master presence in +* Add timeout condition for the validation of master presence in replicaset and for the master connection (#95). * Support Cartridge clusterwide configuration for `crud.cfg` (#332). @@ -47,8 +52,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. * **Breaking**: forbid using space id in `crud.len` (#255). ### Fixed -* Add validation of the master presence in replicaset and the - master connection to the `utils.get_space` method before +* Add validation of the master presence in replicaset and the + master connection to the `utils.get_space` method before receiving the space from the connection (#331). * Fix fiber cancel on schema reload timeout in `call_reload_schema` (PR #336). diff --git a/README.md b/README.md index 68e90541..298e954a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,12 @@ It also provides the `crud-storage` and `crud-router` roles for - [Count](#count) - [Call options for crud methods](#call-options-for-crud-methods) - [Statistics](#statistics) + - [Read view](#read-view) + - [Creating a read view](#creating-a-read-view) + - [Closing a read view](#closing-a-read-view) + - [Read view select](#read-view-select) + - [Read view select conditions](#read-view-select-conditions) + - [Read view pairs](#read-view-pairs) - [Cartridge roles](#cartridge-roles) - [Usage](#usage) - [License](#license) @@ -237,8 +243,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuple (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array contains one inserted row, error. @@ -308,8 +314,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuples (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array with inserted rows, array of errors. @@ -450,8 +456,8 @@ where: vshard router instance. Set this parameter if your space is not a part of the default vshard cluster * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array contains one row, error. @@ -493,8 +499,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuple (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array contains one updated row, error. @@ -535,8 +541,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuple (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array contains one deleted row (empty for vinyl), error. @@ -588,8 +594,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuple (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns inserted or replaced rows and metadata or nil with error. @@ -659,8 +665,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuples (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array with inserted/replaced rows, array of errors. @@ -801,8 +807,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuple (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and empty array of rows or nil, error. @@ -868,8 +874,8 @@ where: * `noreturn` (`?boolean`) - suppress successfully processed tuples (first return value is `nil`). `false` by default * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default Returns metadata and array of errors. @@ -1014,8 +1020,8 @@ where: * `yield_every` (`?number`) - number of tuples processed on storage to yield after, `yield_every` should be > 0, default value is 1000 * `fetch_latest_metadata` (`?boolean`) - guarantees the - up-to-date metadata (space format) in first return value, otherwise - it may not take into account the latest migration of the data format. + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. Performance overhead is up to 15%. `false` by default @@ -1541,6 +1547,196 @@ support preserving stats between role reload (see [tarantool/metrics#334](https://github.com/tarantool/metrics/issues/334)), thus this feature will be unsupported for `metrics` driver. +### Read view + +A read view is an in-memory snapshot of data on instances that isn’t affected by future data modifications. Read views allow you to retrieve data using the `read_view_object:select()` and `read_view_object:pairs()` operations. + +Read views can be used to make complex analytical queries. This reduces the load on the main database and improves RPS for a single Tarantool instance. + +Read views have the following limitations: + + * Only the memtx engine is supported. + * Read view can be used starting from Tarantool Enterprise v2.11.0. + +#### Creating a read view + +To create a read view, call the `crud.readview()` function. + +```lua +local foo = crud.readview(opts) +``` + +where: + +* `opts`: + * `name` (`?string`) - name of the read view + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +**Example:** + +```lua +local foo = crud.readview({name = 'foo', timeout = 3}) +``` + +#### Closing a read view + +When a read view is no longer needed, close it using the `read_view_object:close()` method because a read view may consume a substantial amount of memory. + +```lua +local foo = foo.readview() +foo:close(opts) +``` + +where: + +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +Otherwise, a read view is closed implicitly when the read view object is collected by the Lua garbage collector. + +**Example:** + +```lua +local foo = crud.readview() +foo:close({timeout = 3}) +``` + +#### Read view select + +`read_view_object:select()` supports multi-conditional selects, treating a cluster as a single space, same as `crud.select`. + +```lua +local foo = crud.readview() +local objects, err = foo:select(space_name, conditions, opts) +foo:close() +``` + +where: + +* `space_name` (`string`) - name of the space +* `conditions` (`?table`) - array of [select conditions](#select-conditions) +* `opts`: + * `first` (`?number`) - the maximum count of the objects to return. + If negative value is specified, the objects behind `after` are returned + (`after` option is required in this case). [See pagination examples](doc/select.md#pagination). + * `after` (`?table`) - tuple after which objects should be selected + * `batch_size` (`?number`) - number of tuples to process per one request to storage + * `bucket_id` (`?number|cdata`) - bucket ID + * `force_map_call` (`?boolean`) - if `true` + then the map call is performed without any optimizations even + if full primary key equal condition is specified + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + * `fields` (`?table`) - field names for getting only a subset of fields + * `fullscan` (`?boolean`) - if `true` then a critical log entry will be skipped + on potentially long `select`, see [avoiding full scan](doc/select.md#avoiding-full-scan). + * `vshard_router` (`?string|table`) - Cartridge vshard group name or + vshard router instance. Set this parameter if your space is not + a part of the default vshard cluster + * `yield_every` (`?number`) - number of tuples processed on storage to yield after, + `yield_every` should be > 0, default value is 1000 + * `fetch_latest_metadata` (`?boolean`) - guarantees the + up-to-date metadata (space format) in first return value, otherwise + it may not take into account the latest migration of the data format. + Performance overhead is up to 15%. `false` by default + + +Returns metadata and array of rows, error. + +**Example:** + +```lua +local foo = crud.readview() +foo:select('customers', nil, {batch_size=1, fullscan=true}) +--- +- metadata: [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, + {'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}] + rows: + - [1, 477, 'Elizabeth', 12] + - [2, 401, 'Mary', 46] + - [3, 2804, 'David', 33] + - [4, 1161, 'William', 81] + - [5, 1172, 'Jack', 35] + - [6, 1064, 'William', 25] + - [7, 693, 'Elizabeth', 18] +- null +... +crud.insert('customers', {8, box.NULL, 'Elizabeth', 23}) +--- +- rows: + - [8, 185, 'Elizabeth', 23] + metadata: [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, + {'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}] +- null +... +foo:select('customers', nil, {batch_size=1, fullscan=true}) +--- +- metadata: [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, + {'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}] + rows: + - [1, 477, 'Elizabeth', 12] + - [2, 401, 'Mary', 46] + - [3, 2804, 'David', 33] + - [4, 1161, 'William', 81] + - [5, 1172, 'Jack', 35] + - [6, 1064, 'William', 25] + - [7, 693, 'Elizabeth', 18] +- null +... +foo.close() +``` + +##### Read view select conditions + +Select conditions for `read_view_object:select()` are the same as [select conditions](#select-conditions) for `crud.select`. + +**Example:** + +```lua +foo = crud.readview() +foo:select('customers', {{'<=', 'age', 35}}, {first = 10}) +--- +- metadata: + - {'name': 'id', 'type': 'unsigned'} + - {'name': 'bucket_id', 'type': 'unsigned'} + - {'name': 'name', 'type': 'string'} + - {'name': 'age', 'type': 'number'} + rows: + - [5, 1172, 'Jack', 35] + - [3, 2804, 'David', 33] + - [6, 1064, 'William', 25] + - [7, 693, 'Elizabeth', 18] + - [1, 477, 'Elizabeth', 12] +... +foo.close() +``` + +#### Read view pairs + +You can iterate across a distributed space using the `read_view_object:pairs()` method. +Its arguments are the same as [`crud.readview.select`](#read-view-select) arguments except +`fullscan` (it does not exist because `crud.pairs` does not generate a critical +log entry on potentially long requests) and negative `first` values aren't +allowed. +User could pass use_tomap flag (false by default) to iterate over flat tuples or objects. + +**Example:** + +```lua +foo = crud.readview() +local tuples = {} +for _, tuple in foo:pairs('customers', {{'<=', 'age', 35}}, {use_tomap = false}) do + -- {5, 1172, 'Jack', 35} + table.insert(tuples, tuple) +end + +local objects = {} +for _, object in foo:pairs('customers', {{'<=', 'age', 35}}, {use_tomap = true}) do + -- {id = 5, name = 'Jack', bucket_id = 1172, age = 35} + table.insert(objects, object) +end +foo:close() +``` + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/crud.lua b/crud.lua index 7ceac988..f65ae91b 100644 --- a/crud.lua +++ b/crud.lua @@ -20,6 +20,7 @@ local borders = require('crud.borders') local sharding_metadata = require('crud.common.sharding.sharding_metadata') local utils = require('crud.common.utils') local stats = require('crud.stats') +local readview = require('crud.readview') local crud = {} @@ -147,6 +148,10 @@ crud.reset_stats = stats.reset -- @function storage_info crud.storage_info = utils.storage_info +-- @refer readview.new +-- @function readview +crud.readview = readview.new + --- Initializes crud on node -- -- Exports all functions that are used for calls @@ -174,6 +179,7 @@ function crud.init_storage() count.init() borders.init() sharding_metadata.init() + readview.init() _G._crud.storage_info_on_storage = utils.storage_info_on_storage end diff --git a/crud/common/stash.lua b/crud/common/stash.lua index 9dbf3c84..bfc20639 100644 --- a/crud/common/stash.lua +++ b/crud/common/stash.lua @@ -30,6 +30,7 @@ stash.name = { stats_metrics_registry = '__crud_stats_metrics_registry', ddl_triggers = '__crud_ddl_spaces_triggers', select_module_compat_info = '__select_module_compat_info', + storage_readview = '__crud_storage_readview', } --- Setup Tarantool Cartridge reload. diff --git a/crud/readview.lua b/crud/readview.lua new file mode 100644 index 00000000..ff907ce6 --- /dev/null +++ b/crud/readview.lua @@ -0,0 +1,322 @@ +local fiber = require('fiber') +local checks = require('checks') +local errors = require('errors') +local tarantool = require('tarantool') + +local const = require('crud.common.const') +local stash = require('crud.common.stash') +local utils = require('crud.common.utils') +local sharding = require('crud.common.sharding') +local select_executor = require('crud.select.executor') +local select_filters = require('crud.compare.filters') +local dev_checks = require('crud.common.dev_checks') +local schema = require('crud.common.schema') +local stats = require('crud.stats') + +local ReadviewError = errors.new_class('ReadviewError', {capture_stack = false}) + +local has_merger = (utils.tarantool_supports_external_merger() and + package.search('tuple.merger')) or utils.tarantool_has_builtin_merger() + +if (not utils.tarantool_version_at_least(2, 11, 0)) + or (tarantool.package ~= 'Tarantool Enterprise') or (not has_merger) then + return { + new = function() return nil, ReadviewError:new("Tarantool does not support readview") end, + init = function() return nil end} +end +local select = require('crud.select.compat.select') + +local readview = {} + + +local function readview_open_on_storage(readview_name) + if not utils.tarantool_version_at_least(2, 11, 0) or + tarantool.package ~= 'Tarantool Enterprise' then + error(ReadviewError:new("Tarantool does not support readview")) + return nil + end + -- We store readview in stash because otherwise gc will delete it. + -- e.g master switch. + local read_view = box.read_view.open({name = readview_name}) + local stash_readview = stash.get(stash.name.storage_readview) + stash_readview[read_view.id] = read_view + + if read_view == nil then + error(ReadviewError:new("Error creating readview")) + return nil + end + + local replica_info = {} + replica_info.uuid = box.info().uuid + replica_info.id = read_view.id + + return replica_info, nil +end + +local function readview_close_on_storage(readview_uuid) + dev_checks('table') + + local list = box.read_view.list() + local readview_id + for _, replica_info in pairs(readview_uuid) do + if replica_info.uuid == box.info().uuid then + readview_id = replica_info.id + end + end + + for k,v in pairs(list) do + if v.id == readview_id then + list[k]:close() + local stash_readview = stash.get(stash.name.storage_readview) + stash_readview[readview_id] = nil + return true + end + end + + return false +end + +local function select_readview_on_storage(space_name, index_id, conditions, opts) + dev_checks('string', 'number', '?table', { + scan_value = 'table', + after_tuple = '?table', + tarantool_iter = 'number', + limit = 'number', + scan_condition_num = '?number', + field_names = '?table', + sharding_key_hash = '?number', + sharding_func_hash = '?number', + skip_sharding_hash_check = '?boolean', + yield_every = '?number', + fetch_latest_metadata = '?boolean', + readview_id = 'number' + }) + + local cursor = {} + if opts.fetch_latest_metadata then + local replica_schema_version + if box.info.schema_version ~= nil then + replica_schema_version = box.info.schema_version + else + replica_schema_version = box.internal.schema_version() + end + cursor.storage_info = { + replica_uuid = box.info().uuid, + replica_schema_version = replica_schema_version, + } + end + + local list = box.read_view.list() + local space_readview + + for k,v in pairs(list) do + if v.id == opts.readview_id then + space_readview = list[k].space[space_name] + end + end + + if space_readview == nil then + return cursor, ReadviewError:new("Space %q doesn't exist", space_name) + end + + local space = box.space[space_name] + if space == nil then + return cursor, ReadviewError:new("Space %q doesn't exist", space_name) + end + space_readview.format = space:format() + + local index_readview = space_readview.index[index_id] + if index_readview == nil then + return cursor, ReadviewError:new("Index with ID %s doesn't exist", index_id) + end + local index_format = space.index[index_id] + if index_format == nil then + return cursor, ReadviewError:new("Index with ID %s doesn't exist", index_id) + end + + local _, err = sharding.check_sharding_hash(space_name, + opts.sharding_func_hash, + opts.sharding_key_hash, + opts.skip_sharding_hash_check) + + if err ~= nil then + return nil, err + end + + local filter_func, err = select_filters.gen_func(space, conditions, { + tarantool_iter = opts.tarantool_iter, + scan_condition_num = opts.scan_condition_num, + }) + if err ~= nil then + return cursor, ReadviewError:new("Failed to generate tuples filter: %s", err) + end + + -- execute select + local resp, err = select_executor.execute(space, index_format, filter_func, { + scan_value = opts.scan_value, + after_tuple = opts.after_tuple, + tarantool_iter = opts.tarantool_iter, + limit = opts.limit, + yield_every = opts.yield_every, + readview = true, + readview_index = index_readview, + }) + if err ~= nil then + return cursor, ReadviewError:new("Failed to execute select: %s", err) + end + + if resp.tuples_fetched < opts.limit or opts.limit == 0 then + cursor.is_end = true + else + local last_tuple = resp.tuples[#resp.tuples] + cursor.after_tuple = last_tuple + end + + cursor.stats = { + tuples_lookup = resp.tuples_lookup, + tuples_fetched = resp.tuples_fetched, + } + + -- getting tuples with user defined fields (if `fields` option is specified) + -- and fields that are needed for comparison on router (primary key + scan key) + local filtered_tuples = schema.filter_tuples_fields(resp.tuples, opts.field_names) + + local result = {cursor, filtered_tuples} + + local select_module_compat_info = stash.get(stash.name.select_module_compat_info) + if not select_module_compat_info.has_merger then + if opts.fetch_latest_metadata then + result[3] = cursor.storage_info.replica_schema_version + end + end + + return unpack(result) +end + +local Readview_obj = {} +Readview_obj.__index = Readview_obj + +local select_call = stats.wrap(select.call, stats.op.SELECT) + +function Readview_obj:select(space_name, user_conditions, opts) + opts = opts or {} + opts.readview = true + opts.readview_uuid = self._uuid + + return select_call(space_name, user_conditions, opts) +end + +local pairs_call = stats.wrap(select.pairs, stats.op.SELECT, {pairs = true}) +function Readview_obj:pairs(space_name, user_conditions, opts) + opts = opts or {} + opts.readview = true + opts.readview_uuid = self._uuid + + return pairs_call(space_name, user_conditions, opts) +end + +function readview.init() + _G._crud.readview_open_on_storage = readview_open_on_storage + _G._crud.readview_close_on_storage = readview_close_on_storage + _G._crud.select_readview_on_storage = select_readview_on_storage + end + +function Readview_obj:close(opts) + checks('table', { + timeout = '?number', + }) + opts = opts or {} + if self._opened == false then + return + end + + local vshard_router, err = utils.get_vshard_router_instance(nil) + if err ~= nil then + return ReadviewError:new(err) + end + + local replicasets, err = vshard_router:routeall() + if err ~= nil then + return ReadviewError:new(err) + end + + if opts.timeout == nil then + opts.timeout = const.DEFAULT_VSHARD_CALL_TIMEOUT + end + + local errors = {} + for _, replicaset in pairs(replicasets) do + for replica_uuid, replica in pairs(replicaset.replicas) do + for _, value in pairs(self._uuid) do + if replica_uuid == value.uuid then + local replica_result, replica_err = replica.conn:call('_crud.readview_close_on_storage', + {self._uuid}, {timeout = opts.timeout}) + if replica_err ~= nil then + table.insert(errors, ReadviewError:new("Failed to close Readview on storage: %s", replica_err)) + end + if replica_err == nil and (not replica_result) then + table.insert(errors, ReadviewError:new("Readview was not found on storage: %s", replica_uuid)) + end + end + end + end + end + + if next(errors) ~= nil then + return errors + end + + self._opened = false + return nil + +end + +function Readview_obj:__gc() + fiber.new(self.close, self) +end + +function Readview_obj.create(vshard_router, opts) + local readview = {} + setmetatable(readview, Readview_obj) + readview._name = opts.name + local results, err, err_uuid = vshard_router:map_callrw('_crud.readview_open_on_storage', + {readview._name}, {timeout = opts.timeout}) + if err ~= nil then + return nil, + ReadviewError:new("Failed to call readview_open_on_storage on storage-side: storage uuid: %s err: %s", + err_uuid, err) + end + + local uuid = {} + local errors = {} + for _, replicaset_results in pairs(results) do + for _, replica_result in pairs(replicaset_results) do + table.insert(uuid, replica_result) + end + end + + readview._uuid = uuid + readview._opened = true + + if next(errors) ~= nil then + return nil, errors + end + return readview, nil + end + +function readview.new(opts) + checks({ + name = '?string', + timeout = '?number', + }) + opts = opts or {} + local vshard_router, err = utils.get_vshard_router_instance(nil) + if err ~= nil then + return nil, ReadviewError:new(err) + end + + return Readview_obj.create(vshard_router, opts) +end + + +return readview diff --git a/crud/select/compat/select.lua b/crud/select/compat/select.lua index 9ca8fb88..1fefcd5a 100644 --- a/crud/select/compat/select.lua +++ b/crud/select/compat/select.lua @@ -30,6 +30,8 @@ local function build_select_iterator(vshard_router, space_name, user_conditions, field_names = '?table', yield_every = '?number', call_opts = 'table', + readview = '?boolean', + readview_uuid = '?table', }) opts = opts or {} @@ -173,13 +175,21 @@ local function build_select_iterator(vshard_router, space_name, user_conditions, yield_every = yield_every, fetch_latest_metadata = opts.call_opts.fetch_latest_metadata, } + local merger - local merger = Merger.new(vshard_router, replicasets_to_select, space, plan.index_id, - common.SELECT_FUNC_NAME, - {space_name, plan.index_id, plan.conditions, select_opts}, - {tarantool_iter = plan.tarantool_iter, field_names = plan.field_names, call_opts = opts.call_opts} - ) - + if opts.readview then + merger = Merger.new_readview(vshard_router, replicasets_to_select, opts.readview_uuid, + space, plan.index_id, '_crud.select_readview_on_storage', + {space_name, plan.index_id, plan.conditions, select_opts}, + {tarantool_iter = plan.tarantool_iter, field_names = plan.field_names, call_opts = opts.call_opts} + ) + else + merger = Merger.new(vshard_router, replicasets_to_select, space, plan.index_id, + common.SELECT_FUNC_NAME, + {space_name, plan.index_id, plan.conditions, select_opts}, + {tarantool_iter = plan.tarantool_iter, field_names = plan.field_names, call_opts = opts.call_opts} + ) + end return { tuples_limit = tuples_limit, merger = merger, @@ -204,6 +214,8 @@ function select_module.pairs(space_name, user_conditions, opts) prefer_replica = '?boolean', balance = '?boolean', timeout = '?number', + readview = '?boolean', + readview_uuid = '?table', vshard_router = '?string|table', @@ -212,6 +224,24 @@ function select_module.pairs(space_name, user_conditions, opts) opts = opts or {} + if opts.readview == true then + if opts.mode ~= nil then + return nil, SelectError:new("Readview does not support 'mode' option") + end + + if opts.prefer_replica ~= nil then + return nil, SelectError:new("Readview does not support 'prefer_replica' option") + end + + if opts.balance ~= nil then + return nil, SelectError:new("Readview does not support 'balance' option") + end + + if opts.vshard_router ~= nil then + return nil, SelectError:new("Readview does not support 'vshard_router' option") + end + end + if opts.first ~= nil and opts.first < 0 then error(string.format("Negative first isn't allowed for pairs")) end @@ -236,6 +266,8 @@ function select_module.pairs(space_name, user_conditions, opts) timeout = opts.timeout, fetch_latest_metadata = opts.fetch_latest_metadata, }, + readview = opts.readview, + readview_uuid = opts.readview_uuid, } local iter, err = schema.wrap_func_reload( @@ -299,12 +331,32 @@ local function select_module_call_xc(vshard_router, space_name, user_conditions, prefer_replica = '?boolean', balance = '?boolean', timeout = '?number', + readview = '?boolean', + readview_uuid = '?table', vshard_router = '?string|table', yield_every = '?number', }) + if opts.readview == true then + if opts.mode ~= nil then + return nil, SelectError:new("Readview does not support 'mode' option") + end + + if opts.prefer_replica ~= nil then + return nil, SelectError:new("Readview does not support 'prefer_replica' option") + end + + if opts.balance ~= nil then + return nil, SelectError:new("Readview does not support 'balance' option") + end + + if opts.vshard_router ~= nil then + return nil, SelectError:new("Readview does not support 'vshard_router' option") + end + end + if opts.first ~= nil and opts.first < 0 then if opts.after == nil then return nil, SelectError:new("Negative first should be specified only with after option") @@ -326,6 +378,8 @@ local function select_module_call_xc(vshard_router, space_name, user_conditions, timeout = opts.timeout, fetch_latest_metadata = opts.fetch_latest_metadata, }, + readview = opts.readview, + readview_uuid = opts.readview_uuid, } local iter, err = schema.wrap_func_reload( diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 7c8c36c6..d432c62e 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -81,6 +81,8 @@ function executor.execute(space, index, filter_func, opts) tarantool_iter = 'number', limit = '?number', yield_every = '?number', + readview = '?boolean', + readview_index = '?table' }) opts = opts or {} @@ -132,7 +134,12 @@ function executor.execute(space, index, filter_func, opts) end local tuple - local raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter}) + local raw_gen, param, state + if opts.readview then + raw_gen, param, state = opts.readview_index:pairs(value, {iterator = opts.tarantool_iter}) + else + raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter}) + end local gen = fun.wrap(function(param, state) local next_state, var = raw_gen(param, state) diff --git a/crud/select/merger.lua b/crud/select/merger.lua index 9d53563a..b56e993e 100644 --- a/crud/select/merger.lua +++ b/crud/select/merger.lua @@ -152,9 +152,10 @@ local function fetch_chunk(context, state) stats.update_fetch_stats(cursor.stats, space_name) end + local next_state = {} + -- Check whether we need the next call. if cursor.is_end then - local next_state = {} return next_state, buf end @@ -164,9 +165,13 @@ local function fetch_chunk(context, state) -- change context.func_args too, but it does not matter next_func_args[4].after_tuple = cursor.after_tuple - local next_future = replicaset[vshard_call_name](replicaset, func_name, next_func_args, net_box_opts) - local next_state = {future = next_future} + if context.readview then + next_state = {future = context.future_replica.conn:call(func_name, next_func_args, net_box_opts)} + else + local next_future = replicaset[vshard_call_name](replicaset, func_name, next_func_args, net_box_opts) + next_state = {future = next_future} + end return next_state, buf end @@ -203,6 +208,7 @@ local function new(vshard_router, replicasets, space, index_id, func_name, func_ fetch_latest_metadata = call_opts.fetch_latest_metadata, space_name = space.name, vshard_router = vshard_router, + readview = false, } local state = {future = future} @@ -230,6 +236,67 @@ local function new(vshard_router, replicasets, space, index_id, func_name, func_ return merger end + +local function new_readview(vshard_router, replicasets, readview_uuid, space, index_id, func_name, func_args, opts) + opts = opts or {} + local call_opts = opts.call_opts + + -- Request a first data chunk and create merger sources. + local merger_sources = {} + for _, replicaset in pairs(replicasets) do + for replica_uuid, replica in pairs(replicaset.replicas) do + for _, value in pairs(readview_uuid) do + if replica_uuid == value.uuid then + -- Perform a request. + local buf = buffer.ibuf() + local net_box_opts = {is_async = true, buffer = buf, skip_header = false} + func_args[4].readview_id = value.id + local future = replica.conn:call(func_name, func_args, net_box_opts) + + -- Create a source. + local context = { + net_box_opts = net_box_opts, + buffer = buf, + func_name = func_name, + func_args = func_args, + replicaset = replicaset, + vshard_call_name = nil, + timeout = call_opts.timeout, + fetch_latest_metadata = call_opts.fetch_latest_metadata, + space_name = space.name, + vshard_router = vshard_router, + readview = true, + future_replica = replica + } + local state = {future = future} + local source = merger_lib.new_buffer_source(fetch_chunk, context, state) + table.insert(merger_sources, source) + end + end + end + end + + -- Trick for performance. + -- + -- No need to create merger, key_def and pass tuples over the + -- merger, when we have only one tuple source. + if #merger_sources == 1 then + return merger_sources[1] + end + + local keydef = Keydef.new(space, opts.field_names, index_id) + -- When built-in merger is used with external keydef, `merger_lib.new(keydef)` + -- fails. It's simply fixed by casting `keydef` to 'struct key_def&'. + keydef = ffi.cast('struct key_def&', keydef) + + local merger = merger_lib.new(keydef, merger_sources, { + reverse = reverse_tarantool_iters[opts.tarantool_iter], + }) + + return merger +end + return { new = new, + new_readview = new_readview, } diff --git a/test/integration/pairs_readview_test.lua b/test/integration/pairs_readview_test.lua new file mode 100644 index 00000000..baadaf6a --- /dev/null +++ b/test/integration/pairs_readview_test.lua @@ -0,0 +1,892 @@ +local fio = require('fio') + +local t = require('luatest') + +local crud_utils = require('crud.common.utils') + +local helpers = require('test.helper') + +local pgroup = t.group('pairs_readview', { + {engine = 'memtx'}, +}) + +pgroup.before_all(function(g) + + if (not helpers.tarantool_version_at_least(2, 11, 0)) + or (not require('luatest.tarantool').is_enterprise_package()) then + t.skip('Readview is supported only for Tarantool Enterprise starting from v2.11.0') + end + + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + env = { + ['ENGINE'] = g.params.engine, + }, + }) + + g.cluster:start() + + g.space_format = g.cluster.servers[2].net_box.space.customers:format() + + g.cluster.main_server.net_box:eval([[ + require('crud').cfg{ stats = true } + ]]) +end) + +pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +pgroup.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, 'customers') +end) + + +pgroup.test_pairs_no_conditions = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local raw_rows = { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + {4, 1161, 'William', 'White', 81, 'Chicago'}, + } + + -- without conditions and options + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers') do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + t.assert_equals(objects, raw_rows) + + -- with use_tomap=false (the raw tuples returned) + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {use_tomap = false}) do + table.insert(objects, object) + end + + foo:close() + return objects + ]]) + t.assert_equals(objects, raw_rows) + + -- no after + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + + t.assert_equals(objects, customers) + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local after = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 4})) + + -- after obj 4 (last) + local after = crud_utils.flatten(customers[4], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local after = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {after}) + + t.assert_equals(err, nil) + t.assert_equals(#objects, 0) +end + +pgroup.test_ge_condition_with_index = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>=', 'age', 33}, + } + + -- no after + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 2, 4})) -- in age order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 4})) -- in age order +end + +pgroup.test_le_condition_with_index = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'<=', 'age', 33}, + } + + -- no after + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local objects = {} + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 1})) -- in age order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1})) -- in age order +end + +pgroup.test_first = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- w/ tomap + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {first = 2, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 2})) + + local tuples, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local tuples = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, tuple in foo:pairs('customers', nil, {first = 2}) do + table.insert(tuples, tuple) + end + foo:close() + return tuples + ]]) + t.assert_equals(err, nil) + t.assert_equals(tuples, { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + }) +end + +pgroup.test_negative_first = function(g) + local customers = helpers.insert_objects(g, 'customers',{ + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- negative first + t.assert_error_msg_contains("Negative first isn't allowed for pairs", function() + g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + foo:pairs('customers', nil, {first = -10}) + foo:close() + ]]) + end) +end + +pgroup.test_empty_space = function(g) + local count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local count = 0 + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers') do + count = count + 1 + end + foo:close() + return count + ]]) + t.assert_equals(count, 0) +end + +pgroup.test_luafun_compatibility = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + local count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local count = foo:pairs('customers'):map(function() return 1 end):sum() + foo:close() + return count + ]]) + t.assert_equals(count, 3) + + count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local count = foo:pairs('customers', + {use_tomap = true}):map(function() return 1 end):sum() + foo:close() + return count + ]]) + t.assert_equals(count, 3) +end + +pgroup.test_pairs_partial_result = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + local expected_customers = { + {id = 3, age = 33, name = "David", city = "Los Angeles"}, + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + expected_customers = { + {id = 3, age = 33, name = "David"}, + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition by non-indexed non-unique non-primary field (city): + conditions = {{'>=', 'city', 'Lo'}} + + -- condition field is not in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", age = 12}, + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", city = "Los Angeles"}, + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) +end + +pgroup.test_pairs_force_map_call = function(g) + local key = 1 + + local first_bucket_id = g.cluster.main_server.net_box:eval([[ + local vshard = require('vshard') + + local key = ... + return vshard.router.bucket_id_strcrc32(key) + ]], {key}) + + local second_bucket_id, err = helpers.get_other_storage_bucket_id(g.cluster, first_bucket_id) + + t.assert_equals(err, nil) + + local customers = helpers.insert_objects(g, 'customers', { + { + id = key, bucket_id = first_bucket_id, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = key, bucket_id = second_bucket_id, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + + local conditions = {{'==', 'id', key}} + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(#objects, 1) + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', conditions, {use_tomap = true, force_map_call = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + table.sort(objects, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + t.assert_equals(objects, customers) +end + +pgroup.test_pairs_timeout = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local raw_rows = { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + {4, 1161, 'William', 'White', 81, 'Chicago'}, + } + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + for _, object in foo:pairs('customers', nil, {timeout = 1}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]]) + t.assert_equals(objects, raw_rows) +end + +-- gh-220: bucket_id argument is ignored when it cannot be deduced +-- from provided select/pairs conditions. +pgroup.test_pairs_no_map_reduce = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + -- bucket_id is 477, storage is s-2 + id = 1, name = 'Elizabeth', last_name = 'Jackson', + age = 12, city = 'New York', + }, { + -- bucket_id is 401, storage is s-2 + id = 2, name = 'Mary', last_name = 'Brown', + age = 46, city = 'Los Angeles', + }, { + -- bucket_id is 2804, storage is s-1 + id = 3, name = 'David', last_name = 'Smith', + age = 33, city = 'Los Angeles', + }, { + -- bucket_id is 1161, storage is s-2 + id = 4, name = 'William', last_name = 'White', + age = 81, city = 'Chicago', + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local router = g.cluster:server('router').net_box + local map_reduces_before = helpers.get_map_reduces_stat(router, 'customers') + + -- Case: no conditions, just bucket id. + local rows = router:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + + local rows = foo:pairs(...):totable() + foo:close() + return rows + ]], { + 'customers', + nil, + {bucket_id = 2804, timeout = 1}, + }) + t.assert_equals(rows, { + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + }) + + local map_reduces_after_1 = helpers.get_map_reduces_stat(router, 'customers') + local diff_1 = map_reduces_after_1 - map_reduces_before + t.assert_equals(diff_1, 0, 'Select request was not a map reduce') + + -- Case: EQ on secondary index, which is not in the sharding + -- index (primary index in the case). + local rows = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local rows = foo:pairs(...):totable() + foo:close() + return rows + ]], { + 'customers', + {{'==', 'age', 81}}, + {bucket_id = 1161, timeout = 1}, + }) + t.assert_equals(rows, { + {4, 1161, 'William', 'White', 81, 'Chicago'}, + }) + + local map_reduces_after_2 = helpers.get_map_reduces_stat(router, 'customers') + local diff_2 = map_reduces_after_2 - map_reduces_after_1 + t.assert_equals(diff_2, 0, 'Select request was not a map reduce') +end diff --git a/test/integration/readview_not_supported_test.lua b/test/integration/readview_not_supported_test.lua new file mode 100644 index 00000000..ac79b2a0 --- /dev/null +++ b/test/integration/readview_not_supported_test.lua @@ -0,0 +1,51 @@ +local fio = require('fio') + +local t = require('luatest') + +local helpers = require('test.helper') + +local pgroup = t.group('readview_not_supported', { + {engine = 'memtx'}, +}) + + +pgroup.before_all(function(g) + if helpers.tarantool_version_at_least(2, 11, 0) + and require('luatest.tarantool').is_enterprise_package() then + t.skip() + end + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + env = { + ['ENGINE'] = g.params.engine, + }, + }) + + g.cluster:start() + + g.space_format = g.cluster.servers[2].net_box.space.customers:format() + + g.cluster:server('router').net_box:eval([[ + require('crud').cfg{ stats = true } + ]]) + g.cluster:server('router').net_box:eval([[ + require('crud.ratelimit').disable() + ]]) +end) + +pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +pgroup.test_open = function(g) + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + return foo, err + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.str, 'Tarantool does not support readview') + +end \ No newline at end of file diff --git a/test/integration/select_readview_test.lua b/test/integration/select_readview_test.lua new file mode 100644 index 00000000..9a4ef919 --- /dev/null +++ b/test/integration/select_readview_test.lua @@ -0,0 +1,2374 @@ +local fio = require('fio') + +local fiber = require('fiber') +local t = require('luatest') + +local crud = require('crud') +local crud_utils = require('crud.common.utils') + + +local helpers = require('test.helper') + +local pgroup = t.group('select_readview', { + {engine = 'memtx'}, +}) + + +pgroup.before_all(function(g) + if (not helpers.tarantool_version_at_least(2, 11, 0)) + or (not require('luatest.tarantool').is_enterprise_package()) then + t.skip('Readview is supported only for Tarantool Enterprise starting from v2.11.0') + end + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + env = { + ['ENGINE'] = g.params.engine, + }, + }) + + g.cluster:start() + + g.space_format = g.cluster.servers[2].net_box.space.customers:format() + + g.cluster:server('router').net_box:eval([[ + require('crud').cfg{ stats = true } + ]]) + g.cluster:server('router').net_box:eval([[ + require('crud.ratelimit').disable() + ]]) +end) + +pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +pgroup.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, 'customers') + helpers.truncate_space_on_cluster(g.cluster, 'developers') + helpers.truncate_space_on_cluster(g.cluster, 'cars') +end) + +local function set_master(cluster, uuid, master_uuid) + cluster.main_server:graphql({ + query = [[ + mutation( + $uuid: String! + $master_uuid: [String!]! + ) { + edit_replicaset( + uuid: $uuid + master: $master_uuid + ) + } + ]], + variables = {uuid = uuid, master_uuid = {master_uuid}} + }) +end + +pgroup.test_non_existent_space = function(g) + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + + local result, err = foo:select('non_existent_space', nil, {fullscan=true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Space \"non_existent_space\" doesn't exist") +end + +pgroup.test_select_no_index = function(g) + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('no_index_space', nil, {fullscan=true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Space \"no_index_space\" has no indexes, space should have primary index") +end + +pgroup.test_invalid_value_type = function(g) + local conditions = { + {'=', 'id', 'not_number'} + } + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local conditions = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions) + + foo:close() + + return result, err + ]], {conditions}) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type: expected unsigned") +end + +pgroup.test_gc_on_storage = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + + local _, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + rawset(_G, 'foo', foo) + ]]) + t.assert_equals(err, nil) + local res = {} + helpers.call_on_storages(g.cluster, function(server, replicaset, res) + local instance_res = server.net_box:eval([[ + collectgarbage("collect") + collectgarbage("collect")]]) + res[replicaset.alias] = instance_res + end, res) + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo = rawget(_G, 'foo') + local result, err = foo:select('customers', nil, {fullscan = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_close_gc_on_router = function(g) + local _, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + foo = nil + collectgarbage("collect") + collectgarbage("collect") + ]]) + t.assert_equals(err, nil) + local res = {} + helpers.call_on_storages(g.cluster, function(server, replicaset, res) + local instance_res = server.net_box:eval([[ + return box.read_view.list()]]) + res[replicaset.alias] = instance_res + end, res) + t.assert_equals(res, {["s-1"] = {}, ["s-2"] = {}}) + +end + +pgroup.test_close = function(g) + local _, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + foo:close() + ]]) + t.assert_equals(err, nil) + local res = {} + helpers.call_on_storages(g.cluster, function(server, replicaset, res) + local instance_res = server.net_box:eval([[ + return box.read_view.list()]]) + res[replicaset.alias] = instance_res + end, res) + t.assert_equals(res, {["s-1"] = {}, ["s-2"] = {}}) + +end + +pgroup.test_select_all = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', nil, {fullscan = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_with_same_name = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local boo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + foo:close() + + local result, err = boo:select('customers', nil, {fullscan = true}) + + boo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_without_name = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local boo, err = crud.readview({name = nil}) + if err ~= nil then + return nil, err + end + local result, err = boo:select('customers', nil, {fullscan = true}) + + boo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_with_insert = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local boo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + rawset(_G, 'boo', boo) + + local result, err = boo:select('customers', nil, {fullscan = true}) + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + + helpers.insert_objects(g, 'customers', { + { + id = 5, name = "Andrew", last_name = "White", + age = 55, city = "Chicago" + }, + }) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local boo = rawget(_G, 'boo') + + local result, err = boo:select('customers', nil, {fullscan = true}) + boo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_with_delete = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local boo, err = crud.readview({}) + if err ~= nil then + return nil, err + end + rawset(_G, 'boo', boo) + + local result, err = boo:select('customers', nil, {fullscan = true}) + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + + local _, err = g.cluster.main_server.net_box:call('crud.delete', {'customers', 3}) + t.assert_equals(err, nil) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local boo = rawget(_G, 'boo') + + local result, err = boo:select('customers', nil, {fullscan = true}) + boo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_all_with_batch_size = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, { + id = 5, name = "Jack", last_name = "Sparrow", + age = 35, city = "London", + }, { + id = 6, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 7, name = "Elizabeth", last_name = "Swan", + age = 18, city = "Cambridge", + }, { + id = 8, name = "Hector", last_name = "Barbossa", + age = 45, city = "London", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- batch size 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + + local result, err = foo:select('customers', nil, {batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, customers) + + -- batch size 3 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local bar, err = crud.readview({name = 'bar'}) + if err ~= nil then + return nil, err + end + local result, err = bar:select('customers', nil, {batch_size=3, fullscan = true}) + + bar:close() + return result, err + ]]) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, customers) +end + +pgroup.test_eq_condition_with_index = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 33, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "Smith", + age = 81, city = "Chicago", + },{ + id = 5, name = "Hector", last_name = "Barbossa", + age = 33, city = "Chicago", + },{ + id = 6, name = "William", last_name = "White", + age = 81, city = "Chicago", + },{ + id = 7, name = "Jack", last_name = "Sparrow", + age = 33, city = "Chicago", + }, { + id = 8, name = "Nick", last_name = "Smith", + age = 20, city = "London", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'==', 'age_index', 33}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 7})) -- in id order + + -- after obj 5 with negative first + local after = crud_utils.flatten(customers[5], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3})) -- in id order + + -- after obj 8 + local after = crud_utils.flatten(customers[8], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, first = 10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order + + -- after obj 8 with negative first + local after = crud_utils.flatten(customers[8], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, first = 10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) + + -- after obj 2 with negative first + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order +end + +pgroup.test_lt_condition_with_index = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'<', 'age_index', 33}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1})) -- in age order + + -- after obj 1 + local after = crud_utils.flatten(customers[1], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) -- in age order +end + +pgroup.test_multiple_conditions = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Rodriguez", + age = 20, city = "Los Angeles", + }, { + id = 2, name = "Elizabeth", last_name = "Rodriguez", + age = 44, city = "Chicago", + }, { + id = 3, name = "Elizabeth", last_name = "Rodriguez", + age = 22, city = "New York", + }, { + id = 4, name = "David", last_name = "Brown", + age = 23, city = "Los Angeles", + }, { + id = 5, name = "Elizabeth", last_name = "Rodriguez", + age = 39, city = "Chicago", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>', 'age', 20}, + {'==', 'name', 'Elizabeth'}, + {'==', 'city', 'Chicago'}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 2})) -- in age order + + -- after obj 5 + local after = crud_utils.flatten(customers[5], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in age order +end + +pgroup.test_composite_index = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Rodriguez", + age = 20, city = "Los Angeles", + }, { + id = 2, name = "Elizabeth", last_name = "Johnson", + age = 44, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Brown", + age = 23, city = "Chicago", + }, { + id = 4, name = "Jessica", last_name = "Jones", + age = 22, city = "New York", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>=', 'full_name', {"Elizabeth", "Jo"}}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 1, 4})) -- in full_name order + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 4})) -- in full_name order + + -- partial value in conditions + local conditions = { + {'==', 'full_name', "Elizabeth"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 1})) -- in full_name order + + -- first 1 + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{first=1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in full_name order + + -- first 1 with full specified key + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', {{'==', 'full_name', {'Elizabeth', 'Johnson'}}}, {first = 1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in full_name order +end + +pgroup.test_composite_primary_index = function(g) + local book_translation = helpers.insert_objects(g, 'book_translation', { + { + id = 5, + language = 'Ukrainian', + edition = 55, + translator = 'Mitro Dmitrienko', + comments = 'Translation 55', + } + }) + t.assert_equals(#book_translation, 1) + + local conditions = {{'=', 'id', {5, 'Ukrainian', 55}}} + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('book_translation', conditions) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('book_translation', conditions, {first = 2}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('book_translation', conditions, {first = 1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('book_translation', conditions, {first = 1, after = after}) + + foo:close() + return result, err + ]], {conditions, result.rows[1]}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 0) +end + +pgroup.test_select_with_batch_size_1 = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, { + id = 5, name = "Jack", last_name = "Sparrow", + age = 35, city = "London", + }, { + id = 6, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 7, name = "Elizabeth", last_name = "Swan", + age = 18, city = "Cambridge", + }, { + id = 8, name = "Hector", last_name = "Barbossa", + age = 45, city = "London", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- LE + local conditions = {{'<=', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 3, 6, 7, 1})) + + -- LT + local conditions = {{'<', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 6, 7, 1})) + + -- GE + local conditions = {{'>=', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 8, 2, 4})) + + -- GT + local conditions = {{'>', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {8, 2, 4})) +end + +pgroup.test_select_by_full_sharding_key = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = {{'==', 'id', 3}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3})) +end + +pgroup.test_select_with_collations = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Oxford", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "oxford", + }, { + id = 3, name = "elizabeth", last_name = "brown", + age = 46, city = "Oxford", + }, { + id = 4, name = "Jack", last_name = "Sparrow", + age = 35, city = "oxford", + }, { + id = 5, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 6, name = "elizabeth", last_name = "Brown", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- full name index - unicode ci collation (case-insensitive) + local conditions = {{'==', 'name', "Elizabeth"}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 6, 1})) + + -- city - no collation (case-sensitive) + local conditions = {{'==', 'city', "oxford"}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 4})) +end + +pgroup.test_multipart_primary_index = function(g) + local coords = helpers.insert_objects(g, 'coord', { + { x = 0, y = 0 }, -- 1 + { x = 0, y = 1 }, -- 2 + { x = 0, y = 2 }, -- 3 + { x = 1, y = 3 }, -- 4 + { x = 1, y = 4 }, -- 5 + }) + + local conditions = {{'=', 'primary', 0}} + local result_0, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result_0.rows, result_0.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2, 3})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{after = after}) + + foo:close() + return result, err + ]], {conditions, result_0.rows[1]}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2, 3})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{after = after, first = -2}) + + foo:close() + return result, err + ]], {conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2})) + + local new_conditions = {{'=', 'y', 1}, {'=', 'primary', 0}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{after = after, first = -2}) + + foo:close() + return result, err + ]], {new_conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2})) + + local conditions = {{'=', 'primary', {0, 2}}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions) + + foo:close() + return result, err + ]], {conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {3})) + + local conditions_ge = {{'>=', 'primary', 0}} + local result_ge_0, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions_ge}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result_ge_0.rows, result_ge_0.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2, 3, 4, 5})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{after = after,fullscan = true}) + + foo:close() + return result, err + ]], {conditions_ge, result_ge_0.rows[1]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2, 3, 4, 5})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('coord', conditions,{after = after,first = -3}) + + foo:close() + return result, err + ]], {conditions_ge, result_ge_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2})) +end + +pgroup.test_select_partial_result_bad_input = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local conditions = {{'>=', 'age', 33}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = {'id', 'mame'}, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(result, nil) + t.assert_str_contains(err.err, 'Space format doesn\'t contain field named "mame"') +end + +pgroup.test_select_partial_result = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + local expected_customers = { + {id = 3, age = 33, name = "David", city = "Los Angeles"}, + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + expected_customers = { + {id = 3, age = 33, name = "David"}, + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition by non-indexed non-unique non-primary field (city): + conditions = {{'>=', 'city', 'Lo'}} + + -- condition field is not in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", age = 12}, + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", city = "Los Angeles"}, + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) +end + +pgroup.test_select_force_map_call = function(g) + local key = 1 + + local first_bucket_id = g.cluster.main_server.net_box:eval([[ + local vshard = require('vshard') + + local key = ... + return vshard.router.bucket_id_strcrc32(key) + ]], {key}) + + local second_bucket_id, err = helpers.get_other_storage_bucket_id(g.cluster, first_bucket_id) + + t.assert_equals(err, nil) + + local customers = helpers.insert_objects(g, 'customers', { + { + id = key, bucket_id = first_bucket_id, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = key, bucket_id = second_bucket_id, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', {{'==', 'id', 1}}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', {{'==', 'id', 1}}, {force_map_call = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + table.sort(objects, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + t.assert_equals(objects, customers) +end + +pgroup.test_jsonpath = function(g) + helpers.insert_objects(g, 'developers', { + { + id = 1, name = "Alexey", last_name = "Smith", + age = 20, additional = { a = { b = 140 } }, + }, { + id = 2, name = "Sergey", last_name = "Choppa", + age = 21, additional = { a = { b = 120 } }, + }, { + id = 3, name = "Mikhail", last_name = "Crossman", + age = 42, additional = {}, + }, { + id = 4, name = "Pavel", last_name = "White", + age = 51, additional = { a = { b = 50 } }, + }, { + id = 5, name = "Tatyana", last_name = "May", + age = 17, additional = { a = 55 }, + }, + }) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('developers', {{'>=', '[5]', 40}}, + {fields = {'name', 'last_name'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + {id = 3, name = "Mikhail", last_name = "Crossman"}, + {id = 4, name = "Pavel", last_name = "White"}, + } + t.assert_equals(objects, expected_objects) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('developers', {{'<', '["age"]', 21}}, + {fields = {'name', 'last_name'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + {id = 1, name = "Alexey", last_name = "Smith"}, + {id = 5, name = "Tatyana", last_name = "May"}, + } + t.assert_equals(objects, expected_objects) +end + +pgroup.test_jsonpath_index_field = function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 2, + manufacturer = 'VAG', + data = {car = { model = 'BMW', color = 'Black' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 5, + manufacturer = 'FIAT', + data = {car = { model = 'Cadillac', color = 'White' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 17, + manufacturer = 'Ford', + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'Mercedes', color = 'Yellow' }}, + }, + }) + + -- PK jsonpath index + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, + {fields = {'id', 'age'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 2}}, + age = 5, + }, + { + id = {car_id = {signed = 1}}, + age = 2, + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (partial) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('cars', {{'==', 'data_index', 'Yellow'}}, {fields = {'id', 'age'}}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 3}}, + age = 17, + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (full) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('cars', {{'==', 'data_index', {'Yellow', 'Mercedes'}}}, {fields = {'id', 'age'}}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) +end + +pgroup.test_jsonpath_index_field_pagination = function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local cars = helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 5, + manufacturer = 'VAG', + data = {car = { model = 'A', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 17, + manufacturer = 'FIAT', + data = {car = { model = 'B', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 5, + manufacturer = 'Ford', + data = {car = { model = 'C', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'D', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 5}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'E', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 6}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'F', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 7}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'G', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 8}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'H', color = 'Yellow' }}, + }, + }) + + + -- Pagination (primary index) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + rawset(_G, 'foo', foo) + + local result, err = foo:select('cars', nil, {first = 2}) + + return result, err + ]]) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo = rawget(_G, 'foo') + local after = ... + + local result, err = foo:select('cars', nil, {first = 2, after = after}) + return result, err + ]], {result.rows[2]}) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (primary index) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo = rawget(_G, 'foo') + local after = ... + + local result, err = foo:select('cars', nil, {first = -2, after = after}) + + return result, err + ]], {result.rows[1]}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + -- Pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo = rawget(_G, 'foo') + + local result, err = foo:select('cars', {{'==', 'data_index', 'Yellow'}}, {first = 2}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) +end + +pgroup.test_select_timeout = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', nil, {timeout = 1, fullscan = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) +end + +-- gh-220: bucket_id argument is ignored when it cannot be deduced +-- from provided select/pairs conditions. +pgroup.test_select_no_map_reduce = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + -- bucket_id is 477, storage is s-2 + id = 1, name = 'Elizabeth', last_name = 'Jackson', + age = 12, city = 'New York', + }, { + -- bucket_id is 401, storage is s-2 + id = 2, name = 'Mary', last_name = 'Brown', + age = 46, city = 'Los Angeles', + }, { + -- bucket_id is 2804, storage is s-1 + id = 3, name = 'David', last_name = 'Smith', + age = 33, city = 'Los Angeles', + }, { + -- bucket_id is 1161, storage is s-2 + id = 4, name = 'William', last_name = 'White', + age = 81, city = 'Chicago', + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local router = g.cluster:server('router').net_box + local map_reduces_before = helpers.get_map_reduces_stat(router, 'customers') + + -- Case: no conditions, just bucket id. + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', nil, {bucket_id = 2804, timeout = 1, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + }) + + local map_reduces_after_1 = helpers.get_map_reduces_stat(router, 'customers') + local diff_1 = map_reduces_after_1 - map_reduces_before + t.assert_equals(diff_1, 0, 'Select request was not a map reduce') + + -- Case: EQ on secondary index, which is not in the sharding + -- index (primary index in the case). + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select( 'customers', {{'==', 'age', 81}}, {bucket_id = 1161, timeout = 1}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {4, 1161, 'William', 'White', 81, 'Chicago'}, + }) + + local map_reduces_after_2 = helpers.get_map_reduces_stat(router, 'customers') + local diff_2 = map_reduces_after_2 - map_reduces_after_1 + t.assert_equals(diff_2, 0, 'Select request was not a map reduce') +end + +pgroup.test_select_yield_every_0 = function(g) + local resp, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select('customers', nil, { yield_every = 0, fullscan = true }) + + foo:close() + return result, err + ]]) + t.assert_equals(resp, nil) + t.assert_str_contains(err.err, "yield_every should be > 0") +end + +pgroup.test_stop_select = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local _, _ = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo = crud.readview({name = 'foo'}) + rawset(_G, 'foo', foo) + ]]) + + g.cluster:server('s2-master'):stop() + local _, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo = rawget(_G, 'foo', foo) + local result, err = foo:select('customers', nil, {fullscan = true}) + return result, err + ]]) + t.assert_str_contains(err.err, 'Connection refused') + g.cluster:server('s2-master'):start() + local _, _ = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local foo = rawget(_G, 'foo', foo) + foo:close() + return nil, nil + ]]) +end + +pgroup.test_select_switch_master = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local _, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local temp, err = crud.readview({name = 'temp'}) + if err ~= nil then + return nil, err + end + rawset(_G, 'temp', temp) + return nil, err + ]]) + t.assert_equals(err, nil) + + local replicasets = helpers.get_test_replicasets() + set_master(g.cluster, replicasets[2].uuid, replicasets[2].servers[2].instance_uuid) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local temp = rawget(_G, 'temp') + local result, err = temp:select('customers', nil, {fullscan = true}) + + temp:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_switch_master_first = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local temp, err = crud.readview({name = 'temp'}) + if err ~= nil then + return nil, err + end + local result, err = temp:select('customers', nil, {first = 2}) + rawset(_G, 'temp', temp) + return result, err + ]]) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(obj.rows, obj.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 2})) + + local replicasets = helpers.get_test_replicasets() + set_master(g.cluster, replicasets[3].uuid, replicasets[3].servers[2].instance_uuid) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local temp = rawget(_G, 'temp') + local result, err = temp:select('customers', nil, {fullscan = true}) + + temp:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua index 789cce39..812ff2d3 100644 --- a/test/integration/stats_test.lua +++ b/test/integration/stats_test.lua @@ -494,6 +494,13 @@ local prepare_select_data = function(g) }) end +local is_readview_supported = function() + if (not helpers.tarantool_version_at_least(2, 11, 0)) + or (not require('luatest.tarantool').is_enterprise_package()) then + t.skip('Readview is supported only for Tarantool Enterprise starting from v2.11.0') + end +end + local select_cases = { select_by_primary_index = { func = 'crud.select', @@ -766,6 +773,82 @@ for name, case in pairs(select_cases) do end end +for name, case in pairs(select_cases) do + local test_name = ('test_%s_details_readview'):format(name) + pgroup.before_test(test_name, is_readview_supported) + pgroup.before_test(test_name, prepare_select_data) + + pgroup[test_name] = function(g) + local op = 'select' + local space_name = space_name + + -- Collect stats before call. + local stats_before = get_stats(g, space_name) + t.assert_type(stats_before, 'table') + + -- Call operation. + local _, err + if case.eval ~= nil then + _, err = g.router:eval([[ + local crud = require('crud') + local conditions, space_name = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + + local result = {} + for _, v in foo:pairs(space_name, conditions, { batch_size = 1 }) do + table.insert(result, v) + end + foo:close() + + return result + + ]], {case.conditions, space_name}) + else + _, err = g.router:eval([[ + local crud = require('crud') + local conditions, space_name = ... + + local foo, err = crud.readview({name = 'foo'}) + if err ~= nil then + return nil, err + end + local result, err = foo:select(space_name, conditions) + + foo:close() + + return result, err + ]], {case.conditions, space_name}) + end + + t.assert_equals(err, nil) + + -- Collect stats after call. + local stats_after = get_stats(g, space_name) + t.assert_type(stats_after, 'table') + + local op_before = set_defaults_if_empty(stats_before, op) + local details_before = op_before.details + local op_after = set_defaults_if_empty(stats_after, op) + local details_after = op_after.details + + local tuples_fetched_diff = details_after.tuples_fetched - details_before.tuples_fetched + t.assert_equals(tuples_fetched_diff, case.tuples_fetched, + 'Expected count of tuples fetched') + + local tuples_lookup_diff = details_after.tuples_lookup - details_before.tuples_lookup + t.assert_equals(tuples_lookup_diff, case.tuples_lookup, + 'Expected count of tuples looked up on storage') + + local map_reduces_diff = details_after.map_reduces - details_before.map_reduces + t.assert_equals(map_reduces_diff, case.map_reduces, + 'Expected count of map reduces planned') + end +end + pgroup.before_test( 'test_role_reload_do_not_reset_observations',