Skip to content

Commit

Permalink
sharding func: fix specifying vshard sharding funcs
Browse files Browse the repository at this point in the history
Starting from 0.11.0 user can specify sharding func to
calculate bucket_id with sharding func definition as a part of
DDL schema or insert manually to the space `_ddl_sharding_func`.

Right now ddl fails with setting schema with vshard sharding function.
But even if this bug is fixed, there is also a bug on CRUD side.
Inserting manually to the space `_ddl_sharding_func` showed
that CRUD search vshard sharding func in `_G` but this
approach doesn't work with vshard case.

This patch allows to specify `vshard` sharding func
inserting manually to the space `_ddl_sharding_func`.

Closes #314
  • Loading branch information
AnaNek committed Jul 27, 2022
1 parent a7a3413 commit 1d98a49
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 2 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.

### Fixed

- Fix specifying `vshard` sharding funcs (#314).

## [0.12.1] - 21-07-22

### Fixed
Expand Down
18 changes: 17 additions & 1 deletion crud/common/sharding/sharding_func.lua
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ local ShardingFuncError = errors.new_class('ShardingFuncError', {capture_stack

local sharding_func_module = {}

local sharding_module_names = {
['vshard'] = true,
}

local function is_callable(object)
if type(object) == 'function' then
return true
Expand Down Expand Up @@ -43,13 +47,25 @@ end
local function get_function_from_G(func_name)
local chunks = string.split(func_name, '.')
local sharding_func = _G
local sharding_module = false

if sharding_module_names[chunks[1]] then
sharding_func = require(chunks[1])
sharding_module = true
table.remove(chunks, 1)
end

-- check is the each chunk an identifier
for _, chunk in pairs(chunks) do
if not utils.check_name_isident(chunk) or sharding_func == nil then
return nil
end
sharding_func = rawget(sharding_func, chunk)

if sharding_module then
sharding_func = sharding_func[chunk]
else
sharding_func = rawget(sharding_func, chunk)
end
end

return sharding_func
Expand Down
7 changes: 6 additions & 1 deletion test/entrypoint/srv_ddl.lua
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ package.preload['customers-storage'] = function()
local customers_G_func_schema = table.deepcopy(customers_id_key_schema)
customers_G_func_schema.sharding_func = 'some_module.sharding_func'

local customers_vshard_func_schema = table.deepcopy(customers_id_key_schema)

local schema = {
spaces = {
customers = customers_id_schema,
Expand All @@ -173,6 +175,7 @@ package.preload['customers-storage'] = function()
customers_name_age_key_three_fields_index = customers_name_age_key_three_fields_index_schema,
customers_G_func = customers_G_func_schema,
customers_body_func = customers_body_func_schema,
customers_vshard_func = customers_vshard_func_schema,
}
}

Expand All @@ -188,7 +191,9 @@ package.preload['customers-storage'] = function()
box.space['_ddl_sharding_key']:update(space_name, {{'=', fieldno_sharding_key, sharding_key_def}})
end)
rawset(_G, 'set_sharding_func', function(space_name, fieldno_sharding_func, sharding_func_def)
box.space['_ddl_sharding_func']:update(space_name, {{'=', fieldno_sharding_func, sharding_func_def}})
local record = {space_name, box.NULL, box.NULL}
record[fieldno_sharding_func] = sharding_func_def
box.space['_ddl_sharding_func']:replace(record)
end)
end,
}
Expand Down
66 changes: 66 additions & 0 deletions test/integration/ddl_sharding_func_test.lua
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ cache_group.after_all(function(g) helpers.stop_cluster(g.cluster) end)
cache_group.before_each(function(g)
helpers.truncate_space_on_cluster(g.cluster, 'customers_G_func')
helpers.truncate_space_on_cluster(g.cluster, 'customers_body_func')
helpers.truncate_space_on_cluster(g.cluster, 'customers_vshard_func')
end)

pgroup.test_insert_object = function(g)
Expand Down Expand Up @@ -938,3 +939,68 @@ pgroup.test_gh_278_count_with_explicit_bucket_id_and_ddl = function(g)
t.assert_is_not(obj, nil)
t.assert_equals(obj, 1)
end

local vshard_cases = {
mpcrc32 = {
sharding_func_name = 'vshard.router.bucket_id_mpcrc32',
bucket_id = 1614,
srv_with_data = 's1-master',
srv_without_data = 's2-master',
},
strcrc32 = {
sharding_func_name = 'vshard.router.bucket_id_strcrc32',
bucket_id = 477,
srv_with_data = 's2-master',
srv_without_data = 's1-master',
}
}

for name, case in pairs(vshard_cases) do
local test_name = ('test_vshard_%s_sharding_func'):format(name)

cache_group[test_name] = function(g)
local fieldno_sharding_func_name = 2
local space_name = 'customers_vshard_func'

helpers.call_on_servers(g.cluster, {'s1-master', 's2-master'}, function(server)
server.net_box:call('set_sharding_func',
{space_name, fieldno_sharding_func_name, case.sharding_func_name})
end)

local record_exist, err = helpers.update_sharding_func_cache(g.cluster, space_name)
t.assert_equals(err, nil)
t.assert_equals(record_exist, true)

-- Insert a tuple.
local result, err = g.cluster.main_server.net_box:call(
'crud.insert', {space_name, {1, box.NULL, 'Ivan', 25}})
t.assert_equals(err, nil)
t.assert_equals(result.metadata, {
{is_nullable = false, name = 'id', type = 'unsigned'},
{is_nullable = false, name = 'bucket_id', type = 'unsigned'},
{is_nullable = false, name = 'name', type = 'string'},
{is_nullable = false, name = 'age', type = 'number'},
})
t.assert_equals(#result.rows, 1)
t.assert_equals(result.rows[1], {1, case.bucket_id, 'Ivan', 25})

-- There is a tuple on server that we inserted before using crud.insert().
local conn_srv_with_data = g.cluster:server(case.srv_with_data).net_box
local result = conn_srv_with_data.space[space_name]:get({1})
t.assert_equals(result, {1, case.bucket_id, 'Ivan', 25})

-- There is no tuple on server that we inserted before using crud.insert().
local conn_srv_without_data = g.cluster:server(case.srv_without_data).net_box
local result = conn_srv_without_data.space[space_name]:get({1})
t.assert_equals(result, nil)

local conditions = {{'==', 'id', 1}}
local result, err = g.cluster.main_server.net_box:call('crud.select', {
space_name, conditions,
})

t.assert_equals(err, nil)
t.assert_equals(#result.rows, 1)
t.assert_equals(result.rows[1], {1, case.bucket_id, 'Ivan', 25})
end
end
3 changes: 3 additions & 0 deletions test/integration/ddl_sharding_key_test.lua
Original file line number Diff line number Diff line change
Expand Up @@ -898,6 +898,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g)
customers = {parts = {{fieldno = 1}}},
customers_G_func = {parts = {{fieldno = 1}}},
customers_body_func = {parts = {{fieldno = 1}}},
customers_vshard_func = {parts = {{fieldno = 1}}},
customers_age_key = {parts = {{fieldno = 4}}},
customers_name_age_key_different_indexes = {parts = {{fieldno = 3}, {fieldno = 4}}},
customers_name_age_key_three_fields_index = {parts = {{fieldno = 3}, {fieldno = 4}}},
Expand Down Expand Up @@ -925,6 +926,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g)
customers = {parts = {{fieldno = 1}}},
customers_G_func = {parts = {{fieldno = 1}}},
customers_body_func = {parts = {{fieldno = 1}}},
customers_vshard_func = {parts = {{fieldno = 1}}},
customers_age_key = {parts = {{fieldno = 4}}},
customers_name_age_key_different_indexes = {parts = {{fieldno = 3}, {fieldno = 4}}},
customers_name_age_key_three_fields_index = {parts = {{fieldno = 3}, {fieldno = 4}}},
Expand All @@ -951,6 +953,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g)
customers = {parts = {{fieldno = 1}}},
customers_G_func = {parts = {{fieldno = 1}}},
customers_body_func = {parts = {{fieldno = 1}}},
customers_vshard_func = {parts = {{fieldno = 1}}},
customers_age_key = {parts = {{fieldno = 4}}},
customers_name_age_key_different_indexes = {parts = {{fieldno = 3}, {fieldno = 4}}},
customers_name_age_key_three_fields_index = {parts = {{fieldno = 3}, {fieldno = 4}}},
Expand Down

0 comments on commit 1d98a49

Please sign in to comment.