diff --git a/test/luatest_helpers/cluster.lua b/test/luatest_helpers/cluster.lua index 43e3479f..e247bad8 100644 --- a/test/luatest_helpers/cluster.lua +++ b/test/luatest_helpers/cluster.lua @@ -42,6 +42,7 @@ function Cluster:drop() server:cleanup() end end + self.servers = {} end function Cluster:get_index(server) diff --git a/test/luatest_helpers/vtest.lua b/test/luatest_helpers/vtest.lua index f9219439..551ae898 100644 --- a/test/luatest_helpers/vtest.lua +++ b/test/luatest_helpers/vtest.lua @@ -148,7 +148,8 @@ end -- -- Build new cluster by a given config. -- -local function cluster_new(g, cfg) +local function cluster_new(g, cfg, server_config) + server_config = server_config or {} if not g.cluster then g.cluster = cluster:new({}) end @@ -204,6 +205,7 @@ local function cluster_new(g, cfg) local server = g.cluster:build_server({ alias = replica_name, box_cfg = box_cfg, + env = server_config.env, }, 'storage.lua') g[replica_name] = server -- VShard specific details to use in various helper functions. diff --git a/test/router-luatest/map_callrw_test.lua b/test/router-luatest/map_callrw_test.lua index 27482e2e..0d251dfc 100644 --- a/test/router-luatest/map_callrw_test.lua +++ b/test/router-luatest/map_callrw_test.lua @@ -691,3 +691,180 @@ g.test_map_callrw_with_cdata_bucket_id = function(cg) ilt.assert_not(err) end) end + +g.test_full_map_callrw_with_numeric_bucket_ids = function(cg) + local res = router_do_map(cg.router, {123}, { + mode = 'full', + timeout = vtest.wait_timeout, + bucket_ids = {1, 2, 3} + }) + t.assert(res.err) + t.assert_equals(res.err.message, 'Router can\'t execute map_callrw ' .. + 'with \'full\' mode and numeric bucket_ids') + t.assert_not(res.err_id) + t.assert_not(res.val) +end + +g.test_partial_map_callrw_with_nil_bucket_ids = function(cg) + local res = router_do_map(cg.router, {123}, { + mode = 'partial', + timeout = vtest.wait_timeout + }) + t.assert(res.err) + t.assert_equals(res.err.message, 'Router can\'t execute map_callrw ' .. + 'with \'partial\' mode and nil bucket_ids') + t.assert_not(res.err_id) + t.assert_not(res.val) +end + +local function make_do_map_tracking_bucket_ids(cg) + -- We override 'do_map' function on storages in order to check that + -- default arguments and bucket arguments were successfully passed into + -- destination storages according to mode and bucket_ids options. + vtest.cluster_exec_each_master(cg, function() + rawset(_G, 'old_do_map', _G.do_map) + rawset(_G, 'do_map', function(args, bucket_args) + ilt.assert_gt(require('vshard.storage.ref').count, 0) + return {ivutil.replicaset_uuid(), + {args = args, b_args = bucket_args}} + end) + end) +end + +local function reset_do_map_to_old_state(cg) + vtest.cluster_exec_each_master(cg, function() + rawset(_G, 'do_map', _G.old_do_map) + end) +end + +g.test_full_map_callrw_with_split_args = function(cg) + make_do_map_tracking_bucket_ids(cg) + local bid1 = vtest.storage_first_bucket(g.replica_1_a) + local bid2 = vtest.storage_first_bucket(g.replica_2_a) + local bid3 = vtest.storage_first_bucket(g.replica_3_a) + + local res = router_do_map(cg.router, {0}, { + mode = 'full', + timeout = vtest.wait_timeout, + bucket_ids = {[bid1] = {111}, [bid2] = {222}, [bid3] = {333}} + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, { + [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0, b_args = {[bid1] = {111}}}}}, + [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}}, + [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0, b_args = {[bid3] = {333}}}}}, + }) + + res = router_do_map(cg.router, {0}, { + mode = 'full', + timeout = vtest.wait_timeout, + bucket_ids = {[bid2] = {222}} + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, { + [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}}, + [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}}, + [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0}}}, + }) + reset_do_map_to_old_state(cg) +end + +g.test_full_map_callrw_without_bucket_ids = function(cg) + make_do_map_tracking_bucket_ids(cg) + local res = router_do_map(cg.router, {0}, { + mode = 'full', + timeout = vtest.wait_timeout + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, { + [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}}, + [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0}}}, + [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0}}}, + }) + reset_do_map_to_old_state(cg) +end + +g.test_partial_map_callrw_with_numeric_bucket_ids = function(cg) + make_do_map_tracking_bucket_ids(cg) + local bid1 = vtest.storage_first_bucket(g.replica_1_a) + local res = router_do_map(cg.router, {0}, { + mode = 'partial', + timeout = vtest.wait_timeout, + bucket_ids = {bid1} + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, {[cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}}}) + reset_do_map_to_old_state(cg) +end + +g.test_partial_map_callrw_with_split_args = function(cg) + make_do_map_tracking_bucket_ids(cg) + local bid1 = vtest.storage_first_bucket(g.replica_1_a) + local bid2 = vtest.storage_first_bucket(g.replica_2_a) + local res = router_do_map(cg.router, {0}, { + mode = 'partial', + timeout = vtest.wait_timeout, + bucket_ids = {[bid1] = {111}, [bid2] = {222}} + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, { + [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0, b_args = {[bid1] = {111}}}}}, + [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}}, + }) + reset_do_map_to_old_state(cg) +end + +local function move_bucket(src_storage, dest_storage, bucket_id) + src_storage:exec(function(bucket_id, replicaset_id) + t.helpers.retrying({timeout = 60}, function() + local res, err = ivshard.storage.bucket_send(bucket_id, + replicaset_id) + t.assert_not(err) + t.assert(res) + end) + end, {bucket_id, dest_storage:replicaset_uuid()}) + src_storage:exec(function(bucket_id) + t.helpers.retrying({timeout = 10}, function() + t.assert_equals(box.space._bucket:select(bucket_id), {}) + end) + end, {bucket_id}) + dest_storage:exec(function(bucket_id) + t.helpers.retrying({timeout = 10}, function() + t.assert_equals(box.space._bucket:get(bucket_id).status, 'active') + end) + end, {bucket_id}) +end + +g.test_full_map_callrw_with_split_args_and_broken_cache = function(cg) + make_do_map_tracking_bucket_ids(cg) + cg.router:exec(function() + ivshard.router.internal.errinj.ERRINJ_LONG_DISCOVERY = true + ivshard.router.discovery_wakeup() + end) + + local moved_bucket = vtest.storage_first_bucket(cg.replica_1_a) + move_bucket(g.replica_1_a, g.replica_2_a, moved_bucket) + local res = router_do_map(cg.router, {0}, { + mode = 'partial', + timeout = vtest.wait_timeout, + bucket_ids = {[moved_bucket] = {111}} + }) + t.assert_not(res.err) + t.assert_not(res.err_id) + t.assert_equals(res.val, { + [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, + b_args = {[moved_bucket] = {111}}}}}, + }) + + cg.router:exec(function() + ivshard.router.internal.errinj.ERRINJ_LONG_DISCOVERY = false + ivshard.router.discovery_wakeup() + end) + move_bucket(g.replica_2_a, g.replica_1_a, moved_bucket) + reset_do_map_to_old_state(cg) +end diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua index c1ae28d7..0c43279f 100644 --- a/test/router-luatest/reload_test.lua +++ b/test/router-luatest/reload_test.lua @@ -28,14 +28,20 @@ local cfg_template = { } local global_cfg -g.before_all(function() +local function get_config_for_specific_version(g) + local path = g.vshard_copy_path_load + local lua_path = string.format('%s/?.lua;%s/?/init.lua;', path, path) + -- Force 'require' to use new directory + return {env = {['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH')}} +end + +local function prepare_temp_vshard_workspace(g) -- Override of the built in modules is available only since 2.11.0. t.run_only_if(vutil.version_is_at_least(2, 11, 0, nil, 0, 0)) global_cfg = vtest.config_new(cfg_template) -- The test works in the following directory - local vardir = vtest.vardir or fio.tempdir() - g.vshard_copy_path_load = vardir .. '/vshard_copy' + g.vshard_copy_path_load = '/vshard_copy' t.assert_equals(fio.mkdir(g.vshard_copy_path_load), true) -- -- Tarantool searches for compilation units in the following order: @@ -53,7 +59,7 @@ g.before_all(function() -- So we can use package.setsearchroot() to change cwd, luatest's `chdir` -- or override. The last one is used here, since it's the easiest. -- - g.vshard_copy_path = vardir .. '/vshard_copy/override' + g.vshard_copy_path = g.vshard_copy_path_load .. '/override' t.assert_equals(fio.mkdir(g.vshard_copy_path), true) -- Copy source to the temporary directory t.assert_equals(fio.mkdir(g.vshard_copy_path .. '/.git'), true) @@ -62,13 +68,21 @@ g.before_all(function() -- Hash of the latest commit for testing router on the latest version. g.latest_hash = git_util.log_hashes({args = '-1', dir = vtest.sourcedir})[1] +end + +local function remove_temp_vshard_workspace(g) + local is_success, err = fio.rmtree(g.vshard_copy_path_load) + t.assert(is_success) + t.assert_not(err) +end - -- No need to reload storages. Just run them on the latest version. - vtest.cluster_new(g, global_cfg) +local function create_cluster_on_specific_version(g, hash) + git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path}) + local server_config = get_config_for_specific_version(g) + vtest.cluster_new(g, global_cfg, server_config) vtest.cluster_bootstrap(g, global_cfg) vtest.cluster_rebalancer_disable(g) - -- Basic storage configuration vtest.cluster_exec_each_master(g, function() local test = box.schema.space.create('test', {format = { {'id', 'unsigned'}, @@ -78,7 +92,6 @@ g.before_all(function() test:create_index('primary') test:create_index('bucket_id', {unique = false, parts = {2}}) end) - vtest.cluster_exec_each(g, function() rawset(_G, 'insert', function(space_name, tuple) return box.space[space_name]:insert(tuple) @@ -86,10 +99,21 @@ g.before_all(function() rawset(_G, 'get', function(space_name, key) return box.space[space_name]:get(key) end) + rawset(_G, 'do_map', function(args, bucket_args) + ilt.assert_gt(require('vshard.storage.ref').count, 0) + return {ivutil.replicaset_uuid(), + {args = args, b_args = bucket_args}} + end) end) +end + +g.before_all(function() + prepare_temp_vshard_workspace(g) + create_cluster_on_specific_version(g, g.latest_hash) end) g.after_all(function() + remove_temp_vshard_workspace(g) g.cluster:drop() end) @@ -115,55 +139,56 @@ end -- * Checkout to old version; -- * Create and start a router; -- 3. Test smth on the old version; --- 4. Invoke reload_router(): +-- 4. Invoke reload_server(): -- * Checkout to the latest version; -- * Reload the module. -- 5. Test smth on the new version -- 6. Drop a router with vtest.drop_instance -- -local function create_router_at(hash) +local function create_router_at(g, hash) git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path}) - local path = g.vshard_copy_path_load - local lua_path = string.format("%s/?.lua;%s/?/init.lua;", path, path) - local router = vtest.router_new(g, 'router', nil, { - env = { - -- Force 'require' to use new directory - ['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH') - }, - }) + local server_config = get_config_for_specific_version(g) + local router = vtest.router_new(g, 'router', nil, server_config) router_cfg(router, global_cfg) return router end -- --- Reloads router. If service_name is provided, then +-- Reloads server (router or storage). If service_name is provided, then -- the function also waits until the service is restarted. -local function reload_router(router, service_name) +-- +local function reload_server(g, server, server_type, service_name) git_util.exec('checkout', {args = g.latest_hash .. ' -f', dir = g.vshard_copy_path}) - router:exec(function(service_name) + server:exec(function(server_type, service_name) local service - local internal = ivshard.router.internal + local internal = ivshard[server_type].internal if service_name ~= nil then - service = internal.static_router[service_name] + if server_type == 'router' then + service = internal.static_router[service_name] + else + service = internal[service_name] + end ilt.assert_not_equals(service, nil) end - ilt.assert_equals(ivshard.router.module_version(), 0) - package.loaded['vshard.router'] = nil - ivshard.router = require('vshard.router') + local package_name = string.format('vshard.%s', server_type) + ilt.assert_equals(ivshard[server_type].module_version(), 0) + package.loaded[package_name] = nil + ivshard[server_type] = require(package_name) _G.ivconst = require('vshard.consts') - ilt.assert_equals(ivshard.router.module_version(), 1) + ilt.assert_equals(ivshard[server_type].module_version(), 1) if service ~= nil then ilt.helpers.retrying({timeout = ivtest.wait_timeout, delay = ivtest.busy_step}, function() - if service == internal.static_router[service_name] then + if service == internal.static_router[service_name] or + service == internal[service_name] then error('Service have not been reloaded yet') end end) end - end, {service_name}) + end, {server_type, service_name}) end local function test_basic_template(router) @@ -182,10 +207,10 @@ g.test_basic = function(g) -- Latest meaningful commit: -- "router: fix reload problem with global function refs". local hash = '139223269cddefe2ba4b8e9f6e44712f099f4b35' - local router = create_router_at(hash) + local router = create_router_at(g, hash) router_assert_version_equals(router, nil) test_basic_template(router) - reload_router(router) + reload_server(g, router, 'router') router_assert_version_equals(router, vconsts.VERSION) test_basic_template(router) vtest.drop_instance(g, router) @@ -225,10 +250,10 @@ g.test_discovery = function(g) -- the test cannot be run without it: -- "router: add saving of background service statuses". local hash = 'f5f386a5a35e6e5efd8f4f2ed1b3d208fdae9095' - local router = create_router_at(hash) + local router = create_router_at(g, hash) router_assert_version_equals(router, nil) test_discovery_template(g, router) - reload_router(router, 'discovery_service') + reload_server(g, router, 'router', 'discovery_service') router_assert_version_equals(router, vconsts.VERSION) test_discovery_template(g, router) vtest.drop_instance(g, router) @@ -313,7 +338,7 @@ g.test_master_search = function(g) -- the test cannot be run without it: -- "router: add saving of background service statuses". local hash = 'f5f386a5a35e6e5efd8f4f2ed1b3d208fdae9095' - local router = create_router_at(hash) + local router = create_router_at(g, hash) router_assert_version_equals(router, nil) -- Enable auto master search @@ -327,7 +352,7 @@ g.test_master_search = function(g) local auto_master_cfg = vtest.config_new(auto_master_cfg_template) test_master_search_template(g, router, auto_master_cfg) - reload_router(router) + reload_server(g, router, 'router') router_assert_version_equals(router, vconsts.VERSION) -- Wait for old master_search service to be stopped. router:exec(function() @@ -343,3 +368,79 @@ g.test_master_search = function(g) test_master_search_template(g, router, auto_master_cfg) vtest.drop_instance(g, router) end + +local g2 = t.group('reload_storage') + +g2.before_all(function() + prepare_temp_vshard_workspace(g2) + -- Full map_callrw with split args was introduced just right after + -- this commit. We need to test the behavior of map_callrw on old + -- storage versions in order to check that there will be no crashes + -- of storages due to changes in storage_ref_* functions: + -- "Bump version to 0.1.41" + create_cluster_on_specific_version(g2, + '42081581f25e671a17fddb7f9873ddcf5b0130c6') +end) + +g2.after_all(function() + remove_temp_vshard_workspace(g2) + g2.cluster:drop() +end) + +g2.test_map_callrw = function(g2) + local rs_uuids = {g2.replica_1_a:replicaset_uuid(), + g2.replica_2_a:replicaset_uuid()} + local router = vtest.router_new(g2, 'router') + router_cfg(router, global_cfg) + -- The latest router and old masters + router:exec(function(rs1_uuid, rs2_uuid) + -- Full map_callrw + local res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'}, + {timeout = ivtest.wait_timeout, mode = 'full'}) + t.assert_not(err) + t.assert_not(err_id) + t.assert_equals(res, { + [rs1_uuid] = {{rs1_uuid, {args = 'arg_1'}}}, + [rs2_uuid] = {{rs2_uuid, {args = 'arg_1'}}}, + }) + -- Full map_callrw with split args + res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'}, + {timeout = 3, mode = 'full', bucket_ids = {[1] = {'b_arg_1'}}}) + t.assert_equals(err.name, 'UNSUPPORTED') + t.assert_not(err_id) + t.assert_not(res) + -- Partial map_callrw with numeric buckets' args + res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'}, + {timeout = 3, mode = 'partial', bucket_ids = {1,}}) + t.assert_not(err) + t.assert_not(err_id) + t.assert_equals(res, { + [rs1_uuid] = {{rs1_uuid, {args = 'arg_1'}}}, + }) + -- Partial map_callrw with split args + res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'}, + {timeout = 3, mode = 'partial', bucket_ids = {[1] = {'b_arg_1'}}}) + t.assert_not(err) + t.assert_not(err_id) + t.assert_equals(res, { + [rs1_uuid] = {{rs1_uuid, {args = 'arg_1', + b_args = {{'b_arg_1'}}}}}, + }) + end, rs_uuids) + for _, storage in pairs({g2.replica_1_a, g2.replica_2_a}) do + reload_server(g2, storage, 'storage') + end + -- The latest router and latest masters + router:exec(function(rs1_uuid, rs2_uuid) + -- Full map_callrw with split args + local res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'}, + {timeout = 3, mode = 'full', bucket_ids = {[1] = {'b_arg_1'}}}) + t.assert_not(err) + t.assert_not(err_id) + t.assert_equals(res, { + [rs1_uuid] = {{rs1_uuid, {args = 'arg_1', + b_args = {{'b_arg_1'}}}}}, + [rs2_uuid] = {{rs2_uuid, {args = 'arg_1'}}}, + }) + end, rs_uuids) +end diff --git a/test/storage-luatest/storage_1_test.lua b/test/storage-luatest/storage_1_test.lua index 95a99c44..8f61f475 100644 --- a/test/storage-luatest/storage_1_test.lua +++ b/test/storage-luatest/storage_1_test.lua @@ -250,14 +250,14 @@ test_group.test_ref_with_buckets_basic = function(g) res, err = ivshard.storage._call( 'storage_ref_make_with_buckets', rid, iwait_timeout, {}) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {moved = {}}) + ilt.assert_equals(res, {moved = {}, total = 10}) ilt.assert_equals(lref.count, 0) -- Check for a single ok bucket. res, err = ivshard.storage._call( 'storage_ref_make_with_buckets', rid, iwait_timeout, {bids[1]}) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {is_done = true, moved = {}}) + ilt.assert_equals(res, {is_done = true, moved = {}, total = 10}) ilt.assert_equals(lref.count, 1) _, err = ivshard.storage._call('storage_unref', rid) ilt.assert_equals(err, nil) @@ -268,7 +268,7 @@ test_group.test_ref_with_buckets_basic = function(g) 'storage_ref_make_with_buckets', rid, iwait_timeout, {bids[1], bids[2]}) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {is_done = true, moved = {}}) + ilt.assert_equals(res, {is_done = true, moved = {}, total = 10}) _, err = ivshard.storage._call('storage_unref', rid) ilt.assert_equals(err, nil) @@ -277,7 +277,7 @@ test_group.test_ref_with_buckets_basic = function(g) 'storage_ref_make_with_buckets', rid, iwait_timeout, {bids[1], bids[1]}) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {is_done = true, moved = {}}) + ilt.assert_equals(res, {is_done = true, moved = {}, total = 10}) ilt.assert_equals(lref.count, 1) _, err = ivshard.storage._call('storage_unref', rid) ilt.assert_equals(err, nil) @@ -295,7 +295,8 @@ test_group.test_ref_with_buckets_basic = function(g) {id = bucket_count + 1}, {id = bucket_count + 2}, {id = bucket_count + 3}, - } + }, + total = 10, }) _, err = ivshard.storage._call('storage_unref', rid) ilt.assert_equals(err, nil) @@ -308,7 +309,7 @@ test_group.test_ref_with_buckets_basic = function(g) {bucket_count + 1, bucket_count + 2} ) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {moved = { + ilt.assert_equals(res, {total = 10, moved = { {id = bucket_count + 1}, {id = bucket_count + 2}, }}) @@ -355,7 +356,7 @@ test_group.test_ref_with_buckets_return_last_known_dst = function(g) local res, err = ivshard.storage._call( 'storage_ref_make_with_buckets', rid, iwait_timeout, {bid}) ilt.assert_equals(err, nil) - ilt.assert_equals(res, {moved = {{ + ilt.assert_equals(res, {total = 10, moved = {{ id = bid, dst = id, status = ivconst.BUCKET.SENT, @@ -406,6 +407,7 @@ test_group.test_ref_with_buckets_move_part_while_referencing = function(g) ilt.assert_equals(res, { moved = {{id = bids[2], dst = id}}, is_done = true, + total = 8, }) -- Ref was done, because at least one bucket was ok. ilt.assert_equals(lref.count, 1) @@ -458,7 +460,8 @@ test_group.test_ref_with_buckets_move_all_while_referencing = function(g) moved = { {id = bids[1], dst = id}, {id = bids[2], dst = id}, - } + }, + total = 8, }) -- Ref was not done, because all the buckets moved out. ilt.assert_equals(lref.count, 0) diff --git a/test/upgrade/upgrade.result b/test/upgrade/upgrade.result index b1010604..0272d987 100644 --- a/test/upgrade/upgrade.result +++ b/test/upgrade/upgrade.result @@ -181,6 +181,7 @@ vshard.storage._call('test_api', 1, 2, 3) | - storage_bucket_checkpoint | - storage_map | - storage_ref + | - storage_ref_check_existent | - storage_ref_check_with_buckets | - storage_ref_make_with_buckets | - storage_unref diff --git a/vshard/consts.lua b/vshard/consts.lua index 80c49f5b..84126fbb 100644 --- a/vshard/consts.lua +++ b/vshard/consts.lua @@ -29,6 +29,11 @@ return { RED = 3, }, + MAP_CALLRW_MODE = { + FULL = 'full', + PARTIAL = 'partial', + }, + REPLICATION_THRESHOLD_SOFT = 1, REPLICATION_THRESHOLD_HARD = 5, REPLICATION_THRESHOLD_FAIL = 10, diff --git a/vshard/error.lua b/vshard/error.lua index 5bbc3d5d..b7584e40 100644 --- a/vshard/error.lua +++ b/vshard/error.lua @@ -212,6 +212,11 @@ local error_message_template = { msg = "Master %s of replicaset %s has not synchronized yet", args = {'replica', 'replicaset'}, }, + [43] = { + name = 'UNSUPPORTED', + msg = 'Can\'t perform %s. The instance should be upgraded', + args = {'operation'}, + }, } -- diff --git a/vshard/router/init.lua b/vshard/router/init.lua index 098bca2c..d381f258 100644 --- a/vshard/router/init.lua +++ b/vshard/router/init.lua @@ -29,6 +29,8 @@ local map_serializer = { __serialize = 'map' } local future_wait = util.future_wait local msgpack_is_object = lmsgpack.is_object +local MAP_CALLRW_FULL = consts.MAP_CALLRW_MODE.FULL +local MAP_CALLRW_PARTIAL = consts.MAP_CALLRW_MODE.PARTIAL if not util.feature.msgpack_object then local msg = 'Msgpack object feature is not supported by current '.. @@ -772,59 +774,176 @@ local function router_call(router, bucket_id, opts, ...) end -- --- Perform Ref stage of the Ref-Map-Reduce process on all the known replicasets. +-- Prepares a cluster before sending refs to remote storages: +-- 1) Groups the buckets by replicasets according to the router's cache +-- in case of partial map_callrw or full map_callrw with split args; +-- 2) Builds a table of replicasets on which refs should be sent; +-- 3) Waits necessary masters until all connections are established; -- -local function router_ref_storage_all(router, timeout) - local replicasets = router.replicasets +local function router_map_callrw_prepare(router, timeout, mode, bucket_ids) + local err, err_id + local replicasets_to_wait, grouped_buckets = {}, {} + local replicasets_all = router.replicasets local deadline = fiber_clock() + timeout - local err, err_id, res - local futures = {} + if bucket_ids then + -- Group the buckets by replicasets according to the router cache. + grouped_buckets, err = buckets_group(router, bucket_ids, timeout) + if err ~= nil then + return nil, err + end + end + if mode == MAP_CALLRW_FULL then + replicasets_to_wait = replicasets_all + else + for rs_id, _ in pairs(grouped_buckets) do + table.insert(replicasets_to_wait, replicasets_all[rs_id]) + end + end + -- Netbox async requests work only with active connections. + -- So, we need to wait for the master connection explicitly. + timeout = deadline - fiber_clock() + timeout, err, err_id = lreplicaset.wait_masters_connect( + replicasets_to_wait, timeout) + if not timeout then + return nil, err, err_id + end + return timeout, nil, nil, grouped_buckets +end + +-- +-- Waits until all the future objects are ready. After it returns successfully +-- the result of each future can be read without blocking. +-- +local function router_map_callrw_ref_wait(futures, timeout) + local deadline = fiber_clock() + timeout + for id, future in pairs(futures) do + timeout = deadline - fiber_clock() + local res, err = future_wait(future, timeout) + -- Handle netbox error first. + if res == nil then + return nil, err, id + end + -- Ref returns nil,err or bucket count. + if res[1] == nil then + return nil, res[2], id + end + end + return timeout, nil, nil +end + +-- +-- Handles all buckets which were moved from the remote storages, rewrites the +-- router's cache and builds a new table of bucket_ids for the next iteration +-- of partial map_callrw. +-- +local function router_map_callrw_process_moved(router, results) + local bucket_ids = {} + for _, res in pairs(results) do + if type(res) ~= 'table' or not res.moved then + goto continue + end + local moved = res.moved + for _, bucket in pairs(moved) do + local bid = bucket.id + local dst = bucket.dst + -- 'Reset' regardless of 'set'. So as not to + -- bother with 'set' errors. If it fails, then + -- won't matter. It is a best-effort thing. + bucket_reset(router, bid) + if dst ~= nil then + bucket_set(router, bid, dst) + end + table.insert(bucket_ids, bid) + end + ::continue:: + end + return bucket_ids +end + +-- +-- Perform Ref stage of the Ref-Map-Reduce process on all the known replicasets. +-- +local function router_ref_storage_all(router, bucket_ids, timeout, rid) + local mode = MAP_CALLRW_FULL local bucket_count = 0 + local err, err_id, grouped_buckets, results, args_ref + local future_buckets, future_all, futures = {}, {}, {} + local replicasets_all = router.replicasets local opts_async = {is_async = true} - local rs_count = 0 - local rid = M.ref_id - M.ref_id = rid + 1 - -- Nil checks are done explicitly here (== nil instead of 'not'), because - -- netbox requests return box.NULL instead of nils. - -- - -- Ref stage: send. - -- - -- Netbox async requests work only with active connections. Need to wait - -- for the connection explicitly. - timeout, err, err_id = lreplicaset.wait_masters_connect( - replicasets, timeout) + timeout, err, err_id, grouped_buckets = router_map_callrw_prepare( + router, timeout, mode, bucket_ids) if not timeout then goto fail end - for id, rs in pairs(replicasets) do - res, err = rs:callrw('vshard.storage._call', - {'storage_ref', rid, timeout}, opts_async) + args_ref = {nil, rid, timeout} + for rs_id in pairs(replicasets_all) do + local buckets = grouped_buckets[rs_id] + local target + if buckets then + args_ref[1] = 'storage_ref_make_with_buckets' + args_ref[4] = buckets + target = future_buckets + else + args_ref[1] = 'storage_ref' + args_ref[4] = nil + target = future_all + end + local res, ref_err = replicasets_all[rs_id]:callrw( + 'vshard.storage._call', args_ref, opts_async) if res == nil then - err_id = id + err, err_id = ref_err, rs_id goto fail end - futures[id] = res - rs_count = rs_count + 1 + target[rs_id] = res + futures[rs_id] = res end - -- - -- Ref stage: collect. - -- - for id, future in pairs(futures) do - res, err = future_wait(future, timeout) - -- Handle netbox error first. - if res == nil then - err_id = id + timeout, err, err_id = router_map_callrw_ref_wait(futures, timeout) + if not timeout then + goto fail + end + results = {} + for rs_id, future in pairs(future_buckets) do + results[rs_id] = future:result()[1] + end + for rs_id, future in pairs(future_all) do + results[rs_id] = {total = future:result()[1]} + end + for _, res in pairs(results) do + if not res.total then + -- This error throws only in case when the updated router tries + -- to calculate the total amount of buckets . On old storage + -- versions the function storage_ref_make_with_buckets doesn't + -- return total amount of buckets. + err = lerror.vshard(lerror.code.UNSUPPORTED, + 'full map_callrw with split args') goto fail end - -- Ref returns nil,err or bucket count. - res, err = res[1], res[2] - if res == nil then - err_id = id + bucket_count = bucket_count + res.total + end + bucket_ids = router_map_callrw_process_moved(router, results) + if next(bucket_ids) then + futures = {} + args_ref = {'storage_ref_check_existing', rid, bucket_ids} + for rs_id in pairs(replicasets_all) do + local res, ref_err = replicasets_all[rs_id]:callrw( + 'vshard.storage._call', args_ref, opts_async) + if res == nil then + err, err_id = ref_err, rs_id + goto fail + end + futures[rs_id] = res + end + timeout, err, err_id = router_map_callrw_ref_wait(futures, timeout) + if not timeout then goto fail end - bucket_count = bucket_count + res - timeout = deadline - fiber_clock() + for rs_id, future in pairs(futures) do + for _, bucket_id in pairs(future:result()[1]) do + bucket_reset(router, bucket_id) + bucket_set(router, bucket_id, rs_id) + end + end end -- All refs are done but not all buckets are covered. This is odd and can -- mean many things. The most possible ones: 1) outdated configuration on @@ -837,126 +956,81 @@ local function router_ref_storage_all(router, timeout) router.total_bucket_count - bucket_count) goto fail end - do return timeout, nil, nil, rid, replicasets end + do return timeout, nil, nil, replicasets_all end ::fail:: for _, f in pairs(futures) do f:discard() end - return nil, err, err_id, rid, replicasets + return nil, err, err_id, replicasets_all end -- -- Perform Ref stage of the Ref-Map-Reduce process on a subset of all the -- replicasets, which contains all the listed bucket IDs. -- -local function router_ref_storage_by_buckets(router, bucket_ids, timeout) - local grouped_buckets - local group_count - local err, err_id, res +local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid) + local mode = MAP_CALLRW_PARTIAL + local err, err_id, grouped_buckets, results, args_ref + local replicasets_to_map, futures = {}, {} local replicasets_all = router.replicasets - local replicasets_to_map = {} - local futures = {} local opts_async = {is_async = true} - local deadline = fiber_clock() + timeout - local rid = M.ref_id - M.ref_id = rid + 1 - -- Nil checks are done explicitly here (== nil instead of 'not'), because -- netbox requests return box.NULL instead of nils. - - -- Ref stage. while next(bucket_ids) do - -- Group the buckets by replicasets according to the router cache. - grouped_buckets, err = buckets_group(router, bucket_ids, timeout) - if grouped_buckets == nil then - goto fail - end - timeout = deadline - fiber_clock() - - -- Netbox async requests work only with active connections. - -- So, first need to wait for the master connection explicitly. - local replicasets_to_check = {} - group_count = 0 - for uuid, _ in pairs(grouped_buckets) do - group_count = group_count + 1 - table.insert(replicasets_to_check, replicasets_all[uuid]) - end - timeout, err, err_id = lreplicaset.wait_masters_connect( - replicasets_to_check, timeout) + timeout, err, err_id, grouped_buckets = router_map_callrw_prepare( + router, timeout, mode, bucket_ids) if not timeout then goto fail end - - -- Send ref requests with timeouts to the replicasets. - futures = table_new(0, group_count) - for id, buckets in pairs(grouped_buckets) do - if timeout == nil then - err_id = id - goto fail - end - local args_ref - if replicasets_to_map[id] then + futures = {} + args_ref = {nil, rid} + for rs_id in pairs(grouped_buckets) do + local buckets = grouped_buckets[rs_id] or {} + if replicasets_to_map[rs_id] then -- Replicaset is already referenced on a previous iteration. -- Simply get the moved buckets without double referencing. - args_ref = { - 'storage_ref_check_with_buckets', rid, buckets} + args_ref[1] = 'storage_ref_check_with_buckets' + args_ref[3] = buckets + args_ref[4] = nil else - args_ref = { - 'storage_ref_make_with_buckets', rid, timeout, buckets} + args_ref[1] = 'storage_ref_make_with_buckets' + args_ref[3] = timeout + args_ref[4] = buckets end - res, err = replicasets_all[id]:callrw('vshard.storage._call', - args_ref, opts_async) + local res, ref_err = replicasets_all[rs_id]:callrw( + 'vshard.storage._call', args_ref, opts_async) if res == nil then - err_id = id + err, err_id = ref_err, rs_id goto fail end - futures[id] = res + futures[rs_id] = res end - - -- Wait for the refs to be done and collect moved buckets. - bucket_ids = {} - for id, f in pairs(futures) do - res, err = future_wait(f, timeout) - -- Handle netbox error first. - if res == nil then - err_id = id - goto fail - end - -- Ref returns nil,err or {is_done, moved}. - res, err = res[1], res[2] - if res == nil then - err_id = id - goto fail - end - for _, bucket in pairs(res.moved) do - local bid = bucket.id - local dst = bucket.dst - -- 'Reset' regardless of 'set'. So as not to - -- bother with 'set' errors. If it fails, then - -- won't matter. It is a best-effort thing. - bucket_reset(router, bid) - if dst ~= nil then - bucket_set(router, bid, dst) - end - table.insert(bucket_ids, bid) - end + timeout, err, err_id = router_map_callrw_ref_wait(futures, timeout) + if not timeout then + goto fail + end + results = {} + for rs_id, future in pairs(futures) do + results[rs_id] = future:result()[1] + end + bucket_ids = router_map_callrw_process_moved(router, results) + for rs_id, res in pairs(results) do if res.is_done then - assert(not replicasets_to_map[id]) - -- If there are no buckets on the replicaset, it would not be - -- referenced. - replicasets_to_map[id] = replicasets_all[id] + assert(not replicasets_to_map[rs_id]) + -- If there are no buckets on the replicaset, it would + -- not be referenced. + replicasets_to_map[rs_id] = router.replicasets[rs_id] end - timeout = deadline - fiber_clock() end end - do return timeout, nil, nil, rid, replicasets_to_map end + do return timeout, nil, nil, replicasets_to_map end ::fail:: for _, f in pairs(futures) do f:discard() end - return nil, err, err_id, rid, replicasets_to_map + return nil, err, err_id, replicasets_to_map end -- @@ -979,13 +1053,14 @@ local function replicasets_map_reduce(replicasets, rid, func, args, -- local func_args = {'storage_map', rid, func, args} for id, rs in pairs(replicasets) do - if grouped_args ~= nil then + local rs_args = grouped_args and grouped_args[id] + if rs_args then -- It's cheaper to push and then pop, rather then deepcopy -- arguments table for every call. - table.insert(args, grouped_args[id]) + table.insert(args, rs_args) end local res, err = rs:callrw('vshard.storage._call', func_args, opts_map) - if grouped_args ~= nil then + if rs_args then table.remove(args) end if res == nil then @@ -1083,14 +1158,50 @@ local function router_group_map_callrw_args(router, bucket_ids, bucket_args) return grouped_args end +-- +-- Set the appropriate mode according to bucket_ids option for backward +-- compatibility (in case of opts_mode is nil) and check the given opts_mode +-- correctness in other cases. +-- +local function router_check_map_callrw_mode(opts_mode, bucket_ids) + if opts_mode == nil then + return bucket_ids and MAP_CALLRW_PARTIAL or MAP_CALLRW_FULL + end + if opts_mode == MAP_CALLRW_PARTIAL and bucket_ids == nil then + return nil, lerror.make('Router can\'t execute map_callrw with ' .. + '\'partial\' mode and nil bucket_ids') + end + if opts_mode == MAP_CALLRW_FULL and util.table_is_numeric(bucket_ids) then + return nil, lerror.make('Router can\'t execute map_callrw with ' .. + '\'full\' mode and numeric bucket_ids') + end + return opts_mode +end + -- -- Consistent Map-Reduce. The given function is called on masters in the cluster -- with a guarantee that in case of success it was executed with all buckets -- being accessible for reads and writes. -- --- The selection of masters depends on bucket_ids option. When specified, the --- Map-Reduce is performed only on masters having at least one of these buckets. --- Otherwise it is executed on all the masters in the cluster. +-- The selection of masters depends on 'mode' and 'bucket_ids' options. There +-- are 2 general modes how map_callrw can be executed: +-- 1) mode = 'partial'. In this mode user function will be executed on +-- storages that have at least one bucket of 'bucket_ids'. The +-- 'bucket_ids' option can be presented in two ways: like a numeric array +-- of buckets' ids or like a map of buckets' arguments. In first one user +-- function will only receive args, in second one it will additionally +-- receive buckets' arguments. +-- 2) mode = 'full'. In this mode user function will be executed with args on +-- all storages in cluster. If we pass 'bucket_ids' like a map of bucket's +-- arguments the user function will additionally receive buckets' +-- arguments on those storages that have at least one bucket of +-- 'bucket_ids'. +-- +-- If we didn't specify the 'mode' option, then it is set based on 'bucket_ids' +-- option - if 'bucket_ids' is presented, the mode will be 'partial' otherwise +-- 'full'. Also the next combination of map_callrw options can lead to error: +-- and . -- -- Consistency in scope of map-reduce means all the data was accessible, and -- didn't move during map requests execution. To preserve the consistency there @@ -1113,6 +1224,8 @@ end -- @param func Name of the function to call. -- @param args Function arguments passed in netbox style (as an array). -- @param opts Options. See below: +-- - mode - a string option ('full' / 'partial') that represents a way of +-- execution of user function on destination storages. -- - timeout - a number of seconds. Note that the refs may end up being kept -- on the storages during this entire timeout if something goes wrong. -- For instance, network issues appear. This means better not use a @@ -1136,8 +1249,13 @@ end -- local function router_map_callrw(router, func, args, opts) local replicasets_to_map, err, err_id, map, rid - local timeout, do_return_raw, bucket_ids, plain_bucket_ids, grouped_args + local mode, timeout, do_return_raw, bucket_ids, plain_bucket_ids, + grouped_args if opts then + mode, err = router_check_map_callrw_mode(opts.mode, opts.bucket_ids) + if err then + return nil, err + end timeout = opts.timeout or consts.CALL_TIMEOUT_MIN do_return_raw = opts.return_raw bucket_ids = opts.bucket_ids @@ -1145,24 +1263,26 @@ local function router_map_callrw(router, func, args, opts) util.table_keys(bucket_ids) else timeout = consts.CALL_TIMEOUT_MIN + mode = MAP_CALLRW_FULL end - if plain_bucket_ids then - timeout, err, err_id, rid, replicasets_to_map = - router_ref_storage_by_buckets(router, plain_bucket_ids, timeout) - -- Grouped arguments are only possible with partial Map-Reduce. - if timeout then - grouped_args = router_group_map_callrw_args( - router, plain_bucket_ids, bucket_ids) - end + rid = M.ref_id + M.ref_id = rid + 1 + if mode == MAP_CALLRW_FULL then + timeout, err, err_id, replicasets_to_map = + router_ref_storage_all(router, plain_bucket_ids, timeout, rid) else - timeout, err, err_id, rid, replicasets_to_map = - router_ref_storage_all(router, timeout) + timeout, err, err_id, replicasets_to_map = + router_ref_storage_by_buckets(router, plain_bucket_ids, timeout, + rid) end if timeout then - map, err, err_id = replicasets_map_reduce(replicasets_to_map, rid, func, - args, grouped_args, { - timeout = timeout, return_raw = do_return_raw - }) + if plain_bucket_ids then + grouped_args = router_group_map_callrw_args( + router, plain_bucket_ids, bucket_ids) + end + opts = {timeout = timeout, return_raw = do_return_raw, mode = mode} + map, err, err_id = replicasets_map_reduce( + replicasets_to_map, rid, func, args, grouped_args, opts) if map then return map end diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua index 1ad2d44b..8f28fe25 100644 --- a/vshard/storage/init.lua +++ b/vshard/storage/init.lua @@ -3494,6 +3494,27 @@ local function storage_call(bucket_id, mode, name, args) return ok, ret1, ret2, ret3 end +local function bucket_get_existent(bucket_ids) + local res = {} + for _, bucket_id in pairs(bucket_ids) do + local bucket = M.bucket_refs[bucket_id] or + box.space._bucket:get{bucket_id} + if bucket and bucket.status ~= BGARBAGE and bucket.status ~= BSENT then + table.insert(res, bucket_id) + end + end + return res +end + +local function storage_ref_check_existent(rid, bucket_ids) + local ok, err = lref.check(rid, box.session.id()) + if not ok then + return nil, err + end + bucket_ids = bucket_ids or {} + return bucket_get_existent(bucket_ids) +end + -- -- Bind a new storage ref to the current box session. Is used as a part of -- Map-Reduce API. @@ -3511,7 +3532,6 @@ end -- under any circumstances. -- local function bucket_get_moved(bucket_ids) - local allstatus = consts.BUCKET local res = {} for _, bucket_id in pairs(bucket_ids) do local bucket = box.space._bucket:get{bucket_id} @@ -3520,7 +3540,7 @@ local function bucket_get_moved(bucket_ids) is_moved = true else local status = bucket.status - is_moved = status == allstatus.GARBAGE or status == allstatus.SENT + is_moved = status == BGARBAGE or status == BSENT end if is_moved then table.insert(res, { @@ -3559,11 +3579,11 @@ local function storage_ref_make_with_buckets(rid, timeout, bucket_ids) if #moved == #bucket_ids then -- If all the passed buckets are absent, there is no need to create a -- ref. - return {moved = moved} + return {moved = moved, total = bucket_count()} end local bucket_generation = M.bucket_generation - local ok, err = storage_ref(rid, timeout) - if not ok then + local bucket_count, err = storage_ref(rid, timeout) + if not bucket_count then return nil, err end if M.bucket_generation ~= bucket_generation then @@ -3572,10 +3592,10 @@ local function storage_ref_make_with_buckets(rid, timeout, bucket_ids) moved = bucket_get_moved(bucket_ids) if #moved == #bucket_ids then storage_unref(rid) - return {moved = moved} + return {moved = moved, total = bucket_count} end end - return {is_done = true, moved = moved} + return {is_done = true, moved = moved, total = bucket_count} end -- @@ -3697,6 +3717,7 @@ service_call_api = setmetatable({ rebalancer_request_state = rebalancer_request_state, recovery_bucket_stat = recovery_bucket_stat, storage_ref = storage_ref, + storage_ref_check_existent = storage_ref_check_existent, storage_ref_make_with_buckets = storage_ref_make_with_buckets, storage_ref_check_with_buckets = storage_ref_check_with_buckets, storage_unref = storage_unref,