diff --git a/test/cluster-luatest/reload_test.lua b/test/cluster-luatest/reload_test.lua new file mode 100644 index 00000000..7b1d08c9 --- /dev/null +++ b/test/cluster-luatest/reload_test.lua @@ -0,0 +1,427 @@ +local t = require('luatest') +local vutil = require('vshard.util') +local vtest = require('test.luatest_helpers.vtest') +local vconsts = require('vshard.consts') +local git_util = require('test.lua_libs.git_util') +local fio = require('fio') + +local group_config = { + { update_mode = 'hot_reload' }, + { update_mode = 'restart' }, +} +local g = t.group('reload_storage_router', group_config) +local cfg_template = { + sharding = { + { + master = 'auto', + replicas = { + replica_1_a = { + read_only = false, + }, + replica_1_b = { + read_only = true, + }, + }, + }, + { + master = 'auto', + replicas = { + replica_2_a = { + read_only = false, + }, + }, + }, + }, + bucket_count = 100, +} +local global_cfg +local update_mode +local storage_names = { 'replica_1_a', 'replica_1_b', 'replica_2_a' } +-- Oldest commit that supports master = 'auto' on storage. +local legacy_storage_reload_hash = '775f5e67a02c3f71b1954c397765ed99dbf4ff7d' + +local function checkout_vshard(path, hash) + git_util.exec('checkout', { args = hash .. ' -f', dir = path }) +end + +g.before_all(function(cg) + -- Override of the built in modules is available only since 2.11.0. + t.run_only_if(vutil.version_is_at_least(2, 11, 0, nil, 0, 0)) + update_mode = cg.params.update_mode + + -- The test works in the following directory. + local vardir = fio.tempdir() + + -- + -- Tarantool searches for compilation units in the following order: + -- 1. preload --> override --> builtin + -- 2. path.cwd.dot + -- 3. cpath.cwd.dot + -- 4. path.cwd.rocks + -- 5. cpath.cwd.rocks + -- 6. package.path + -- 7. package.cpath + -- 8. croot + -- + -- Since the test is launched from the repository directory, newest + -- vshard will always be loaded, regardless of package.path or `LUA_PATH`. + -- So we can use package.setsearchroot() to change cwd, luatest's `chdir` + -- or override. The last one is used here, since it's the easiest. + -- + g.storage_vshard_copy_path = vardir .. '/vshard_storage_copy_' .. + update_mode .. '/override' + g.storage_vshard_lua_path = git_util.vshard_copy_new( + vtest.sourcedir, g.storage_vshard_copy_path) + g.router_vshard_copy_path = vardir .. '/vshard_router_copy_' .. + update_mode .. '/override' + g.router_vshard_lua_path = git_util.vshard_copy_new( + vtest.sourcedir, g.router_vshard_copy_path) + + g.latest_hash = git_util.log_hashes({ args = '-1', + dir = vtest.sourcedir })[1] + + local temp_file = os.tmpname() + git_util.exec('describe', { + args = "--tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*' HEAD^", + dir = vtest.sourcedir, + fout = temp_file, + }) + g.previous_version = io.lines(temp_file)() + os.remove(temp_file) + t.assert_not_equals(g.previous_version, nil, 'previous release tag') +end) + +local function assert_versions(router, storage_versions, router_version) + for _, storage_version in ipairs(storage_versions) do + local name = storage_version[1] + local expected_version = storage_version[2] + local _, err = g[name]:exec(function(version) + ilt.assert_equals(ivconst.VERSION, version) + end, { expected_version }) + t.assert_equals(err, nil) + end + router:exec(function(version) + ilt.assert_equals(ivconst.VERSION, version) + end, { router_version }) +end + +local function storage_test_data_create() + local test = box.schema.space.create('test', { format = { + { 'id', 'unsigned' }, + { 'bucket_id', 'unsigned' }, + } }) + test:create_index('primary') + test:create_index('bucket_id', { + unique = false, + parts = { 2 }, + }) + + box.schema.func.create('replace', { + body = [[function(space_name, tuple) + return box.space[space_name]:replace(tuple) + end]] + }) + box.schema.func.create('get', { + body = [[function(space_name, key) + return box.space[space_name]:get(key) + end]] + }) +end + +g.after_each(function() + for _, server in ipairs(g.cluster.servers) do + if server.log_file ~= nil and fio.path.exists(server.log_file) then + local msg = server:grep_log('[Aa]ssertion failed', + 64 * 1024 * 1024, {noreset = true}) + t.assert_equals(msg, nil) + end + end + + g.cluster:drop() + g.cluster = nil +end) + +local function storage_cfg_by_name(names, cfg) + for _, name in ipairs(names) do + vtest.storage_cfg(g[name], cfg) + end +end + +-- +-- Upgrade test workflow: +-- 1. Start storages and router on the old vshard version; +-- 2. Run all checks; +-- 3. Update one storage in a separate replicaset; +-- 4. Run all checks; +-- 5. Update the future master replica; +-- 6. Run all checks; +-- 7. Move master to the updated replica; +-- 8. Run all checks; +-- 9. Update the previous master replica; +-- 10. Run all checks. +-- 11. Update router; +-- 12. Run all checks. +-- +local function update_storages(names) + checkout_vshard(g.storage_vshard_copy_path, g.latest_hash) + for _, name in ipairs(names) do + local storage = g[name] + if update_mode == 'hot_reload' then + local _, err = storage:exec(function(lua_path) + package.path = lua_path + package.loaded['vshard.storage'] = nil + _G.vshard.storage = require('vshard.storage') + _G.ivconst = require('vshard.consts') + end, { git_util.vshard_lua_path(g.storage_vshard_copy_path) }) + t.assert_equals(err, nil) + else + vtest.storage_stop(storage) + vtest.storage_start(storage, global_cfg) + end + local _, err = storage:exec(function() + ivshard.storage.rebalancer_disable() + end) + t.assert_equals(err, nil) + end +end + +local function update_router(router) + checkout_vshard(g.router_vshard_copy_path, g.latest_hash) + if update_mode == 'hot_reload' then + router:exec(function() + ilt.assert_equals(ivshard.router.module_version(), 0) + package.loaded['vshard.router'] = nil + ivshard.router = require('vshard.router') + _G.ivconst = require('vshard.consts') + ilt.assert_equals(ivshard.router.module_version(), 1) + end) + else + router:restart() + vtest.router_cfg(router, global_cfg) + end +end + +local function check_cluster(router, master_uuid, move_src, move_dst) + local bucket_ids = { + vtest.storage_first_bucket(g.replica_1_a), + vtest.storage_first_bucket(g.replica_2_a), + } + t.assert_not_equals(bucket_ids[1], nil, 'replica_1_a first active bucket') + t.assert_not_equals(bucket_ids[2], nil, 'replica_2_a first active bucket') + + local rs_uuid = g.replica_1_a:replicaset_uuid() + router:exec(function(check_rs_uuid, check_master_uuid, check_bucket_ids) + local static_router = ivshard.router.static + local function wait_worker_service_ok(worker, name) + local function wakeup() + worker:wakeup_service(name) + end + local service = worker.services[name] + ivtest.wait_for_not_nil(service.data, 'info', + { on_yield = wakeup }) + ivtest.service_wait_for_ok(service.data.info, + { on_yield = wakeup }) + end + + -- Wait until master search services finish without errors. + if static_router.master_search_service ~= nil then + ivtest.service_wait_for_ok(static_router.master_search_service, + { on_yield = ivshard.router.master_search_wakeup }) + else + for _, replicaset in pairs(static_router.replicasets) do + wait_worker_service_ok(replicaset.worker, + 'replicaset_master_search') + end + end + + -- Wait until failover services finish without errors. + if static_router.failover_service ~= nil then + ivtest.service_wait_for_ok(static_router.failover_service, { + on_yield = function() + static_router.failover_fiber:wakeup() + end, + }) + else + for _, replicaset in pairs(static_router.replicasets) do + wait_worker_service_ok(replicaset.worker, + 'replicaset_failover') + for _, replica in pairs(replicaset.replicas) do + wait_worker_service_ok(replica.worker, 'replica_failover') + end + end + end + + -- Check the router sees the expected master. + ilt.helpers.retrying({}, function() + ivshard.router.master_search_wakeup() + local replicaset = static_router.replicasets[check_rs_uuid] + ilt.assert_not_equals(replicaset.master, nil) + ilt.assert_equals(check_master_uuid, replicaset.master.uuid) + end) + + -- Wait until background discovery has found all buckets. + ilt.helpers.retrying({}, function() + ivshard.router.discovery_wakeup() + ilt.assert_equals(ivshard.router.info().bucket.unknown, 0) + end) + + -- Check that routing still works for both write and read + -- requests on both replicasets. + for _, bucket_id in ipairs(check_bucket_ids) do + local tuple = { bucket_id, bucket_id } + local res, err = ivshard.router.callrw(bucket_id, 'replace', + { 'test', tuple }) + ilt.assert_equals(err, nil) + ilt.assert_equals(res, tuple) + local _, sync_err = ivshard.router.callrw(bucket_id, + 'vshard.storage.sync', + {}) + ilt.assert_equals(sync_err, nil) + res, err = ivshard.router.callro(bucket_id, 'get', + { 'test', { bucket_id } }) + ilt.assert_equals(err, nil) + ilt.assert_equals(res, tuple) + end + end, { rs_uuid, master_uuid, bucket_ids }) + + -- Move a bucket and check that router discovery sees the new owner. + local moved_bucket_id = vtest.storage_first_bucket(move_src) + local move_dst_rs_uuid = move_dst:replicaset_uuid() + local move_dst_uuid = move_dst:instance_uuid() + local _, send_err = move_src:exec(function(bucket_id_to_move, + dst_rs_uuid, dst_master_uuid) + -- When the source runs legacy vshard, master = 'auto' works only for + -- its own replicaset. Tell it the remote master before bucket_send(). + local replicaset = ivshard.storage.internal.replicasets[dst_rs_uuid] + local master = replicaset.replicas[dst_master_uuid] + replicaset.master = master + + local ok, err = ivshard.storage.bucket_send(bucket_id_to_move, + dst_rs_uuid) + ilt.assert_equals(err, nil) + ilt.assert(ok) + end, { moved_bucket_id, move_dst_rs_uuid, move_dst_uuid }) + t.assert_equals(send_err, nil) + + local _, route_err = router:exec(function(route_bucket_id, + expected_rs_uuid) + ilt.helpers.retrying({}, function() + ivshard.router.discovery_wakeup() + local routed_rs, err = ivshard.router.route(route_bucket_id) + ilt.assert_equals(err, nil) + ilt.assert_equals(routed_rs.uuid, expected_rs_uuid) + end) + end, { moved_bucket_id, move_dst_rs_uuid }) + t.assert_equals(route_err, nil) +end + +local function create_cluster_at(opts) + global_cfg = vtest.config_new(cfg_template) + checkout_vshard(g.storage_vshard_copy_path, opts.hash) + local env = { + LUA_PATH = g.storage_vshard_lua_path, + } + vtest.cluster_new(g, global_cfg, { env = env }) + + vtest.cluster_rebalancer_disable(g) + vtest.cluster_bootstrap(g, global_cfg) + -- Create user data and simple functions for router callrw/callro checks. + for _, name in ipairs({ 'replica_1_a', 'replica_2_a' }) do + local _, err = g[name]:exec(storage_test_data_create) + t.assert_equals(err, nil) + end + vtest.cluster_wait_vclock_all(g) + + checkout_vshard(g.router_vshard_copy_path, opts.hash) + local router = vtest.router_new(g, 'router', nil, { + env = { + -- Force 'require' to use new directory. + LUA_PATH = g.router_vshard_lua_path, + }, + }) + vtest.router_cfg(router, global_cfg) + return router +end + +local function run_rolling_upgrade(opts) + + local router = create_cluster_at(opts) + local old_master_uuid = g.replica_1_a:instance_uuid() + local new_master_uuid = g.replica_1_b:instance_uuid() + local old_master = g.replica_1_a + local new_master = g.replica_1_b + + local all_old = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', opts.storage_version }, + { 'replica_2_a', opts.storage_version }, + } + local one_new = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', opts.storage_version }, + { 'replica_2_a', vconsts.VERSION }, + } + local mixed = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', vconsts.VERSION }, + { 'replica_2_a', vconsts.VERSION }, + } + local all_new = { + { 'replica_1_a', vconsts.VERSION }, + { 'replica_1_b', vconsts.VERSION }, + { 'replica_2_a', vconsts.VERSION }, + } + + -- First phase: everything is on old vshard. + assert_versions(router, all_old, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Second phase: one replica is updated on new vshard. + update_storages({ 'replica_2_a' }) + assert_versions(router, one_new, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Third phase: another replica is updated on new vshard. + update_storages({ 'replica_1_b' }) + assert_versions(router, mixed, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Fourth phase: master is moved to the updated replica, previous master + -- is still on old vshard. + old_master:update_box_cfg({ read_only = true }) + new_master:wait_vclock_of(old_master) + new_master:update_box_cfg({ read_only = false }) + local rs_uuid = old_master:replicaset_uuid() + global_cfg.sharding[rs_uuid].replicas[old_master_uuid].read_only = true + global_cfg.sharding[rs_uuid].replicas[new_master_uuid].read_only = false + storage_cfg_by_name({ 'replica_1_a', 'replica_1_b' }, global_cfg) + vtest.router_cfg(router, global_cfg) + assert_versions(router, mixed, opts.storage_version) + check_cluster(router, new_master_uuid, g.replica_2_a, g.replica_1_b) + + -- Fifth phase: previous master is updated. + update_storages({ 'replica_1_a' }) + storage_cfg_by_name({ 'replica_1_a' }, global_cfg) + assert_versions(router, all_new, opts.storage_version) + check_cluster(router, new_master_uuid, g.replica_1_b, g.replica_2_a) + + -- Sixth phase: router is updated. + update_router(router) + storage_cfg_by_name(storage_names, global_cfg) + assert_versions(router, all_new, vconsts.VERSION) + check_cluster(router, new_master_uuid, g.replica_2_a, g.replica_1_b) +end + +g.test_rolling_legacy_storage_then_router_upgrade = function() + run_rolling_upgrade({ + hash = legacy_storage_reload_hash, + storage_version = '0.1.24', + }) +end + +g.test_rolling_previous_storage_then_router_upgrade = function() + run_rolling_upgrade({ + hash = g.previous_version, + storage_version = g.previous_version, + }) +end diff --git a/test/cluster-luatest/suite.ini b/test/cluster-luatest/suite.ini new file mode 100644 index 00000000..44dd65d8 --- /dev/null +++ b/test/cluster-luatest/suite.ini @@ -0,0 +1,5 @@ +[default] +core = luatest +description = Cluster tests +is_parallel = True +release_disabled = diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua index 48f98278..12c25f9f 100644 --- a/test/lua_libs/git_util.lua +++ b/test/lua_libs/git_util.lua @@ -2,6 +2,7 @@ -- Lua bridge for some of the git commands. -- local os = require('os') +local fio = require('fio') -- -- Exec a git command. @@ -40,7 +41,23 @@ local function log_hashes(params) return lines end +local function vshard_lua_path(path) + return string.format('%s/?.lua;%s/?/init.lua;', path, path) .. + (os.getenv('LUA_PATH') or '') +end + +local function vshard_copy_new(sourcedir, copy_path) + local copy_path_load = fio.dirname(copy_path) + assert(fio.mkdir(copy_path_load) == true) + assert(fio.mkdir(copy_path) == true) + assert(fio.mkdir(copy_path .. '/.git') == true) + assert(fio.copytree(sourcedir .. '/.git', copy_path .. '/.git') == true) + return vshard_lua_path(copy_path_load) +end + return { exec = exec, - log_hashes = log_hashes + log_hashes = log_hashes, + vshard_copy_new = vshard_copy_new, + vshard_lua_path = vshard_lua_path, } diff --git a/test/luatest_helpers/vtest.lua b/test/luatest_helpers/vtest.lua index f9219439..406333cf 100644 --- a/test/luatest_helpers/vtest.lua +++ b/test/luatest_helpers/vtest.lua @@ -146,12 +146,13 @@ local function config_new(templ) end -- --- Build new cluster by a given config. +-- Build a new cluster by a given config and optional server config. -- -local function cluster_new(g, cfg) +local function cluster_new(g, cfg, server_config) if not g.cluster then g.cluster = cluster:new({}) end + server_config = server_config or {} local all_servers = {} local masters = {} local replicas = {} @@ -201,10 +202,11 @@ local function cluster_new(g, cfg) box_cfg.instance_name = replica_name box_cfg.replicaset_name = replicaset_name end - local server = g.cluster:build_server({ - alias = replica_name, - box_cfg = box_cfg, - }, 'storage.lua') + local instance_config = table.deepcopy(server_config) + instance_config.alias = replica_name + instance_config.box_cfg = box_cfg + local server = g.cluster:build_server(instance_config, + 'storage.lua') g[replica_name] = server -- VShard specific details to use in various helper functions. server.vtest = { @@ -234,8 +236,11 @@ local function cluster_new(g, cfg) box.session.su('admin') local grant_range = cfg.test_user_grant_range - ivtest.clear_test_cfg_options(cfg) - + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end if cfg.schema_management_mode == 'manual_access' then local vexports = require('vshard.storage.exports') local username = 'storage' @@ -264,7 +269,11 @@ local function cluster_new(g, cfg) for _, replica in pairs(replicas) do replica:wait_for_readiness() replica:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end, {cfg}) end @@ -387,8 +396,12 @@ local function cluster_bootstrap(g, cfg) local replicaset_count = 0 local master_info, err = cluster_exec_each(g, function(is_named) local info = box.info + local is_master = ivshard.storage.internal.is_master + if is_master == nil then + is_master = box.cfg.read_only == false + end return { - is_master = ivshard.storage.internal.is_master, + is_master = is_master, rs_id = is_named and info.replicaset.name or ivutil.replicaset_uuid(info), id = is_named and info.name or info.uuid @@ -447,7 +460,11 @@ local function cluster_bootstrap(g, cfg) end local function storage_cfg_impl(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end return ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end @@ -676,7 +693,11 @@ end local function storage_start(storage, cfg) storage:start() local _, err = storage:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end return ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end, {cfg}) t.assert_equals(err, nil, 'storage cfg on start') @@ -687,7 +708,11 @@ end -- local function router_cfg(router, cfg) router:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end ivshard.router.cfg(cfg) end, {cfg}) end diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua index c1ae28d7..b9620f0f 100644 --- a/test/router-luatest/reload_test.lua +++ b/test/router-luatest/reload_test.lua @@ -35,8 +35,6 @@ g.before_all(function() -- The test works in the following directory local vardir = vtest.vardir or fio.tempdir() - g.vshard_copy_path_load = vardir .. '/vshard_copy' - t.assert_equals(fio.mkdir(g.vshard_copy_path_load), true) -- -- Tarantool searches for compilation units in the following order: -- 1. preload --> override --> builtin @@ -53,12 +51,9 @@ g.before_all(function() -- So we can use package.setsearchroot() to change cwd, luatest's `chdir` -- or override. The last one is used here, since it's the easiest. -- - g.vshard_copy_path = vardir .. '/vshard_copy/override' - t.assert_equals(fio.mkdir(g.vshard_copy_path), true) - -- Copy source to the temporary directory - t.assert_equals(fio.mkdir(g.vshard_copy_path .. '/.git'), true) - t.assert_equals(fio.copytree(vtest.sourcedir .. '/.git', - g.vshard_copy_path .. '/.git'), true) + g.vshard_copy_path = vardir .. '/vshard_copy/override' + g.vshard_lua_path = git_util.vshard_copy_new(vtest.sourcedir, + g.vshard_copy_path) -- Hash of the latest commit for testing router on the latest version. g.latest_hash = git_util.log_hashes({args = '-1', dir = vtest.sourcedir})[1] @@ -93,16 +88,6 @@ g.after_all(function() g.cluster:drop() end) --- --- `vtest.router_cfg` cannot be used in this test, since --- `ivtest.clear_test_cfg_options` may be nil on old versions. --- -local function router_cfg(router, cfg) - router:exec(function(cfg) - ivshard.router.cfg(cfg) - end, {cfg}) -end - local function router_assert_version_equals(router, version) router:exec(function(version) ilt.assert_equals(ivconst.VERSION, version) @@ -123,15 +108,13 @@ end -- local function create_router_at(hash) git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path}) - local path = g.vshard_copy_path_load - local lua_path = string.format("%s/?.lua;%s/?/init.lua;", path, path) local router = vtest.router_new(g, 'router', nil, { env = { -- Force 'require' to use new directory - ['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH') + ['LUA_PATH'] = g.vshard_lua_path }, }) - router_cfg(router, global_cfg) + vtest.router_cfg(router, global_cfg) return router end @@ -235,7 +218,7 @@ g.test_discovery = function(g) end local function test_master_search_template(g, router, auto_master_cfg) - router_cfg(router, auto_master_cfg) + vtest.router_cfg(router, auto_master_cfg) -- Working with first replicaset (2 instances) local rs_uuid = g.replica_1_a:replicaset_uuid() diff --git a/vshard/router/init.lua b/vshard/router/init.lua index 098bca2c..0962944c 100644 --- a/vshard/router/init.lua +++ b/vshard/router/init.lua @@ -1831,9 +1831,9 @@ else if router.is_enabled == nil then router.is_enabled = true end - if router.api_call_cache == nil then - router.api_call_cache = router_api_call_unsafe - end + -- After reloading from an old vshard version, api_call_cache + -- can be nil or point to the old router_api_call signature. + router.api_call_cache = router_api_call_unsafe router_cfg_fiber_safe(router, router.current_cfg, true) setmetatable(router, router_mt) end diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua index 1ad2d44b..7f50d31e 100644 --- a/vshard/storage/init.lua +++ b/vshard/storage/init.lua @@ -3734,6 +3734,20 @@ local function master_sync_service_f(service, limiter) map_res, err, err_id = M.this_replicaset:map_call('vshard.storage._call', {'storage_bucket_checkpoint', call_timeout / 1.5}, call_opts) + if err and (err.type == 'LuajitError' or err.type == 'ClientError') and + err.message and err.message:match( + '/vshard/storage/init%.lua:%d+: attempt to call a nil value$') + then + -- Storages predating storage_bucket_checkpoint() have no + -- production capability endpoint. Their service dispatcher fails + -- with a plain Lua nil-call error when a service is missing. + map_res, err, err_id = + M.this_replicaset:map_call('vshard.storage.sync', + {call_timeout / 1.5}, call_opts) + if not err then + map_res = {} + end + end if err then err.replica_id = err_id limiter:log_warn(err, service:set_status_error(err_msg, err)) @@ -4583,6 +4597,13 @@ if not rawget(_G, MODULE_INTERNALS) then rawset(_G, MODULE_INTERNALS, M) else reload_evolution.upgrade(M) + -- Hot reload from old vshard can leave master sync internals incompatible + -- with the reloaded code. Refresh the function and restart a dead fiber. + M.master_sync_f = master_sync_f + if M.master_sync_fiber ~= nil and + M.master_sync_fiber:status() == 'dead' then + M.master_sync_fiber = nil + end if M.current_cfg then storage_cfg(M.current_cfg, M.this_replica.id or M.this_replica.uuid, true)