From f3336feb4541f4d04b4ac923d56cb09ee6cb346a Mon Sep 17 00:00:00 2001 From: kamenkremen Date: Wed, 8 Jul 2026 13:09:22 +0300 Subject: [PATCH 1/2] test: share vshard copy helper Router reload test copy vshard sources to a temporary directory and checkout old revisions there. The cluster reload test needs the same setup for router and storage copies. Move the copy and LUA_PATH construction to git_util so reload tests reuse the same helper. Needed for #654 NO_DOC=test NO_TEST=refactoring --- test/lua_libs/git_util.lua | 19 ++++++++++++++++++- test/router-luatest/reload_test.lua | 15 ++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua index 48f98278..12c25f9f 100644 --- a/test/lua_libs/git_util.lua +++ b/test/lua_libs/git_util.lua @@ -2,6 +2,7 @@ -- Lua bridge for some of the git commands. -- local os = require('os') +local fio = require('fio') -- -- Exec a git command. @@ -40,7 +41,23 @@ local function log_hashes(params) return lines end +local function vshard_lua_path(path) + return string.format('%s/?.lua;%s/?/init.lua;', path, path) .. + (os.getenv('LUA_PATH') or '') +end + +local function vshard_copy_new(sourcedir, copy_path) + local copy_path_load = fio.dirname(copy_path) + assert(fio.mkdir(copy_path_load) == true) + assert(fio.mkdir(copy_path) == true) + assert(fio.mkdir(copy_path .. '/.git') == true) + assert(fio.copytree(sourcedir .. '/.git', copy_path .. '/.git') == true) + return vshard_lua_path(copy_path_load) +end + return { exec = exec, - log_hashes = log_hashes + log_hashes = log_hashes, + vshard_copy_new = vshard_copy_new, + vshard_lua_path = vshard_lua_path, } diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua index c1ae28d7..0fabbcb2 100644 --- a/test/router-luatest/reload_test.lua +++ b/test/router-luatest/reload_test.lua @@ -35,8 +35,6 @@ g.before_all(function() -- The test works in the following directory local vardir = vtest.vardir or fio.tempdir() - g.vshard_copy_path_load = vardir .. '/vshard_copy' - t.assert_equals(fio.mkdir(g.vshard_copy_path_load), true) -- -- Tarantool searches for compilation units in the following order: -- 1. preload --> override --> builtin @@ -53,12 +51,9 @@ g.before_all(function() -- So we can use package.setsearchroot() to change cwd, luatest's `chdir` -- or override. The last one is used here, since it's the easiest. -- - g.vshard_copy_path = vardir .. '/vshard_copy/override' - t.assert_equals(fio.mkdir(g.vshard_copy_path), true) - -- Copy source to the temporary directory - t.assert_equals(fio.mkdir(g.vshard_copy_path .. '/.git'), true) - t.assert_equals(fio.copytree(vtest.sourcedir .. '/.git', - g.vshard_copy_path .. '/.git'), true) + g.vshard_copy_path = vardir .. '/vshard_copy/override' + g.vshard_lua_path = git_util.vshard_copy_new(vtest.sourcedir, + g.vshard_copy_path) -- Hash of the latest commit for testing router on the latest version. g.latest_hash = git_util.log_hashes({args = '-1', dir = vtest.sourcedir})[1] @@ -123,12 +118,10 @@ end -- local function create_router_at(hash) git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path}) - local path = g.vshard_copy_path_load - local lua_path = string.format("%s/?.lua;%s/?/init.lua;", path, path) local router = vtest.router_new(g, 'router', nil, { env = { -- Force 'require' to use new directory - ['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH') + ['LUA_PATH'] = g.vshard_lua_path }, }) router_cfg(router, global_cfg) From 3117fb65c305cb2f4ff8cc0de177c19d0f386b3c Mon Sep 17 00:00:00 2001 From: kamenkremen Date: Wed, 8 Jul 2026 13:10:08 +0300 Subject: [PATCH 2/2] cluster: test storage-first upgrade The intended upgrade order for clusters is to upgrade storage first and routers afterwards. Cover that upgrade order from legacy and previous vshard versions in both hot reload and restart modes. The test starts a cluster on an old checkout, upgrades storages first, switches the master to an already upgraded replica, upgrades the previous master, and updates the router last. After each phase it checks basic vshard functionality. Add vtest fixes to let luatest helpers work with old vshard versions. Fix compatibility issues exposed by the test. After hot reload, a router could keep an old api_call_cache while the router api call signature had changed. Storage hot reload could also keep stale master_sync_f state and a dead master_sync_fiber, so master sync was not restarted with the reloaded implementation. For mixed-version bucket movement, fall back to vshard.storage.sync when an old storage doesn't have storage_bucket_checkpoint(). Closes #654 @TarantoolBot document Title: vshard: storage-first cluster upgrade order Root document: https://www.tarantool.io/en/doc/latest/admin/upgrades/upgrade_cluster/ Update the cluster upgrade procedure to describe the storage-first order. The current documentation describes the router-first order, but the intended order is the opposite: upgrade storage first, then upgrade routers. The motivation is that storages do not depend on routers during upgrade. Routers initiate communication with storages. --- test/cluster-luatest/reload_test.lua | 427 +++++++++++++++++++++++++++ test/cluster-luatest/suite.ini | 5 + test/luatest_helpers/vtest.lua | 51 +++- test/router-luatest/reload_test.lua | 14 +- vshard/router/init.lua | 6 +- vshard/storage/init.lua | 21 ++ 6 files changed, 496 insertions(+), 28 deletions(-) create mode 100644 test/cluster-luatest/reload_test.lua create mode 100644 test/cluster-luatest/suite.ini diff --git a/test/cluster-luatest/reload_test.lua b/test/cluster-luatest/reload_test.lua new file mode 100644 index 00000000..7b1d08c9 --- /dev/null +++ b/test/cluster-luatest/reload_test.lua @@ -0,0 +1,427 @@ +local t = require('luatest') +local vutil = require('vshard.util') +local vtest = require('test.luatest_helpers.vtest') +local vconsts = require('vshard.consts') +local git_util = require('test.lua_libs.git_util') +local fio = require('fio') + +local group_config = { + { update_mode = 'hot_reload' }, + { update_mode = 'restart' }, +} +local g = t.group('reload_storage_router', group_config) +local cfg_template = { + sharding = { + { + master = 'auto', + replicas = { + replica_1_a = { + read_only = false, + }, + replica_1_b = { + read_only = true, + }, + }, + }, + { + master = 'auto', + replicas = { + replica_2_a = { + read_only = false, + }, + }, + }, + }, + bucket_count = 100, +} +local global_cfg +local update_mode +local storage_names = { 'replica_1_a', 'replica_1_b', 'replica_2_a' } +-- Oldest commit that supports master = 'auto' on storage. +local legacy_storage_reload_hash = '775f5e67a02c3f71b1954c397765ed99dbf4ff7d' + +local function checkout_vshard(path, hash) + git_util.exec('checkout', { args = hash .. ' -f', dir = path }) +end + +g.before_all(function(cg) + -- Override of the built in modules is available only since 2.11.0. + t.run_only_if(vutil.version_is_at_least(2, 11, 0, nil, 0, 0)) + update_mode = cg.params.update_mode + + -- The test works in the following directory. + local vardir = fio.tempdir() + + -- + -- Tarantool searches for compilation units in the following order: + -- 1. preload --> override --> builtin + -- 2. path.cwd.dot + -- 3. cpath.cwd.dot + -- 4. path.cwd.rocks + -- 5. cpath.cwd.rocks + -- 6. package.path + -- 7. package.cpath + -- 8. croot + -- + -- Since the test is launched from the repository directory, newest + -- vshard will always be loaded, regardless of package.path or `LUA_PATH`. + -- So we can use package.setsearchroot() to change cwd, luatest's `chdir` + -- or override. The last one is used here, since it's the easiest. + -- + g.storage_vshard_copy_path = vardir .. '/vshard_storage_copy_' .. + update_mode .. '/override' + g.storage_vshard_lua_path = git_util.vshard_copy_new( + vtest.sourcedir, g.storage_vshard_copy_path) + g.router_vshard_copy_path = vardir .. '/vshard_router_copy_' .. + update_mode .. '/override' + g.router_vshard_lua_path = git_util.vshard_copy_new( + vtest.sourcedir, g.router_vshard_copy_path) + + g.latest_hash = git_util.log_hashes({ args = '-1', + dir = vtest.sourcedir })[1] + + local temp_file = os.tmpname() + git_util.exec('describe', { + args = "--tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*' HEAD^", + dir = vtest.sourcedir, + fout = temp_file, + }) + g.previous_version = io.lines(temp_file)() + os.remove(temp_file) + t.assert_not_equals(g.previous_version, nil, 'previous release tag') +end) + +local function assert_versions(router, storage_versions, router_version) + for _, storage_version in ipairs(storage_versions) do + local name = storage_version[1] + local expected_version = storage_version[2] + local _, err = g[name]:exec(function(version) + ilt.assert_equals(ivconst.VERSION, version) + end, { expected_version }) + t.assert_equals(err, nil) + end + router:exec(function(version) + ilt.assert_equals(ivconst.VERSION, version) + end, { router_version }) +end + +local function storage_test_data_create() + local test = box.schema.space.create('test', { format = { + { 'id', 'unsigned' }, + { 'bucket_id', 'unsigned' }, + } }) + test:create_index('primary') + test:create_index('bucket_id', { + unique = false, + parts = { 2 }, + }) + + box.schema.func.create('replace', { + body = [[function(space_name, tuple) + return box.space[space_name]:replace(tuple) + end]] + }) + box.schema.func.create('get', { + body = [[function(space_name, key) + return box.space[space_name]:get(key) + end]] + }) +end + +g.after_each(function() + for _, server in ipairs(g.cluster.servers) do + if server.log_file ~= nil and fio.path.exists(server.log_file) then + local msg = server:grep_log('[Aa]ssertion failed', + 64 * 1024 * 1024, {noreset = true}) + t.assert_equals(msg, nil) + end + end + + g.cluster:drop() + g.cluster = nil +end) + +local function storage_cfg_by_name(names, cfg) + for _, name in ipairs(names) do + vtest.storage_cfg(g[name], cfg) + end +end + +-- +-- Upgrade test workflow: +-- 1. Start storages and router on the old vshard version; +-- 2. Run all checks; +-- 3. Update one storage in a separate replicaset; +-- 4. Run all checks; +-- 5. Update the future master replica; +-- 6. Run all checks; +-- 7. Move master to the updated replica; +-- 8. Run all checks; +-- 9. Update the previous master replica; +-- 10. Run all checks. +-- 11. Update router; +-- 12. Run all checks. +-- +local function update_storages(names) + checkout_vshard(g.storage_vshard_copy_path, g.latest_hash) + for _, name in ipairs(names) do + local storage = g[name] + if update_mode == 'hot_reload' then + local _, err = storage:exec(function(lua_path) + package.path = lua_path + package.loaded['vshard.storage'] = nil + _G.vshard.storage = require('vshard.storage') + _G.ivconst = require('vshard.consts') + end, { git_util.vshard_lua_path(g.storage_vshard_copy_path) }) + t.assert_equals(err, nil) + else + vtest.storage_stop(storage) + vtest.storage_start(storage, global_cfg) + end + local _, err = storage:exec(function() + ivshard.storage.rebalancer_disable() + end) + t.assert_equals(err, nil) + end +end + +local function update_router(router) + checkout_vshard(g.router_vshard_copy_path, g.latest_hash) + if update_mode == 'hot_reload' then + router:exec(function() + ilt.assert_equals(ivshard.router.module_version(), 0) + package.loaded['vshard.router'] = nil + ivshard.router = require('vshard.router') + _G.ivconst = require('vshard.consts') + ilt.assert_equals(ivshard.router.module_version(), 1) + end) + else + router:restart() + vtest.router_cfg(router, global_cfg) + end +end + +local function check_cluster(router, master_uuid, move_src, move_dst) + local bucket_ids = { + vtest.storage_first_bucket(g.replica_1_a), + vtest.storage_first_bucket(g.replica_2_a), + } + t.assert_not_equals(bucket_ids[1], nil, 'replica_1_a first active bucket') + t.assert_not_equals(bucket_ids[2], nil, 'replica_2_a first active bucket') + + local rs_uuid = g.replica_1_a:replicaset_uuid() + router:exec(function(check_rs_uuid, check_master_uuid, check_bucket_ids) + local static_router = ivshard.router.static + local function wait_worker_service_ok(worker, name) + local function wakeup() + worker:wakeup_service(name) + end + local service = worker.services[name] + ivtest.wait_for_not_nil(service.data, 'info', + { on_yield = wakeup }) + ivtest.service_wait_for_ok(service.data.info, + { on_yield = wakeup }) + end + + -- Wait until master search services finish without errors. + if static_router.master_search_service ~= nil then + ivtest.service_wait_for_ok(static_router.master_search_service, + { on_yield = ivshard.router.master_search_wakeup }) + else + for _, replicaset in pairs(static_router.replicasets) do + wait_worker_service_ok(replicaset.worker, + 'replicaset_master_search') + end + end + + -- Wait until failover services finish without errors. + if static_router.failover_service ~= nil then + ivtest.service_wait_for_ok(static_router.failover_service, { + on_yield = function() + static_router.failover_fiber:wakeup() + end, + }) + else + for _, replicaset in pairs(static_router.replicasets) do + wait_worker_service_ok(replicaset.worker, + 'replicaset_failover') + for _, replica in pairs(replicaset.replicas) do + wait_worker_service_ok(replica.worker, 'replica_failover') + end + end + end + + -- Check the router sees the expected master. + ilt.helpers.retrying({}, function() + ivshard.router.master_search_wakeup() + local replicaset = static_router.replicasets[check_rs_uuid] + ilt.assert_not_equals(replicaset.master, nil) + ilt.assert_equals(check_master_uuid, replicaset.master.uuid) + end) + + -- Wait until background discovery has found all buckets. + ilt.helpers.retrying({}, function() + ivshard.router.discovery_wakeup() + ilt.assert_equals(ivshard.router.info().bucket.unknown, 0) + end) + + -- Check that routing still works for both write and read + -- requests on both replicasets. + for _, bucket_id in ipairs(check_bucket_ids) do + local tuple = { bucket_id, bucket_id } + local res, err = ivshard.router.callrw(bucket_id, 'replace', + { 'test', tuple }) + ilt.assert_equals(err, nil) + ilt.assert_equals(res, tuple) + local _, sync_err = ivshard.router.callrw(bucket_id, + 'vshard.storage.sync', + {}) + ilt.assert_equals(sync_err, nil) + res, err = ivshard.router.callro(bucket_id, 'get', + { 'test', { bucket_id } }) + ilt.assert_equals(err, nil) + ilt.assert_equals(res, tuple) + end + end, { rs_uuid, master_uuid, bucket_ids }) + + -- Move a bucket and check that router discovery sees the new owner. + local moved_bucket_id = vtest.storage_first_bucket(move_src) + local move_dst_rs_uuid = move_dst:replicaset_uuid() + local move_dst_uuid = move_dst:instance_uuid() + local _, send_err = move_src:exec(function(bucket_id_to_move, + dst_rs_uuid, dst_master_uuid) + -- When the source runs legacy vshard, master = 'auto' works only for + -- its own replicaset. Tell it the remote master before bucket_send(). + local replicaset = ivshard.storage.internal.replicasets[dst_rs_uuid] + local master = replicaset.replicas[dst_master_uuid] + replicaset.master = master + + local ok, err = ivshard.storage.bucket_send(bucket_id_to_move, + dst_rs_uuid) + ilt.assert_equals(err, nil) + ilt.assert(ok) + end, { moved_bucket_id, move_dst_rs_uuid, move_dst_uuid }) + t.assert_equals(send_err, nil) + + local _, route_err = router:exec(function(route_bucket_id, + expected_rs_uuid) + ilt.helpers.retrying({}, function() + ivshard.router.discovery_wakeup() + local routed_rs, err = ivshard.router.route(route_bucket_id) + ilt.assert_equals(err, nil) + ilt.assert_equals(routed_rs.uuid, expected_rs_uuid) + end) + end, { moved_bucket_id, move_dst_rs_uuid }) + t.assert_equals(route_err, nil) +end + +local function create_cluster_at(opts) + global_cfg = vtest.config_new(cfg_template) + checkout_vshard(g.storage_vshard_copy_path, opts.hash) + local env = { + LUA_PATH = g.storage_vshard_lua_path, + } + vtest.cluster_new(g, global_cfg, { env = env }) + + vtest.cluster_rebalancer_disable(g) + vtest.cluster_bootstrap(g, global_cfg) + -- Create user data and simple functions for router callrw/callro checks. + for _, name in ipairs({ 'replica_1_a', 'replica_2_a' }) do + local _, err = g[name]:exec(storage_test_data_create) + t.assert_equals(err, nil) + end + vtest.cluster_wait_vclock_all(g) + + checkout_vshard(g.router_vshard_copy_path, opts.hash) + local router = vtest.router_new(g, 'router', nil, { + env = { + -- Force 'require' to use new directory. + LUA_PATH = g.router_vshard_lua_path, + }, + }) + vtest.router_cfg(router, global_cfg) + return router +end + +local function run_rolling_upgrade(opts) + + local router = create_cluster_at(opts) + local old_master_uuid = g.replica_1_a:instance_uuid() + local new_master_uuid = g.replica_1_b:instance_uuid() + local old_master = g.replica_1_a + local new_master = g.replica_1_b + + local all_old = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', opts.storage_version }, + { 'replica_2_a', opts.storage_version }, + } + local one_new = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', opts.storage_version }, + { 'replica_2_a', vconsts.VERSION }, + } + local mixed = { + { 'replica_1_a', opts.storage_version }, + { 'replica_1_b', vconsts.VERSION }, + { 'replica_2_a', vconsts.VERSION }, + } + local all_new = { + { 'replica_1_a', vconsts.VERSION }, + { 'replica_1_b', vconsts.VERSION }, + { 'replica_2_a', vconsts.VERSION }, + } + + -- First phase: everything is on old vshard. + assert_versions(router, all_old, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Second phase: one replica is updated on new vshard. + update_storages({ 'replica_2_a' }) + assert_versions(router, one_new, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Third phase: another replica is updated on new vshard. + update_storages({ 'replica_1_b' }) + assert_versions(router, mixed, opts.storage_version) + check_cluster(router, old_master_uuid, g.replica_2_a, g.replica_1_a) + + -- Fourth phase: master is moved to the updated replica, previous master + -- is still on old vshard. + old_master:update_box_cfg({ read_only = true }) + new_master:wait_vclock_of(old_master) + new_master:update_box_cfg({ read_only = false }) + local rs_uuid = old_master:replicaset_uuid() + global_cfg.sharding[rs_uuid].replicas[old_master_uuid].read_only = true + global_cfg.sharding[rs_uuid].replicas[new_master_uuid].read_only = false + storage_cfg_by_name({ 'replica_1_a', 'replica_1_b' }, global_cfg) + vtest.router_cfg(router, global_cfg) + assert_versions(router, mixed, opts.storage_version) + check_cluster(router, new_master_uuid, g.replica_2_a, g.replica_1_b) + + -- Fifth phase: previous master is updated. + update_storages({ 'replica_1_a' }) + storage_cfg_by_name({ 'replica_1_a' }, global_cfg) + assert_versions(router, all_new, opts.storage_version) + check_cluster(router, new_master_uuid, g.replica_1_b, g.replica_2_a) + + -- Sixth phase: router is updated. + update_router(router) + storage_cfg_by_name(storage_names, global_cfg) + assert_versions(router, all_new, vconsts.VERSION) + check_cluster(router, new_master_uuid, g.replica_2_a, g.replica_1_b) +end + +g.test_rolling_legacy_storage_then_router_upgrade = function() + run_rolling_upgrade({ + hash = legacy_storage_reload_hash, + storage_version = '0.1.24', + }) +end + +g.test_rolling_previous_storage_then_router_upgrade = function() + run_rolling_upgrade({ + hash = g.previous_version, + storage_version = g.previous_version, + }) +end diff --git a/test/cluster-luatest/suite.ini b/test/cluster-luatest/suite.ini new file mode 100644 index 00000000..44dd65d8 --- /dev/null +++ b/test/cluster-luatest/suite.ini @@ -0,0 +1,5 @@ +[default] +core = luatest +description = Cluster tests +is_parallel = True +release_disabled = diff --git a/test/luatest_helpers/vtest.lua b/test/luatest_helpers/vtest.lua index f9219439..406333cf 100644 --- a/test/luatest_helpers/vtest.lua +++ b/test/luatest_helpers/vtest.lua @@ -146,12 +146,13 @@ local function config_new(templ) end -- --- Build new cluster by a given config. +-- Build a new cluster by a given config and optional server config. -- -local function cluster_new(g, cfg) +local function cluster_new(g, cfg, server_config) if not g.cluster then g.cluster = cluster:new({}) end + server_config = server_config or {} local all_servers = {} local masters = {} local replicas = {} @@ -201,10 +202,11 @@ local function cluster_new(g, cfg) box_cfg.instance_name = replica_name box_cfg.replicaset_name = replicaset_name end - local server = g.cluster:build_server({ - alias = replica_name, - box_cfg = box_cfg, - }, 'storage.lua') + local instance_config = table.deepcopy(server_config) + instance_config.alias = replica_name + instance_config.box_cfg = box_cfg + local server = g.cluster:build_server(instance_config, + 'storage.lua') g[replica_name] = server -- VShard specific details to use in various helper functions. server.vtest = { @@ -234,8 +236,11 @@ local function cluster_new(g, cfg) box.session.su('admin') local grant_range = cfg.test_user_grant_range - ivtest.clear_test_cfg_options(cfg) - + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end if cfg.schema_management_mode == 'manual_access' then local vexports = require('vshard.storage.exports') local username = 'storage' @@ -264,7 +269,11 @@ local function cluster_new(g, cfg) for _, replica in pairs(replicas) do replica:wait_for_readiness() replica:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end, {cfg}) end @@ -387,8 +396,12 @@ local function cluster_bootstrap(g, cfg) local replicaset_count = 0 local master_info, err = cluster_exec_each(g, function(is_named) local info = box.info + local is_master = ivshard.storage.internal.is_master + if is_master == nil then + is_master = box.cfg.read_only == false + end return { - is_master = ivshard.storage.internal.is_master, + is_master = is_master, rs_id = is_named and info.replicaset.name or ivutil.replicaset_uuid(info), id = is_named and info.name or info.uuid @@ -447,7 +460,11 @@ local function cluster_bootstrap(g, cfg) end local function storage_cfg_impl(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end return ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end @@ -676,7 +693,11 @@ end local function storage_start(storage, cfg) storage:start() local _, err = storage:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end return ivshard.storage.cfg(cfg, _G.box_vstorage_id(cfg)) end, {cfg}) t.assert_equals(err, nil, 'storage cfg on start') @@ -687,7 +708,11 @@ end -- local function router_cfg(router, cfg) router:exec(function(cfg) - ivtest.clear_test_cfg_options(cfg) + if ivtest.clear_test_cfg_options ~= nil then + ivtest.clear_test_cfg_options(cfg) + else + cfg.test_user_grant_range = nil + end ivshard.router.cfg(cfg) end, {cfg}) end diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua index 0fabbcb2..b9620f0f 100644 --- a/test/router-luatest/reload_test.lua +++ b/test/router-luatest/reload_test.lua @@ -88,16 +88,6 @@ g.after_all(function() g.cluster:drop() end) --- --- `vtest.router_cfg` cannot be used in this test, since --- `ivtest.clear_test_cfg_options` may be nil on old versions. --- -local function router_cfg(router, cfg) - router:exec(function(cfg) - ivshard.router.cfg(cfg) - end, {cfg}) -end - local function router_assert_version_equals(router, version) router:exec(function(version) ilt.assert_equals(ivconst.VERSION, version) @@ -124,7 +114,7 @@ local function create_router_at(hash) ['LUA_PATH'] = g.vshard_lua_path }, }) - router_cfg(router, global_cfg) + vtest.router_cfg(router, global_cfg) return router end @@ -228,7 +218,7 @@ g.test_discovery = function(g) end local function test_master_search_template(g, router, auto_master_cfg) - router_cfg(router, auto_master_cfg) + vtest.router_cfg(router, auto_master_cfg) -- Working with first replicaset (2 instances) local rs_uuid = g.replica_1_a:replicaset_uuid() diff --git a/vshard/router/init.lua b/vshard/router/init.lua index 098bca2c..0962944c 100644 --- a/vshard/router/init.lua +++ b/vshard/router/init.lua @@ -1831,9 +1831,9 @@ else if router.is_enabled == nil then router.is_enabled = true end - if router.api_call_cache == nil then - router.api_call_cache = router_api_call_unsafe - end + -- After reloading from an old vshard version, api_call_cache + -- can be nil or point to the old router_api_call signature. + router.api_call_cache = router_api_call_unsafe router_cfg_fiber_safe(router, router.current_cfg, true) setmetatable(router, router_mt) end diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua index 1ad2d44b..7f50d31e 100644 --- a/vshard/storage/init.lua +++ b/vshard/storage/init.lua @@ -3734,6 +3734,20 @@ local function master_sync_service_f(service, limiter) map_res, err, err_id = M.this_replicaset:map_call('vshard.storage._call', {'storage_bucket_checkpoint', call_timeout / 1.5}, call_opts) + if err and (err.type == 'LuajitError' or err.type == 'ClientError') and + err.message and err.message:match( + '/vshard/storage/init%.lua:%d+: attempt to call a nil value$') + then + -- Storages predating storage_bucket_checkpoint() have no + -- production capability endpoint. Their service dispatcher fails + -- with a plain Lua nil-call error when a service is missing. + map_res, err, err_id = + M.this_replicaset:map_call('vshard.storage.sync', + {call_timeout / 1.5}, call_opts) + if not err then + map_res = {} + end + end if err then err.replica_id = err_id limiter:log_warn(err, service:set_status_error(err_msg, err)) @@ -4583,6 +4597,13 @@ if not rawget(_G, MODULE_INTERNALS) then rawset(_G, MODULE_INTERNALS, M) else reload_evolution.upgrade(M) + -- Hot reload from old vshard can leave master sync internals incompatible + -- with the reloaded code. Refresh the function and restart a dead fiber. + M.master_sync_f = master_sync_f + if M.master_sync_fiber ~= nil and + M.master_sync_fiber:status() == 'dead' then + M.master_sync_fiber = nil + end if M.current_cfg then storage_cfg(M.current_cfg, M.this_replica.id or M.this_replica.uuid, true)