From f6b594a3ca929f9082b894e7f12d9e1022ece51f Mon Sep 17 00:00:00 2001
From: Roman Gromov <r.gromov.official@gmail.com>
Date: Tue, 3 Mar 2026 15:35:16 +0300
Subject: [PATCH 1/5] router: refactoring of router_ref_storage_* functions

Before this patch the main `map_callrw` ref functions such as
`router_ref_storage_all` and `router_ref_storage_by_buckets` were
enormous (71 and 108 lines of code). Also these functions have a large
number of similar functional code blocks such as "preparing refs",
"waiting refs" e.t.c. Since in tarantool/vshard#559 patch we will
extend the logic of full map_callrw making it able to work with split
args, the `router_ref_storage_all` can double in size. It can lead to
degradation of our codebase due to less readability.

To fix it we firstly determine general and repeated code blocks in ref
functions:
1) `ref-prepare`: groups buckets by replicasets with router's cache,
   builds a table of "target" replicasets and waits necessary masters.
2) `ref-wait`: waits until future objects are ready in order to
   extract payload from it (responses of storages' functions).
3) `ref-process`: a custom logic for `full` or `partial` map_callrw
   modes which describes how we should process results from future
   objects.

After defining the main stages of ref map_callrw functions we should
unify them so that we can use them in both `router_ref_storage_all` and
`router_ref_storage_by_buckets`.

Needed for tarantool/vshard#559

NO_TEST=refactoring
NO_DOC=refactoring
---
 test/router-luatest/router_2_2_test.lua |   4 +-
 vshard/router/init.lua                  | 238 +++++++++++++-----------
 2 files changed, 128 insertions(+), 114 deletions(-)

diff --git a/test/router-luatest/router_2_2_test.lua b/test/router-luatest/router_2_2_test.lua
index 218accb7..deb96323 100644
--- a/test/router-luatest/router_2_2_test.lua
+++ b/test/router-luatest/router_2_2_test.lua
@@ -209,11 +209,11 @@ end
 
 g.test_group_buckets = function(g)
     local bids = vtest.storage_get_n_buckets(g.replica_1_a, 2)
-    local val, err = g.router:exec(function(bid1, bid2)
+    local val, _ = g.router:exec(function(bid1, bid2)
         return ivshard.router._buckets_group({bid2, bid1, bid1, bid2},
                                              ivtest.wait_timeout)
     end, {bids[1], bids[2]})
-    assert(err == nil)
+    assert(val)
     local rs1_uuid = g.replica_1_a:replicaset_uuid()
     local expected = {
         [rs1_uuid] = {bids[2], bids[1]},
diff --git a/vshard/router/init.lua b/vshard/router/init.lua
index 098bca2c..2511c6a0 100644
--- a/vshard/router/init.lua
+++ b/vshard/router/init.lua
@@ -229,8 +229,8 @@ local function buckets_group(router, bucket_ids, timeout)
         end
         ::continue::
     end
-
-    return replicaset_buckets
+    timeout = deadline - fiber_clock()
+    return replicaset_buckets, timeout
 end
 
 --
@@ -771,60 +771,125 @@ local function router_call(router, bucket_id, opts, ...)
                             ...)
 end
 
+--
+-- Prepares a cluster before sending refs to remote storages:
+--    1) Groups the buckets by replicasets according to the router's cache
+--       in case of partial map_callrw;
+--    2) Builds a table of replicasets on which refs should be sent;
+--    3) Waits necessary masters until all connections are established;
+--
+local function router_map_callrw_prepare(router, timeout, bucket_ids)
+    local err, err_id
+    local replicasets_to_wait, grouped_buckets = {}, {}
+    local replicasets_all = router.replicasets
+    local deadline = fiber_clock() + timeout
+    if bucket_ids then
+        -- Group the buckets by replicasets according to the router cache.
+        grouped_buckets, err = buckets_group(router, bucket_ids, timeout)
+        if not grouped_buckets then
+            return nil, err
+        end
+        for rs_id, _ in pairs(grouped_buckets) do
+            table.insert(replicasets_to_wait, replicasets_all[rs_id])
+        end
+    else
+        replicasets_to_wait = replicasets_all
+    end
+    -- Netbox async requests work only with active connections.
+    -- So, we need to wait for the master connection explicitly.
+    timeout = deadline - fiber_clock()
+    timeout, err, err_id = lreplicaset.wait_masters_connect(
+        replicasets_to_wait, timeout)
+    if not timeout then
+        return nil, err, err_id
+    end
+    return timeout, grouped_buckets
+end
+
+--
+-- Waits until all future objects are ready and extracts results from it.
+--
+local function router_map_callrw_ref_wait(futures, timeout)
+    local results = {}
+    local deadline = fiber_clock() + timeout
+    for id, future in pairs(futures) do
+        timeout = deadline - fiber_clock()
+        local res, err = future_wait(future, timeout)
+        -- Handle netbox error first.
+        if res == nil then
+            return nil, err, id
+        end
+        -- Ref returns nil,err.
+        res, err = res[1], res[2]
+        if res == nil then
+            return nil, err, id
+        end
+        results[id] = res
+    end
+    return timeout, results
+end
+
+--
+-- Handles all buckets which were moved from the remote storages, rewrites the
+-- router's cache and builds a new table of bucket_ids for the next iteration
+-- of partial map_callrw.
+--
+local function router_map_callrw_process_moved(router, results)
+    local bucket_ids = {}
+    for _, res in pairs(results) do
+        for _, bucket in pairs(res.moved) do
+            local bid = bucket.id
+            local dst = bucket.dst
+            -- 'Reset' regardless of 'set'. So as not to
+            -- bother with 'set' errors. If it fails, then
+            -- won't matter. It is a best-effort thing.
+            bucket_reset(router, bid)
+            if dst ~= nil then
+                bucket_set(router, bid, dst)
+            end
+            table.insert(bucket_ids, bid)
+        end
+    end
+    return bucket_ids
+end
+
 --
 -- Perform Ref stage of the Ref-Map-Reduce process on all the known replicasets.
 --
 local function router_ref_storage_all(router, timeout)
     local replicasets = router.replicasets
-    local deadline = fiber_clock() + timeout
-    local err, err_id, res
+    local err, err_id, results, grouped_buckets
     local futures = {}
     local bucket_count = 0
     local opts_async = {is_async = true}
-    local rs_count = 0
     local rid = M.ref_id
     M.ref_id = rid + 1
-    -- Nil checks are done explicitly here (== nil instead of 'not'), because
-    -- netbox requests return box.NULL instead of nils.
 
-    --
-    -- Ref stage: send.
-    --
-    -- Netbox async requests work only with active connections. Need to wait
-    -- for the connection explicitly.
-    timeout, err, err_id = lreplicaset.wait_masters_connect(
-        replicasets, timeout)
+    timeout, grouped_buckets, err_id = router_map_callrw_prepare(
+        router, timeout)
     if not timeout then
+        err = grouped_buckets
         goto fail
     end
-    for id, rs in pairs(replicasets) do
-        res, err = rs:callrw('vshard.storage._call',
-                              {'storage_ref', rid, timeout}, opts_async)
+    futures = {}
+    for rs_id, rs in pairs(replicasets) do
+        local buckets = replicasets[rs_id]
+        local res, ref_err = rs:callrw('vshard.storage._call',
+            {'storage_ref', rid, timeout}, opts_async)
         if res == nil then
-            err_id = id
+            err, err_id = ref_err, rs_id
             goto fail
         end
-        futures[id] = res
-        rs_count = rs_count + 1
+        futures[rs_id] = res
     end
-    --
-    -- Ref stage: collect.
-    --
-    for id, future in pairs(futures) do
-        res, err = future_wait(future, timeout)
-        -- Handle netbox error first.
-        if res == nil then
-            err_id = id
-            goto fail
-        end
-        -- Ref returns nil,err or bucket count.
-        res, err = res[1], res[2]
-        if res == nil then
-            err_id = id
-            goto fail
-        end
+    timeout, results, err_id = router_map_callrw_ref_wait(
+        futures, timeout)
+    if not timeout then
+        err = results
+        goto fail
+    end
+    for _, res in pairs(results) do
         bucket_count = bucket_count + res
-        timeout = deadline - fiber_clock()
     end
     -- All refs are done but not all buckets are covered. This is odd and can
     -- mean many things. The most possible ones: 1) outdated configuration on
@@ -851,103 +916,52 @@ end
 -- replicasets, which contains all the listed bucket IDs.
 --
 local function router_ref_storage_by_buckets(router, bucket_ids, timeout)
-    local grouped_buckets
-    local group_count
-    local err, err_id, res
+    local err, err_id, grouped_buckets, results, args_ref
+    local replicasets_to_map, futures = {}, {}
     local replicasets_all = router.replicasets
-    local replicasets_to_map = {}
-    local futures = {}
     local opts_async = {is_async = true}
-    local deadline = fiber_clock() + timeout
     local rid = M.ref_id
     M.ref_id = rid + 1
-
     -- Nil checks are done explicitly here (== nil instead of 'not'), because
     -- netbox requests return box.NULL instead of nils.
-
-    -- Ref stage.
     while next(bucket_ids) do
-        -- Group the buckets by replicasets according to the router cache.
-        grouped_buckets, err = buckets_group(router, bucket_ids, timeout)
-        if grouped_buckets == nil then
-            goto fail
-        end
-        timeout = deadline - fiber_clock()
-
-        -- Netbox async requests work only with active connections.
-        -- So, first need to wait for the master connection explicitly.
-        local replicasets_to_check = {}
-        group_count = 0
-        for uuid, _ in pairs(grouped_buckets) do
-            group_count = group_count + 1
-            table.insert(replicasets_to_check, replicasets_all[uuid])
-        end
-        timeout, err, err_id = lreplicaset.wait_masters_connect(
-            replicasets_to_check, timeout)
+        timeout, grouped_buckets, err_id = router_map_callrw_prepare(
+            router, timeout, bucket_ids)
         if not timeout then
+            err = grouped_buckets
             goto fail
         end
-
-        -- Send ref requests with timeouts to the replicasets.
-        futures = table_new(0, group_count)
-        for id, buckets in pairs(grouped_buckets) do
-            if timeout == nil then
-                err_id = id
-                goto fail
-            end
-            local args_ref
-            if replicasets_to_map[id] then
-                -- Replicaset is already referenced on a previous iteration.
-                -- Simply get the moved buckets without double referencing.
-                args_ref = {
-                    'storage_ref_check_with_buckets', rid, buckets}
+        futures = {}
+        for rs_id in pairs(grouped_buckets) do
+            local buckets = grouped_buckets[rs_id] or {}
+            if replicasets_to_map[rs_id] then
+                args_ref = {'storage_ref_check_with_buckets', rid, buckets}
             else
-                args_ref = {
-                    'storage_ref_make_with_buckets', rid, timeout, buckets}
+                args_ref = {'storage_ref_make_with_buckets',
+                            rid, timeout, buckets}
             end
-            res, err = replicasets_all[id]:callrw('vshard.storage._call',
-                                                  args_ref, opts_async)
+            local res, ref_err = replicasets_all[rs_id]:callrw(
+                'vshard.storage._call', args_ref, opts_async)
             if res == nil then
-                err_id = id
+                err, err_id = ref_err, rs_id
                 goto fail
             end
-            futures[id] = res
+            futures[rs_id] = res
         end
-
-        -- Wait for the refs to be done and collect moved buckets.
-        bucket_ids = {}
-        for id, f in pairs(futures) do
-            res, err = future_wait(f, timeout)
-            -- Handle netbox error first.
-            if res == nil then
-                err_id = id
-                goto fail
-            end
-            -- Ref returns nil,err or {is_done, moved}.
-            res, err = res[1], res[2]
-            if res == nil then
-                err_id = id
-                goto fail
-            end
-            for _, bucket in pairs(res.moved) do
-                local bid = bucket.id
-                local dst = bucket.dst
-                -- 'Reset' regardless of 'set'. So as not to
-                -- bother with 'set' errors. If it fails, then
-                -- won't matter. It is a best-effort thing.
-                bucket_reset(router, bid)
-                if dst ~= nil then
-                    bucket_set(router, bid, dst)
-                end
-                table.insert(bucket_ids, bid)
-            end
+        timeout, results, err_id = router_map_callrw_ref_wait(
+            futures, timeout)
+        if not timeout then
+            err = results
+            goto fail
+        end
+        bucket_ids = router_map_callrw_process_moved(router, results)
+        for id, res in pairs(results) do
             if res.is_done then
                 assert(not replicasets_to_map[id])
                 -- If there are no buckets on the replicaset, it would not be
                 -- referenced.
                 replicasets_to_map[id] = replicasets_all[id]
             end
-            timeout = deadline - fiber_clock()
         end
     end
     do return timeout, nil, nil, rid, replicasets_to_map end

From 8efa413221272067c6a5bab0bdbc52d168985380 Mon Sep 17 00:00:00 2001
From: Roman Gromov <r.gromov.official@gmail.com>
Date: Mon, 30 Mar 2026 13:04:36 +0300
Subject: [PATCH 2/5] storage: refactoring of bucket_get_moved and
 bucket_are_all_rw_not_cache

In this patch we change `status.SENDING/SENT/GARBAGE/RECEIVING/ACTIVE` on
`BSENDING/BSENT/BGARBAGE/BRECEIVING/BACTIVE` to not repeat and
complicate the code.

Needed for tarantool/vshard#559

NO_DOC=refactoring
NO_TEST=refactoring
---
 vshard/storage/init.lua | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua
index 1ad2d44b..e67f11d1 100644
--- a/vshard/storage/init.lua
+++ b/vshard/storage/init.lua
@@ -442,12 +442,11 @@ end
 
 local function bucket_are_all_rw_not_cache()
     local status_index = box.space._bucket.index.status
-    local status = consts.BUCKET
-    local res = not index_has(status_index, status.SENDING) and
-       not index_has(status_index, status.SENT) and
-       not index_has(status_index, status.RECEIVING) and
-       not index_has(status_index, status.GARBAGE) and
-       not index_has(status_index, status.READONLY)
+    local res = not index_has(status_index, BSENDING) and
+       not index_has(status_index, BSENT) and
+       not index_has(status_index, BRECEIVING) and
+       not index_has(status_index, BGARBAGE) and
+       not index_has(status_index, BREADONLY)
 
     M.bucket_are_all_rw_cache = res
     bucket_are_all_rw = bucket_are_all_rw_cache
@@ -3511,7 +3510,6 @@ end
 -- under any circumstances.
 --
 local function bucket_get_moved(bucket_ids)
-    local allstatus = consts.BUCKET
     local res = {}
     for _, bucket_id in pairs(bucket_ids) do
         local bucket = box.space._bucket:get{bucket_id}
@@ -3520,7 +3518,7 @@ local function bucket_get_moved(bucket_ids)
             is_moved = true
         else
             local status = bucket.status
-            is_moved = status == allstatus.GARBAGE or status == allstatus.SENT
+            is_moved = status == BGARBAGE or status == BSENT
         end
         if is_moved then
             table.insert(res, {

From b654504a57e9f7ef66107fcd6881f5e40ca99d62 Mon Sep 17 00:00:00 2001
From: Roman Gromov <r.gromov.official@gmail.com>
Date: Tue, 3 Mar 2026 15:49:08 +0300
Subject: [PATCH 3/5] router: refactoring of ref_ids' logic in
 router_map_callrw

This patch takes initialization of `rid` out to `router_map_callrw` and
passes this variable to ref-functions. It is needed for future features
tidiness, for example - `make full map_callrw with split args` in which
the logic of `router_map_callrw` becomes more complex.

Needed for tarantool/vshard#559

NO_DOC=refactoring
NO_TEST=refactoring
---
 vshard/router/init.lua | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/vshard/router/init.lua b/vshard/router/init.lua
index 2511c6a0..0010d9bd 100644
--- a/vshard/router/init.lua
+++ b/vshard/router/init.lua
@@ -856,14 +856,12 @@ end
 --
 -- Perform Ref stage of the Ref-Map-Reduce process on all the known replicasets.
 --
-local function router_ref_storage_all(router, timeout)
+local function router_ref_storage_all(router, timeout, rid)
     local replicasets = router.replicasets
     local err, err_id, results, grouped_buckets
     local futures = {}
     local bucket_count = 0
     local opts_async = {is_async = true}
-    local rid = M.ref_id
-    M.ref_id = rid + 1
 
     timeout, grouped_buckets, err_id = router_map_callrw_prepare(
         router, timeout)
@@ -902,26 +900,24 @@ local function router_ref_storage_all(router, timeout)
                             router.total_bucket_count - bucket_count)
         goto fail
     end
-    do return timeout, nil, nil, rid, replicasets end
+    do return timeout, nil, nil, replicasets end
 
     ::fail::
     for _, f in pairs(futures) do
         f:discard()
     end
-    return nil, err, err_id, rid, replicasets
+    return nil, err, err_id, replicasets
 end
 
 --
 -- Perform Ref stage of the Ref-Map-Reduce process on a subset of all the
 -- replicasets, which contains all the listed bucket IDs.
 --
-local function router_ref_storage_by_buckets(router, bucket_ids, timeout)
+local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid)
     local err, err_id, grouped_buckets, results, args_ref
     local replicasets_to_map, futures = {}, {}
     local replicasets_all = router.replicasets
     local opts_async = {is_async = true}
-    local rid = M.ref_id
-    M.ref_id = rid + 1
     -- Nil checks are done explicitly here (== nil instead of 'not'), because
     -- netbox requests return box.NULL instead of nils.
     while next(bucket_ids) do
@@ -964,13 +960,13 @@ local function router_ref_storage_by_buckets(router, bucket_ids, timeout)
             end
         end
     end
-    do return timeout, nil, nil, rid, replicasets_to_map end
+    do return timeout, nil, nil, replicasets_to_map end
 
     ::fail::
     for _, f in pairs(futures) do
         f:discard()
     end
-    return nil, err, err_id, rid, replicasets_to_map
+    return nil, err, err_id, replicasets_to_map
 end
 
 --
@@ -1160,17 +1156,20 @@ local function router_map_callrw(router, func, args, opts)
     else
         timeout = consts.CALL_TIMEOUT_MIN
     end
+    rid = M.ref_id
+    M.ref_id = rid + 1
     if plain_bucket_ids then
-        timeout, err, err_id, rid, replicasets_to_map =
-            router_ref_storage_by_buckets(router, plain_bucket_ids, timeout)
+        timeout, err, err_id, replicasets_to_map =
+            router_ref_storage_by_buckets(router, plain_bucket_ids, timeout,
+                                          rid)
         -- Grouped arguments are only possible with partial Map-Reduce.
         if timeout then
             grouped_args = router_group_map_callrw_args(
                 router, plain_bucket_ids, bucket_ids)
         end
     else
-        timeout, err, err_id, rid, replicasets_to_map =
-            router_ref_storage_all(router, timeout)
+        timeout, err, err_id, replicasets_to_map =
+            router_ref_storage_all(router, timeout, rid)
     end
     if timeout then
         map, err, err_id = replicasets_map_reduce(replicasets_to_map, rid, func,

From 5c72de78d28bf26d95c407b8317d418910ee3951 Mon Sep 17 00:00:00 2001
From: Roman Gromov <r.gromov.official@gmail.com>
Date: Fri, 27 Mar 2026 15:11:03 +0300
Subject: [PATCH 4/5] test: refactoring of router/reload_test

Before this patch the `router-luatest/reload_test` checked router's
services only with old routers. However in future patch (gh-214) we
need to check map_callrw with old storages.

In order to make it able we:
1) change `vtest.cluster_new` so that we can pass server_config with
   certain ENV (LUA_PATH) variable into it. It can help us to create
   a new cluster on old version of vshard.
2) change `reload_router` to more general `reload_server` in order to
   unify the process of servers (router / storage) upgrade in
   `router-luatest/reload_test`.
3) unify the process of cluster creation -
   `create_cluster_on_specific_version` and the process of getting
   server's config with new ENV (LUA_PATH) variable -
   `get_config_for_specific_vshard_version`.

Needed for tarantool/vshard#214

NO_DOC=refactoring
NO_TEST=refactoring
---
 test/luatest_helpers/cluster.lua    |   1 +
 test/luatest_helpers/vtest.lua      |   4 +-
 test/router-luatest/reload_test.lua | 106 +++++++++++++++-------------
 3 files changed, 62 insertions(+), 49 deletions(-)

diff --git a/test/luatest_helpers/cluster.lua b/test/luatest_helpers/cluster.lua
index 43e3479f..e247bad8 100644
--- a/test/luatest_helpers/cluster.lua
+++ b/test/luatest_helpers/cluster.lua
@@ -42,6 +42,7 @@ function Cluster:drop()
             server:cleanup()
         end
     end
+    self.servers = {}
 end
 
 function Cluster:get_index(server)
diff --git a/test/luatest_helpers/vtest.lua b/test/luatest_helpers/vtest.lua
index f9219439..551ae898 100644
--- a/test/luatest_helpers/vtest.lua
+++ b/test/luatest_helpers/vtest.lua
@@ -148,7 +148,8 @@ end
 --
 -- Build new cluster by a given config.
 --
-local function cluster_new(g, cfg)
+local function cluster_new(g, cfg, server_config)
+    server_config = server_config or {}
     if not g.cluster then
         g.cluster = cluster:new({})
     end
@@ -204,6 +205,7 @@ local function cluster_new(g, cfg)
             local server = g.cluster:build_server({
                 alias = replica_name,
                 box_cfg = box_cfg,
+                env = server_config.env,
             }, 'storage.lua')
             g[replica_name] = server
             -- VShard specific details to use in various helper functions.
diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua
index c1ae28d7..1cff79d1 100644
--- a/test/router-luatest/reload_test.lua
+++ b/test/router-luatest/reload_test.lua
@@ -28,6 +28,39 @@ local cfg_template = {
 }
 local global_cfg
 
+local function get_config_for_specific_version()
+    local path = g.vshard_copy_path_load
+    local lua_path = string.format('%s/?.lua;%s/?/init.lua;', path, path)
+    -- Force 'require' to use new directory
+    return {env = {['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH')}}
+end
+
+local function create_cluster_on_specific_version(hash)
+    git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path})
+    local server_config = get_config_for_specific_version()
+    vtest.cluster_new(g, global_cfg, server_config)
+    vtest.cluster_bootstrap(g, global_cfg)
+    vtest.cluster_rebalancer_disable(g)
+
+    vtest.cluster_exec_each_master(g, function()
+        local test = box.schema.space.create('test', {format = {
+            {'id', 'unsigned'},
+            {'bucket_id', 'unsigned'},
+        }})
+
+        test:create_index('primary')
+        test:create_index('bucket_id', {unique = false, parts = {2}})
+    end)
+    vtest.cluster_exec_each(g, function()
+        rawset(_G, 'insert', function(space_name, tuple)
+            return box.space[space_name]:insert(tuple)
+        end)
+        rawset(_G, 'get', function(space_name, key)
+            return box.space[space_name]:get(key)
+        end)
+    end)
+end
+
 g.before_all(function()
     -- Override of the built in modules is available only since 2.11.0.
     t.run_only_if(vutil.version_is_at_least(2, 11, 0, nil, 0, 0))
@@ -62,31 +95,7 @@ g.before_all(function()
 
     -- Hash of the latest commit for testing router on the latest version.
     g.latest_hash = git_util.log_hashes({args = '-1', dir = vtest.sourcedir})[1]
-
-    -- No need to reload storages. Just run them on the latest version.
-    vtest.cluster_new(g, global_cfg)
-    vtest.cluster_bootstrap(g, global_cfg)
-    vtest.cluster_rebalancer_disable(g)
-
-    -- Basic storage configuration
-    vtest.cluster_exec_each_master(g, function()
-        local test = box.schema.space.create('test', {format = {
-            {'id', 'unsigned'},
-            {'bucket_id', 'unsigned'},
-        }})
-
-        test:create_index('primary')
-        test:create_index('bucket_id', {unique = false, parts = {2}})
-    end)
-
-    vtest.cluster_exec_each(g, function()
-        rawset(_G, 'insert', function(space_name, tuple)
-            return box.space[space_name]:insert(tuple)
-        end)
-        rawset(_G, 'get', function(space_name, key)
-            return box.space[space_name]:get(key)
-        end)
-    end)
+    create_cluster_on_specific_version(g.latest_hash)
 end)
 
 g.after_all(function()
@@ -115,7 +124,7 @@ end
 --         * Checkout to old version;
 --         * Create and start a router;
 --     3. Test smth on the old version;
---     4. Invoke reload_router():
+--     4. Invoke reload_server():
 --         * Checkout to the latest version;
 --         * Reload the module.
 --     5. Test smth on the new version
@@ -123,47 +132,48 @@ end
 --
 local function create_router_at(hash)
     git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path})
-    local path = g.vshard_copy_path_load
-    local lua_path = string.format("%s/?.lua;%s/?/init.lua;", path, path)
-    local router = vtest.router_new(g, 'router', nil, {
-        env = {
-            -- Force 'require' to use new directory
-            ['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH')
-        },
-    })
+    local server_config = get_config_for_specific_version()
+    local router = vtest.router_new(g, 'router', nil, server_config)
     router_cfg(router, global_cfg)
     return router
 end
 
 --
--- Reloads router. If service_name is provided, then
+-- Reloads server (router or storage). If service_name is provided, then
 -- the function also waits until the service is restarted.
-local function reload_router(router, service_name)
+--
+local function reload_server(server, server_type, service_name)
     git_util.exec('checkout', {args = g.latest_hash .. ' -f',
                                dir = g.vshard_copy_path})
-    router:exec(function(service_name)
+    server:exec(function(server_type, service_name)
         local service
-        local internal = ivshard.router.internal
+        local internal = ivshard[server_type].internal
         if service_name ~= nil then
-            service = internal.static_router[service_name]
+            if server_type == 'router' then
+                service = internal.static_router[service_name]
+            else
+                service = internal[service_name]
+            end
             ilt.assert_not_equals(service, nil)
         end
 
-        ilt.assert_equals(ivshard.router.module_version(), 0)
-        package.loaded['vshard.router'] = nil
-        ivshard.router = require('vshard.router')
+        local package_name = string.format('vshard.%s', server_type)
+        ilt.assert_equals(ivshard[server_type].module_version(), 0)
+        package.loaded[package_name] = nil
+        ivshard[server_type] = require(package_name)
         _G.ivconst = require('vshard.consts')
-        ilt.assert_equals(ivshard.router.module_version(), 1)
+        ilt.assert_equals(ivshard[server_type].module_version(), 1)
 
         if service ~= nil then
             ilt.helpers.retrying({timeout = ivtest.wait_timeout,
                                   delay = ivtest.busy_step}, function()
-                if service == internal.static_router[service_name] then
+                if service == internal.static_router[service_name] or
+                   service == internal[service_name] then
                     error('Service have not been reloaded yet')
                 end
             end)
         end
-    end, {service_name})
+    end, {server_type, service_name})
 end
 
 local function test_basic_template(router)
@@ -185,7 +195,7 @@ g.test_basic = function(g)
     local router = create_router_at(hash)
     router_assert_version_equals(router, nil)
     test_basic_template(router)
-    reload_router(router)
+    reload_server(router, 'router')
     router_assert_version_equals(router, vconsts.VERSION)
     test_basic_template(router)
     vtest.drop_instance(g, router)
@@ -228,7 +238,7 @@ g.test_discovery = function(g)
     local router = create_router_at(hash)
     router_assert_version_equals(router, nil)
     test_discovery_template(g, router)
-    reload_router(router, 'discovery_service')
+    reload_server(router, 'router', 'discovery_service')
     router_assert_version_equals(router, vconsts.VERSION)
     test_discovery_template(g, router)
     vtest.drop_instance(g, router)
@@ -327,7 +337,7 @@ g.test_master_search = function(g)
     local auto_master_cfg = vtest.config_new(auto_master_cfg_template)
 
     test_master_search_template(g, router, auto_master_cfg)
-    reload_router(router)
+    reload_server(router, 'router')
     router_assert_version_equals(router, vconsts.VERSION)
     -- Wait for old master_search service to be stopped.
     router:exec(function()

From 23abb9a4a6e8feaaf1cfe8358de16e43649e73ec Mon Sep 17 00:00:00 2001
From: Roman Gromov <r.gromov.official@gmail.com>
Date: Fri, 27 Mar 2026 15:11:48 +0300
Subject: [PATCH 5/5] router: make full map_callrw with split args

This patch introduces a new way of `map_callrw` execution by which we can
pass some arguments to all storages and split buckets' arguments to those
storages that have at least one bucket of `bucket_ids`. To achieve this we
introduce a new string option - `mode` to `map_callrw` api.

Also we change the logic of `router_ref_storage_all` ref function. Firstly
we ref all storages and get back an amount of "moved" buckets according
to the previously built router's cache. Then if there are no "moved"
buckets we accumulate and check total amount of buckets on all storages
and finish map_callrw ref stage. Otherwise, if there are some "moved"
buckets we perform the second network hop by checking on which replicasets
do the remaining "moved" buckets reside on.

Closes tarantool#559

@TarantoolBot document

Title: vshard: `mode` option for `router.map_callrw()`

This string option regulates on which storages the user function will be
executed via `map_callrw`. Possible values:
1) mode = 'partial'. In this mode user function will be executed on
   storages that have at least one bucket of 'bucket_ids'. The
   'bucket_ids' option can be presented in two ways: like a numeric array
   of buckets' ids or like a map of buckets' arguments. In first one user
   function will only receive args, in second one it will additionally
   receive buckets' arguments.
2) mode = 'full'. In this mode user function will be executed with args on
   all storages in cluster. Also we can pass 'bucket_ids' only like a map
   of bucket's arguments - `bucket_ids = {id = {args}, id = {args}, ...}`.
   In this way the user function will additionally receive buckets'
   arguments on those storages that have at least one bucket of
   `bucket_ids`.

If we didn't specify the 'mode' option, then it is set based on
'bucket_ids' option for backward compatibility. If 'bucket_ids' is
presented, the mode will be 'partial' otherwise 'full'.

Also now `map_callrw` ends with error in cases when we
1) pass `mode = 'full'` and a numeric array of buckets' ids -
   `bucket_ids = {id, id, ...}`
2) pass `mode = 'partial'` and `bucket_ids = nil`.
---
 test/router-luatest/map_callrw_test.lua | 177 ++++++++++++++++++++++
 test/router-luatest/reload_test.lua     | 126 +++++++++++++---
 test/storage-luatest/storage_1_test.lua |  19 ++-
 test/upgrade/upgrade.result             |   1 +
 vshard/consts.lua                       |   5 +
 vshard/error.lua                        |   5 +
 vshard/router/init.lua                  | 190 ++++++++++++++++++------
 vshard/storage/init.lua                 |  39 ++++-
 8 files changed, 485 insertions(+), 77 deletions(-)

diff --git a/test/router-luatest/map_callrw_test.lua b/test/router-luatest/map_callrw_test.lua
index 27482e2e..0d251dfc 100644
--- a/test/router-luatest/map_callrw_test.lua
+++ b/test/router-luatest/map_callrw_test.lua
@@ -691,3 +691,180 @@ g.test_map_callrw_with_cdata_bucket_id = function(cg)
         ilt.assert_not(err)
     end)
 end
+
+g.test_full_map_callrw_with_numeric_bucket_ids = function(cg)
+    local res = router_do_map(cg.router, {123}, {
+        mode = 'full',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {1, 2, 3}
+    })
+    t.assert(res.err)
+    t.assert_equals(res.err.message, 'Router can\'t execute map_callrw ' ..
+                    'with \'full\' mode and numeric bucket_ids')
+    t.assert_not(res.err_id)
+    t.assert_not(res.val)
+end
+
+g.test_partial_map_callrw_with_nil_bucket_ids = function(cg)
+    local res = router_do_map(cg.router, {123}, {
+        mode = 'partial',
+        timeout = vtest.wait_timeout
+    })
+    t.assert(res.err)
+    t.assert_equals(res.err.message, 'Router can\'t execute map_callrw ' ..
+                    'with \'partial\' mode and nil bucket_ids')
+    t.assert_not(res.err_id)
+    t.assert_not(res.val)
+end
+
+local function make_do_map_tracking_bucket_ids(cg)
+    -- We override 'do_map' function on storages in order to check that
+    -- default arguments and bucket arguments were successfully passed into
+    -- destination storages according to mode and bucket_ids options.
+    vtest.cluster_exec_each_master(cg, function()
+        rawset(_G, 'old_do_map', _G.do_map)
+        rawset(_G, 'do_map', function(args, bucket_args)
+            ilt.assert_gt(require('vshard.storage.ref').count, 0)
+            return {ivutil.replicaset_uuid(),
+                    {args = args, b_args = bucket_args}}
+        end)
+    end)
+end
+
+local function reset_do_map_to_old_state(cg)
+    vtest.cluster_exec_each_master(cg, function()
+        rawset(_G, 'do_map', _G.old_do_map)
+    end)
+end
+
+g.test_full_map_callrw_with_split_args = function(cg)
+    make_do_map_tracking_bucket_ids(cg)
+    local bid1 = vtest.storage_first_bucket(g.replica_1_a)
+    local bid2 = vtest.storage_first_bucket(g.replica_2_a)
+    local bid3 = vtest.storage_first_bucket(g.replica_3_a)
+
+    local res = router_do_map(cg.router, {0}, {
+        mode = 'full',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {[bid1] = {111}, [bid2] = {222}, [bid3] = {333}}
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {
+        [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0, b_args = {[bid1] = {111}}}}},
+        [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}},
+        [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0, b_args = {[bid3] = {333}}}}},
+    })
+
+    res = router_do_map(cg.router, {0}, {
+        mode = 'full',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {[bid2] = {222}}
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {
+        [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}},
+        [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}},
+        [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0}}},
+    })
+    reset_do_map_to_old_state(cg)
+end
+
+g.test_full_map_callrw_without_bucket_ids = function(cg)
+    make_do_map_tracking_bucket_ids(cg)
+    local res = router_do_map(cg.router, {0}, {
+        mode = 'full',
+        timeout = vtest.wait_timeout
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {
+        [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}},
+        [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0}}},
+        [cg.rs3_uuid] = {{cg.rs3_uuid, {args = 0}}},
+    })
+    reset_do_map_to_old_state(cg)
+end
+
+g.test_partial_map_callrw_with_numeric_bucket_ids = function(cg)
+    make_do_map_tracking_bucket_ids(cg)
+    local bid1 = vtest.storage_first_bucket(g.replica_1_a)
+    local res = router_do_map(cg.router, {0}, {
+        mode = 'partial',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {bid1}
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {[cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0}}}})
+    reset_do_map_to_old_state(cg)
+end
+
+g.test_partial_map_callrw_with_split_args = function(cg)
+    make_do_map_tracking_bucket_ids(cg)
+    local bid1 = vtest.storage_first_bucket(g.replica_1_a)
+    local bid2 = vtest.storage_first_bucket(g.replica_2_a)
+    local res = router_do_map(cg.router, {0}, {
+        mode = 'partial',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {[bid1] = {111}, [bid2] = {222}}
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {
+        [cg.rs1_uuid] = {{cg.rs1_uuid, {args = 0, b_args = {[bid1] = {111}}}}},
+        [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0, b_args = {[bid2] = {222}}}}},
+    })
+    reset_do_map_to_old_state(cg)
+end
+
+local function move_bucket(src_storage, dest_storage, bucket_id)
+    src_storage:exec(function(bucket_id, replicaset_id)
+        t.helpers.retrying({timeout = 60}, function()
+            local res, err = ivshard.storage.bucket_send(bucket_id,
+                                                         replicaset_id)
+            t.assert_not(err)
+            t.assert(res)
+        end)
+    end, {bucket_id, dest_storage:replicaset_uuid()})
+    src_storage:exec(function(bucket_id)
+        t.helpers.retrying({timeout = 10}, function()
+            t.assert_equals(box.space._bucket:select(bucket_id), {})
+        end)
+    end, {bucket_id})
+    dest_storage:exec(function(bucket_id)
+        t.helpers.retrying({timeout = 10}, function()
+            t.assert_equals(box.space._bucket:get(bucket_id).status, 'active')
+        end)
+    end, {bucket_id})
+end
+
+g.test_full_map_callrw_with_split_args_and_broken_cache = function(cg)
+    make_do_map_tracking_bucket_ids(cg)
+    cg.router:exec(function()
+        ivshard.router.internal.errinj.ERRINJ_LONG_DISCOVERY = true
+        ivshard.router.discovery_wakeup()
+    end)
+
+    local moved_bucket = vtest.storage_first_bucket(cg.replica_1_a)
+    move_bucket(g.replica_1_a, g.replica_2_a, moved_bucket)
+    local res = router_do_map(cg.router, {0}, {
+        mode = 'partial',
+        timeout = vtest.wait_timeout,
+        bucket_ids = {[moved_bucket] = {111}}
+    })
+    t.assert_not(res.err)
+    t.assert_not(res.err_id)
+    t.assert_equals(res.val, {
+        [cg.rs2_uuid] = {{cg.rs2_uuid, {args = 0,
+                                        b_args = {[moved_bucket] = {111}}}}},
+    })
+
+    cg.router:exec(function()
+        ivshard.router.internal.errinj.ERRINJ_LONG_DISCOVERY = false
+        ivshard.router.discovery_wakeup()
+    end)
+    move_bucket(g.replica_2_a, g.replica_1_a, moved_bucket)
+    reset_do_map_to_old_state(cg)
+end
diff --git a/test/router-luatest/reload_test.lua b/test/router-luatest/reload_test.lua
index 1cff79d1..762efe65 100644
--- a/test/router-luatest/reload_test.lua
+++ b/test/router-luatest/reload_test.lua
@@ -28,16 +28,16 @@ local cfg_template = {
 }
 local global_cfg
 
-local function get_config_for_specific_version()
+local function get_config_for_specific_version(g)
     local path = g.vshard_copy_path_load
     local lua_path = string.format('%s/?.lua;%s/?/init.lua;', path, path)
     -- Force 'require' to use new directory
     return {env = {['LUA_PATH'] = lua_path .. os.getenv('LUA_PATH')}}
 end
 
-local function create_cluster_on_specific_version(hash)
+local function create_cluster_on_specific_version(g, hash)
     git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path})
-    local server_config = get_config_for_specific_version()
+    local server_config = get_config_for_specific_version(g)
     vtest.cluster_new(g, global_cfg, server_config)
     vtest.cluster_bootstrap(g, global_cfg)
     vtest.cluster_rebalancer_disable(g)
@@ -58,17 +58,21 @@ local function create_cluster_on_specific_version(hash)
         rawset(_G, 'get', function(space_name, key)
             return box.space[space_name]:get(key)
         end)
+        rawset(_G, 'do_map', function(args, bucket_args)
+            ilt.assert_gt(require('vshard.storage.ref').count, 0)
+            return {ivutil.replicaset_uuid(),
+                    {args = args, b_args = bucket_args}}
+        end)
     end)
 end
 
-g.before_all(function()
+local function prepare_temp_workspace(g)
     -- Override of the built in modules is available only since 2.11.0.
     t.run_only_if(vutil.version_is_at_least(2, 11, 0, nil, 0, 0))
     global_cfg = vtest.config_new(cfg_template)
 
     -- The test works in the following directory
-    local vardir = vtest.vardir or fio.tempdir()
-    g.vshard_copy_path_load =  vardir .. '/vshard_copy'
+    g.vshard_copy_path_load = fio.tempdir() .. '/vshard_copy'
     t.assert_equals(fio.mkdir(g.vshard_copy_path_load), true)
     --
     -- Tarantool searches for compilation units in the following order:
@@ -86,7 +90,7 @@ g.before_all(function()
     -- So we can use package.setsearchroot() to change cwd, luatest's `chdir`
     -- or override. The last one is used here, since it's the easiest.
     --
-    g.vshard_copy_path =  vardir .. '/vshard_copy/override'
+    g.vshard_copy_path = g.vshard_copy_path_load .. '/override'
     t.assert_equals(fio.mkdir(g.vshard_copy_path), true)
     -- Copy source to the temporary directory
     t.assert_equals(fio.mkdir(g.vshard_copy_path .. '/.git'), true)
@@ -95,10 +99,21 @@ g.before_all(function()
 
     -- Hash of the latest commit for testing router on the latest version.
     g.latest_hash = git_util.log_hashes({args = '-1', dir = vtest.sourcedir})[1]
-    create_cluster_on_specific_version(g.latest_hash)
+end
+
+local function remove_temp_workspace(g)
+    local is_success, err = fio.rmtree(g.vshard_copy_path_load)
+    t.assert_not(err)
+    t.assert(is_success)
+end
+
+g.before_all(function()
+    prepare_temp_workspace(g)
+    create_cluster_on_specific_version(g, g.latest_hash)
 end)
 
 g.after_all(function()
+    remove_temp_workspace(g)
     g.cluster:drop()
 end)
 
@@ -130,9 +145,9 @@ end
 --     5. Test smth on the new version
 --     6. Drop a router with vtest.drop_instance
 --
-local function create_router_at(hash)
+local function create_router_at(g, hash)
     git_util.exec('checkout', {args = hash .. ' -f', dir = g.vshard_copy_path})
-    local server_config = get_config_for_specific_version()
+    local server_config = get_config_for_specific_version(g)
     local router = vtest.router_new(g, 'router', nil, server_config)
     router_cfg(router, global_cfg)
     return router
@@ -142,7 +157,7 @@ end
 -- Reloads server (router or storage). If service_name is provided, then
 -- the function also waits until the service is restarted.
 --
-local function reload_server(server, server_type, service_name)
+local function reload_server(g, server, server_type, service_name)
     git_util.exec('checkout', {args = g.latest_hash .. ' -f',
                                dir = g.vshard_copy_path})
     server:exec(function(server_type, service_name)
@@ -192,10 +207,10 @@ g.test_basic = function(g)
     -- Latest meaningful commit:
     --     "router: fix reload problem with global function refs".
     local hash = '139223269cddefe2ba4b8e9f6e44712f099f4b35'
-    local router = create_router_at(hash)
+    local router = create_router_at(g, hash)
     router_assert_version_equals(router, nil)
     test_basic_template(router)
-    reload_server(router, 'router')
+    reload_server(g, router, 'router')
     router_assert_version_equals(router, vconsts.VERSION)
     test_basic_template(router)
     vtest.drop_instance(g, router)
@@ -235,10 +250,10 @@ g.test_discovery = function(g)
     -- the test cannot be run without it:
     --     "router: add saving of background service statuses".
     local hash = 'f5f386a5a35e6e5efd8f4f2ed1b3d208fdae9095'
-    local router = create_router_at(hash)
+    local router = create_router_at(g, hash)
     router_assert_version_equals(router, nil)
     test_discovery_template(g, router)
-    reload_server(router, 'router', 'discovery_service')
+    reload_server(g, router, 'router', 'discovery_service')
     router_assert_version_equals(router, vconsts.VERSION)
     test_discovery_template(g, router)
     vtest.drop_instance(g, router)
@@ -323,7 +338,7 @@ g.test_master_search = function(g)
     -- the test cannot be run without it:
     --     "router: add saving of background service statuses".
     local hash = 'f5f386a5a35e6e5efd8f4f2ed1b3d208fdae9095'
-    local router = create_router_at(hash)
+    local router = create_router_at(g, hash)
     router_assert_version_equals(router, nil)
 
     -- Enable auto master search
@@ -337,7 +352,7 @@ g.test_master_search = function(g)
     local auto_master_cfg = vtest.config_new(auto_master_cfg_template)
 
     test_master_search_template(g, router, auto_master_cfg)
-    reload_server(router, 'router')
+    reload_server(g, router, 'router')
     router_assert_version_equals(router, vconsts.VERSION)
     -- Wait for old master_search service to be stopped.
     router:exec(function()
@@ -353,3 +368,80 @@ g.test_master_search = function(g)
     test_master_search_template(g, router, auto_master_cfg)
     vtest.drop_instance(g, router)
 end
+
+local g2 = t.group('reload_storage')
+
+g2.before_all(function()
+    prepare_temp_workspace(g2)
+    -- Full map_callrw with split args was introduced just right after
+    -- this commit. We need to test the behavior of map_callrw on old
+    -- storage versions in order to check that there will be no crashes
+    -- of storages due to changes in storage_ref_* functions.
+    -- Latest meaningful commit:
+    --     "Bump version to 0.1.41"
+    create_cluster_on_specific_version(g2,
+        '42081581f25e671a17fddb7f9873ddcf5b0130c6')
+end)
+
+g2.after_all(function()
+    remove_temp_workspace(g2)
+    g2.cluster:drop()
+end)
+
+g2.test_map_callrw = function(g2)
+    local rs_uuids = {g2.replica_1_a:replicaset_uuid(),
+                      g2.replica_2_a:replicaset_uuid()}
+    local router = vtest.router_new(g2, 'router')
+    router_cfg(router, global_cfg)
+    -- The latest router and old masters
+    router:exec(function(rs1_uuid, rs2_uuid)
+        -- Full map_callrw
+        local res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'},
+            {timeout = ivtest.wait_timeout, mode = 'full'})
+        t.assert_not(err)
+        t.assert_not(err_id)
+        t.assert_equals(res, {
+            [rs1_uuid] = {{rs1_uuid, {args = 'arg_1'}}},
+            [rs2_uuid] = {{rs2_uuid, {args = 'arg_1'}}},
+        })
+        -- Full map_callrw with split args
+        res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'},
+            {timeout = 3, mode = 'full', bucket_ids = {[1] = {'b_arg_1'}}})
+        t.assert_equals(err.name, 'UNSUPPORTED')
+        t.assert_not(err_id)
+        t.assert_not(res)
+        -- Partial map_callrw with numeric buckets' args
+        res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'},
+            {timeout = 3, mode = 'partial', bucket_ids = {1,}})
+        t.assert_not(err)
+        t.assert_not(err_id)
+        t.assert_equals(res, {
+            [rs1_uuid] = {{rs1_uuid, {args = 'arg_1'}}},
+        })
+        -- Partial map_callrw with split args
+        res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'},
+            {timeout = 3, mode = 'partial', bucket_ids = {[1] = {'b_arg_1'}}})
+        t.assert_not(err)
+        t.assert_not(err_id)
+        t.assert_equals(res, {
+            [rs1_uuid] = {{rs1_uuid, {args = 'arg_1',
+                                      b_args = {{'b_arg_1'}}}}},
+        })
+    end, rs_uuids)
+    for _, storage in pairs({g2.replica_1_a, g2.replica_2_a}) do
+        reload_server(g2, storage, 'storage')
+    end
+    -- The latest router and latest masters
+    router:exec(function(rs1_uuid, rs2_uuid)
+        -- Full map_callrw with split args
+        local res, err, err_id = ivshard.router.map_callrw('do_map', {'arg_1'},
+            {timeout = 3, mode = 'full', bucket_ids = {[1] = {'b_arg_1'}}})
+        t.assert_not(err)
+        t.assert_not(err_id)
+        t.assert_equals(res, {
+            [rs1_uuid] = {{rs1_uuid, {args = 'arg_1',
+                                      b_args = {{'b_arg_1'}}}}},
+            [rs2_uuid] = {{rs2_uuid, {args = 'arg_1'}}},
+        })
+    end, rs_uuids)
+end
diff --git a/test/storage-luatest/storage_1_test.lua b/test/storage-luatest/storage_1_test.lua
index 95a99c44..8f61f475 100644
--- a/test/storage-luatest/storage_1_test.lua
+++ b/test/storage-luatest/storage_1_test.lua
@@ -250,14 +250,14 @@ test_group.test_ref_with_buckets_basic = function(g)
         res, err = ivshard.storage._call(
             'storage_ref_make_with_buckets', rid, iwait_timeout, {})
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {moved = {}})
+        ilt.assert_equals(res, {moved = {}, total = 10})
         ilt.assert_equals(lref.count, 0)
 
         -- Check for a single ok bucket.
         res, err = ivshard.storage._call(
             'storage_ref_make_with_buckets', rid, iwait_timeout, {bids[1]})
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {is_done = true, moved = {}})
+        ilt.assert_equals(res, {is_done = true, moved = {}, total = 10})
         ilt.assert_equals(lref.count, 1)
         _, err = ivshard.storage._call('storage_unref', rid)
         ilt.assert_equals(err, nil)
@@ -268,7 +268,7 @@ test_group.test_ref_with_buckets_basic = function(g)
             'storage_ref_make_with_buckets', rid, iwait_timeout,
             {bids[1], bids[2]})
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {is_done = true, moved = {}})
+        ilt.assert_equals(res, {is_done = true, moved = {}, total = 10})
         _, err = ivshard.storage._call('storage_unref', rid)
         ilt.assert_equals(err, nil)
 
@@ -277,7 +277,7 @@ test_group.test_ref_with_buckets_basic = function(g)
             'storage_ref_make_with_buckets', rid, iwait_timeout,
             {bids[1], bids[1]})
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {is_done = true, moved = {}})
+        ilt.assert_equals(res, {is_done = true, moved = {}, total = 10})
         ilt.assert_equals(lref.count, 1)
         _, err = ivshard.storage._call('storage_unref', rid)
         ilt.assert_equals(err, nil)
@@ -295,7 +295,8 @@ test_group.test_ref_with_buckets_basic = function(g)
                 {id = bucket_count + 1},
                 {id = bucket_count + 2},
                 {id = bucket_count + 3},
-            }
+            },
+            total = 10,
         })
         _, err = ivshard.storage._call('storage_unref', rid)
         ilt.assert_equals(err, nil)
@@ -308,7 +309,7 @@ test_group.test_ref_with_buckets_basic = function(g)
             {bucket_count + 1, bucket_count + 2}
         )
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {moved = {
+        ilt.assert_equals(res, {total = 10, moved = {
             {id = bucket_count + 1},
             {id = bucket_count + 2},
         }})
@@ -355,7 +356,7 @@ test_group.test_ref_with_buckets_return_last_known_dst = function(g)
         local res, err = ivshard.storage._call(
             'storage_ref_make_with_buckets', rid, iwait_timeout, {bid})
         ilt.assert_equals(err, nil)
-        ilt.assert_equals(res, {moved = {{
+        ilt.assert_equals(res, {total = 10, moved = {{
             id = bid,
             dst = id,
             status = ivconst.BUCKET.SENT,
@@ -406,6 +407,7 @@ test_group.test_ref_with_buckets_move_part_while_referencing = function(g)
         ilt.assert_equals(res, {
             moved = {{id = bids[2], dst = id}},
             is_done = true,
+            total = 8,
         })
         -- Ref was done, because at least one bucket was ok.
         ilt.assert_equals(lref.count, 1)
@@ -458,7 +460,8 @@ test_group.test_ref_with_buckets_move_all_while_referencing = function(g)
             moved = {
                 {id = bids[1], dst = id},
                 {id = bids[2], dst = id},
-            }
+            },
+            total = 8,
         })
         -- Ref was not done, because all the buckets moved out.
         ilt.assert_equals(lref.count, 0)
diff --git a/test/upgrade/upgrade.result b/test/upgrade/upgrade.result
index b1010604..fb61e825 100644
--- a/test/upgrade/upgrade.result
+++ b/test/upgrade/upgrade.result
@@ -181,6 +181,7 @@ vshard.storage._call('test_api', 1, 2, 3)
  |   - storage_bucket_checkpoint
  |   - storage_map
  |   - storage_ref
+ |   - storage_ref_check_existing
  |   - storage_ref_check_with_buckets
  |   - storage_ref_make_with_buckets
  |   - storage_unref
diff --git a/vshard/consts.lua b/vshard/consts.lua
index 80c49f5b..84126fbb 100644
--- a/vshard/consts.lua
+++ b/vshard/consts.lua
@@ -29,6 +29,11 @@ return {
         RED = 3,
     },
 
+    MAP_CALLRW_MODE = {
+        FULL = 'full',
+        PARTIAL = 'partial',
+    },
+
     REPLICATION_THRESHOLD_SOFT = 1,
     REPLICATION_THRESHOLD_HARD = 5,
     REPLICATION_THRESHOLD_FAIL = 10,
diff --git a/vshard/error.lua b/vshard/error.lua
index 5bbc3d5d..95552969 100644
--- a/vshard/error.lua
+++ b/vshard/error.lua
@@ -212,6 +212,11 @@ local error_message_template = {
         msg = "Master %s of replicaset %s has not synchronized yet",
         args = {'replica', 'replicaset'},
     },
+    [43] = {
+        name = 'UNSUPPORTED',
+        msg = 'Can\'t perform %s. Operation is not supported',
+        args = {'operation'},
+    },
 }
 
 --
diff --git a/vshard/router/init.lua b/vshard/router/init.lua
index 0010d9bd..c29f4056 100644
--- a/vshard/router/init.lua
+++ b/vshard/router/init.lua
@@ -29,6 +29,8 @@ local map_serializer = { __serialize = 'map' }
 local future_wait = util.future_wait
 
 local msgpack_is_object = lmsgpack.is_object
+local MAP_CALLRW_FULL = consts.MAP_CALLRW_MODE.FULL
+local MAP_CALLRW_PARTIAL = consts.MAP_CALLRW_MODE.PARTIAL
 
 if not util.feature.msgpack_object then
     local msg = 'Msgpack object feature is not supported by current '..
@@ -774,11 +776,11 @@ end
 --
 -- Prepares a cluster before sending refs to remote storages:
 --    1) Groups the buckets by replicasets according to the router's cache
---       in case of partial map_callrw;
+--       in case of partial map_callrw or full map_callrw with split args;
 --    2) Builds a table of replicasets on which refs should be sent;
 --    3) Waits necessary masters until all connections are established;
 --
-local function router_map_callrw_prepare(router, timeout, bucket_ids)
+local function router_map_callrw_prepare(router, timeout, mode, bucket_ids)
     local err, err_id
     local replicasets_to_wait, grouped_buckets = {}, {}
     local replicasets_all = router.replicasets
@@ -789,8 +791,10 @@ local function router_map_callrw_prepare(router, timeout, bucket_ids)
         if not grouped_buckets then
             return nil, err
         end
-        for rs_id, _ in pairs(grouped_buckets) do
-            table.insert(replicasets_to_wait, replicasets_all[rs_id])
+        if mode ~= MAP_CALLRW_FULL then
+            for rs_id, _ in pairs(grouped_buckets) do
+                table.insert(replicasets_to_wait, replicasets_all[rs_id])
+            end
         end
     else
         replicasets_to_wait = replicasets_all
@@ -807,10 +811,10 @@ local function router_map_callrw_prepare(router, timeout, bucket_ids)
 end
 
 --
--- Waits until all future objects are ready and extracts results from it.
+-- Waits until all the future objects are ready. After it returns successfully
+-- the result of each future can be read without blocking.
 --
 local function router_map_callrw_ref_wait(futures, timeout)
-    local results = {}
     local deadline = fiber_clock() + timeout
     for id, future in pairs(futures) do
         timeout = deadline - fiber_clock()
@@ -824,9 +828,8 @@ local function router_map_callrw_ref_wait(futures, timeout)
         if res == nil then
             return nil, err, id
         end
-        results[id] = res
     end
-    return timeout, results
+    return timeout
 end
 
 --
@@ -837,7 +840,8 @@ end
 local function router_map_callrw_process_moved(router, results)
     local bucket_ids = {}
     for _, res in pairs(results) do
-        for _, bucket in pairs(res.moved) do
+        local moved = res.moved or {}
+        for _, bucket in pairs(moved) do
             local bid = bucket.id
             local dst = bucket.dst
             -- 'Reset' regardless of 'set'. So as not to
@@ -856,38 +860,87 @@ end
 --
 -- Perform Ref stage of the Ref-Map-Reduce process on all the known replicasets.
 --
-local function router_ref_storage_all(router, timeout, rid)
-    local replicasets = router.replicasets
-    local err, err_id, results, grouped_buckets
-    local futures = {}
+local function router_ref_storage_all(router, bucket_ids, timeout, rid)
+    local mode = MAP_CALLRW_FULL
     local bucket_count = 0
+    local err, err_id, grouped_buckets, results, args_ref
+    local future_buckets, future_all, futures = {}, {}, {}
+    local replicasets = router.replicasets
     local opts_async = {is_async = true}
 
     timeout, grouped_buckets, err_id = router_map_callrw_prepare(
-        router, timeout)
+        router, timeout, mode, bucket_ids)
     if not timeout then
         err = grouped_buckets
         goto fail
     end
-    futures = {}
-    for rs_id, rs in pairs(replicasets) do
-        local buckets = replicasets[rs_id]
-        local res, ref_err = rs:callrw('vshard.storage._call',
-            {'storage_ref', rid, timeout}, opts_async)
+    for rs_id in pairs(replicasets) do
+        local buckets = grouped_buckets[rs_id]
+        local target
+        if buckets then
+            args_ref = {'storage_ref_make_with_buckets',
+                        rid, timeout, buckets, {force = true}}
+            target = future_buckets
+        else
+            args_ref = {'storage_ref', rid, timeout}
+            target = future_all
+        end
+        local res, ref_err = replicasets[rs_id]:callrw(
+            'vshard.storage._call', args_ref, opts_async)
         if res == nil then
             err, err_id = ref_err, rs_id
             goto fail
         end
+        target[rs_id] = res
         futures[rs_id] = res
     end
-    timeout, results, err_id = router_map_callrw_ref_wait(
+    timeout, err, err_id = router_map_callrw_ref_wait(
         futures, timeout)
     if not timeout then
-        err = results
         goto fail
     end
+    results = {}
+    for rs_id, future in pairs(future_buckets) do
+        results[rs_id] = future:result()[1]
+    end
+    for rs_id, future in pairs(future_all) do
+        results[rs_id] = {total = future:result()[1]}
+    end
     for _, res in pairs(results) do
-        bucket_count = bucket_count + res
+        if not res.total then
+            -- This error throws only in case when the updated router tries
+            -- to calculate the total amount of buckets . On old storage
+            -- versions the function storage_ref_make_with_buckets doesn't
+            -- return total amount of buckets.
+            err = lerror.vshard(lerror.code.UNSUPPORTED,
+                                'full map_callrw with split args')
+            goto fail
+        end
+        bucket_count = bucket_count + res.total
+    end
+    bucket_ids = router_map_callrw_process_moved(router, results)
+    if next(bucket_ids) then
+        futures = {}
+        args_ref = {'storage_ref_check_existing', rid, bucket_ids}
+        for rs_id in pairs(replicasets) do
+            local res, ref_err = replicasets[rs_id]:callrw(
+                'vshard.storage._call', args_ref, opts_async)
+            if res == nil then
+                err, err_id = ref_err, rs_id
+                goto fail
+            end
+            futures[rs_id] = res
+        end
+        timeout, err, err_id = router_map_callrw_ref_wait(futures, timeout)
+        if not timeout then
+            goto fail
+        end
+        for rs_id, future in pairs(futures) do
+            for _, bucket_id in pairs(future:result()[1]) do
+                bucket_reset(router, bucket_id)
+                bucket_set(router, bucket_id, rs_id)
+            end
+        end
     end
     -- All refs are done but not all buckets are covered. This is odd and can
     -- mean many things. The most possible ones: 1) outdated configuration on
@@ -914,6 +967,7 @@ end
 -- replicasets, which contains all the listed bucket IDs.
 --
 local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid)
+    local mode = MAP_CALLRW_PARTIAL
     local err, err_id, grouped_buckets, results, args_ref
     local replicasets_to_map, futures = {}, {}
     local replicasets_all = router.replicasets
@@ -922,7 +976,7 @@ local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid)
     -- netbox requests return box.NULL instead of nils.
     while next(bucket_ids) do
         timeout, grouped_buckets, err_id = router_map_callrw_prepare(
-            router, timeout, bucket_ids)
+            router, timeout, mode, bucket_ids)
         if not timeout then
             err = grouped_buckets
             goto fail
@@ -931,6 +985,8 @@ local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid)
         for rs_id in pairs(grouped_buckets) do
             local buckets = grouped_buckets[rs_id] or {}
             if replicasets_to_map[rs_id] then
+                -- Replicaset is already referenced on a previous iteration.
+                -- Simply get the moved buckets without double referencing.
                 args_ref = {'storage_ref_check_with_buckets', rid, buckets}
             else
                 args_ref = {'storage_ref_make_with_buckets',
@@ -944,12 +1000,15 @@ local function router_ref_storage_by_buckets(router, bucket_ids, timeout, rid)
             end
             futures[rs_id] = res
         end
-        timeout, results, err_id = router_map_callrw_ref_wait(
+        timeout, err, err_id = router_map_callrw_ref_wait(
             futures, timeout)
         if not timeout then
-            err = results
             goto fail
         end
+        results = {}
+        for rs_id, future in pairs(futures) do
+            results[rs_id] = future:result()[1]
+        end
         bucket_ids = router_map_callrw_process_moved(router, results)
         for id, res in pairs(results) do
             if res.is_done then
@@ -989,13 +1048,14 @@ local function replicasets_map_reduce(replicasets, rid, func, args,
     --
     local func_args = {'storage_map', rid, func, args}
     for id, rs in pairs(replicasets) do
-        if grouped_args ~= nil then
+        local rs_args = grouped_args and grouped_args[id]
+        if rs_args then
             -- It's cheaper to push and then pop, rather then deepcopy
             -- arguments table for every call.
-            table.insert(args, grouped_args[id])
+            table.insert(args, rs_args)
         end
         local res, err = rs:callrw('vshard.storage._call', func_args, opts_map)
-        if grouped_args ~= nil then
+        if rs_args then
             table.remove(args)
         end
         if res == nil then
@@ -1093,14 +1153,50 @@ local function router_group_map_callrw_args(router, bucket_ids, bucket_args)
     return grouped_args
 end
 
+--
+-- Set the appropriate mode according to bucket_ids option for backward
+-- compatibility (in case of opts_mode is nil) and check the given opts_mode
+-- correctness in other cases.
+--
+local function router_check_map_callrw_mode(opts_mode, bucket_ids)
+    if opts_mode == nil then
+        return bucket_ids and MAP_CALLRW_PARTIAL or MAP_CALLRW_FULL
+    end
+    if opts_mode == MAP_CALLRW_PARTIAL and bucket_ids == nil then
+        return nil, lerror.make('Router can\'t execute map_callrw with ' ..
+                                '\'partial\' mode and nil bucket_ids')
+    end
+    if opts_mode == MAP_CALLRW_FULL and util.table_is_numeric(bucket_ids) then
+        return nil, lerror.make('Router can\'t execute map_callrw with ' ..
+                                '\'full\' mode and numeric bucket_ids')
+    end
+    return opts_mode
+end
+
 --
 -- Consistent Map-Reduce. The given function is called on masters in the cluster
 -- with a guarantee that in case of success it was executed with all buckets
 -- being accessible for reads and writes.
 --
--- The selection of masters depends on bucket_ids option. When specified, the
--- Map-Reduce is performed only on masters having at least one of these buckets.
--- Otherwise it is executed on all the masters in the cluster.
+-- The selection of masters depends on 'mode' and 'bucket_ids' options. There
+-- are 2 general modes how map_callrw can be executed:
+--    1) mode = 'partial'. In this mode user function will be executed on
+--       storages that have at least one bucket of 'bucket_ids'. The
+--       'bucket_ids' option can be presented in two ways: like a numeric array
+--       of buckets' ids or like a map of buckets' arguments. In first one user
+--       function will only receive args, in second one it will additionally
+--       receive buckets' arguments.
+--    2) mode = 'full'. In this mode user function will be executed with args on
+--       all storages in cluster. If we pass 'bucket_ids' like a map of bucket's
+--       arguments the user function will additionally receive buckets'
+--       arguments on those storages that have at least one bucket of
+--       'bucket_ids'.
+--
+-- If we didn't specify the 'mode' option, then it is set based on 'bucket_ids'
+-- option - if 'bucket_ids' is presented, the mode will be 'partial' otherwise
+-- 'full'. Also the next combination of map_callrw options can lead to error:
+-- <mode = 'partial', bucket_ids = nil> and <mode = 'full', bucket_ids =
+-- numeric_array>.
 --
 -- Consistency in scope of map-reduce means all the data was accessible, and
 -- didn't move during map requests execution. To preserve the consistency there
@@ -1123,6 +1219,8 @@ end
 -- @param func Name of the function to call.
 -- @param args Function arguments passed in netbox style (as an array).
 -- @param opts Options. See below:
+--     - mode - a string option ('full' / 'partial') that represents a way of
+--         execution of user function on destination storages.
 --     - timeout - a number of seconds. Note that the refs may end up being kept
 --         on the storages during this entire timeout if something goes wrong.
 --         For instance, network issues appear. This means better not use a
@@ -1146,8 +1244,13 @@ end
 --
 local function router_map_callrw(router, func, args, opts)
     local replicasets_to_map, err, err_id, map, rid
-    local timeout, do_return_raw, bucket_ids, plain_bucket_ids, grouped_args
+    local mode, timeout, do_return_raw, bucket_ids, plain_bucket_ids,
+        grouped_args
     if opts then
+        mode, err = router_check_map_callrw_mode(opts.mode, opts.bucket_ids)
+        if err then
+            return nil, err
+        end
         timeout = opts.timeout or consts.CALL_TIMEOUT_MIN
         do_return_raw = opts.return_raw
         bucket_ids = opts.bucket_ids
@@ -1155,27 +1258,26 @@ local function router_map_callrw(router, func, args, opts)
             util.table_keys(bucket_ids)
     else
         timeout = consts.CALL_TIMEOUT_MIN
+        mode = MAP_CALLRW_FULL
     end
     rid = M.ref_id
     M.ref_id = rid + 1
-    if plain_bucket_ids then
+    if mode == MAP_CALLRW_FULL then
+        timeout, err, err_id, replicasets_to_map =
+            router_ref_storage_all(router, plain_bucket_ids, timeout, rid)
+    else
         timeout, err, err_id, replicasets_to_map =
             router_ref_storage_by_buckets(router, plain_bucket_ids, timeout,
                                           rid)
-        -- Grouped arguments are only possible with partial Map-Reduce.
-        if timeout then
+    end
+    if timeout then
+        if plain_bucket_ids then
             grouped_args = router_group_map_callrw_args(
                 router, plain_bucket_ids, bucket_ids)
         end
-    else
-        timeout, err, err_id, replicasets_to_map =
-            router_ref_storage_all(router, timeout, rid)
-    end
-    if timeout then
-        map, err, err_id = replicasets_map_reduce(replicasets_to_map, rid, func,
-            args, grouped_args, {
-                timeout = timeout, return_raw = do_return_raw
-            })
+        opts = {timeout = timeout, return_raw = do_return_raw}
+        map, err, err_id = replicasets_map_reduce(
+            replicasets_to_map, rid, func, args, grouped_args, opts)
         if map then
             return map
         end
diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua
index e67f11d1..31ad2761 100644
--- a/vshard/storage/init.lua
+++ b/vshard/storage/init.lua
@@ -3493,6 +3493,27 @@ local function storage_call(bucket_id, mode, name, args)
     return ok, ret1, ret2, ret3
 end
 
+local function storage_ref_check_existing(rid, bucket_ids)
+    local ok, err = lref.check(rid, box.session.id())
+    if not ok then
+        return nil, err
+    end
+    bucket_ids = bucket_ids or {}
+    local res = {}
+    local limit = consts.BUCKET_CHUNK_SIZE + 1
+    for i, bucket_id in pairs(bucket_ids) do
+        local bucket = M.bucket_refs[bucket_id] or
+                       box.space._bucket:get{bucket_id}
+        if bucket then
+            table.insert(res, bucket_id)
+        end
+        if i % limit == 0 then
+            fiber_yield()
+        end
+    end
+    return res
+end
+
 --
 -- Bind a new storage ref to the current box session. Is used as a part of
 -- Map-Reduce API.
@@ -3552,28 +3573,29 @@ end
 --         are absent, the reference is not created and a nil reference id
 --         with the list of absent buckets is returned.
 --
-local function storage_ref_make_with_buckets(rid, timeout, bucket_ids)
+local function storage_ref_make_with_buckets(rid, timeout, bucket_ids, opts)
+    local force = opts and opts.force
     local moved = bucket_get_moved(bucket_ids)
-    if #moved == #bucket_ids then
+    if not force and #moved == #bucket_ids then
         -- If all the passed buckets are absent, there is no need to create a
         -- ref.
-        return {moved = moved}
+        return {moved = moved, total = bucket_count()}
     end
     local bucket_generation = M.bucket_generation
-    local ok, err = storage_ref(rid, timeout)
-    if not ok then
+    local bucket_count, err = storage_ref(rid, timeout)
+    if not bucket_count then
         return nil, err
     end
     if M.bucket_generation ~= bucket_generation then
         -- Need to redo it. Otherwise there is a risk that some buckets were
         -- moved while waiting for the ref.
         moved = bucket_get_moved(bucket_ids)
-        if #moved == #bucket_ids then
+        if not force and #moved == #bucket_ids then
             storage_unref(rid)
-            return {moved = moved}
+            return {moved = moved, total = bucket_count}
         end
     end
-    return {is_done = true, moved = moved}
+    return {is_done = true, moved = moved, total = bucket_count}
 end
 
 --
@@ -3695,6 +3717,7 @@ service_call_api = setmetatable({
     rebalancer_request_state = rebalancer_request_state,
     recovery_bucket_stat = recovery_bucket_stat,
     storage_ref = storage_ref,
+    storage_ref_check_existing = storage_ref_check_existing,
     storage_ref_make_with_buckets = storage_ref_make_with_buckets,
     storage_ref_check_with_buckets = storage_ref_check_with_buckets,
     storage_unref = storage_unref,