diff --git a/DESCRIPTION b/DESCRIPTION index a5819855..af5c6967 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -53,6 +53,7 @@ Suggests: mgcv, mirai (>= 2.4.0), modeldata, + mori, probably, scales, spelling, diff --git a/R/loop_over_all_stages.R b/R/loop_over_all_stages.R index 4cf35eb4..44b6acbc 100644 --- a/R/loop_over_all_stages.R +++ b/R/loop_over_all_stages.R @@ -26,6 +26,9 @@ } split <- resamples$splits[[1]] + if (!is.null(static$mori_shared_name)) { + split$data <- mori::map_shared(static$mori_shared_name) + } split_labs <- resamples |> dplyr::select(dplyr::starts_with("id")) # Determine if we're using prediction-based or model-based metrics diff --git a/R/tune_grid_loop.R b/R/tune_grid_loop.R index 1e146bd1..adf7c6dc 100644 --- a/R/tune_grid_loop.R +++ b/R/tune_grid_loop.R @@ -108,6 +108,20 @@ tune_grid_loop <- function( inds <- vec_list_rowwise(inds) } + # Share the full training data via mori to avoid serializing it to every + # worker. Instead, workers receive lightweight split objects (indices + schema + # only) and map the shared memory region directly. + mori_ref <- NULL + if (strategy != "sequential" && rlang::is_installed("mori")) { + full_data <- resamples[[1]]$splits[[1]]$data + mori_ref <- mori::share(full_data) + static$mori_shared_name <- mori::shared_name(mori_ref) + resamples <- purrr::map(resamples, function(res_row) { + res_row$splits[[1]]$data <- full_data[0L, ] + res_row + }) + } + cl <- loop_call(control$parallel_over, strategy, par_opt) res <- rlang::eval_bare(cl)