diff --git a/R/logging.R b/R/logging.R index 2d2e8f20..190653c2 100644 --- a/R/logging.R +++ b/R/logging.R @@ -105,12 +105,12 @@ tune_env <- rlang::new_environment( progress_env = NULL, progress_active = FALSE, progress_catalog = NULL, - progress_started = FALSE + progress_status_ids = list(), + progress_modulus = 1L, + progress_in_unit = 0L ) ) -lbls <- c(LETTERS, letters, 1:1e3) - # determines whether a currently running tuning process uses the cataloger. uses_catalog <- function() { isTRUE(tune_env$progress_active && !is_testing()) @@ -185,40 +185,174 @@ initialize_catalog <- function( rlang::env_bind(tune_env, progress_active = FALSE), envir = env ) + + rlang::env_bind(tune_env, progress_status_ids = list()) withr::defer( - rlang::env_bind(tune_env, progress_started = FALSE), + catalog_cleanup(), envir = env ) invisible(NULL) } -# given a catalog, summarize errors and warnings in a 1-length glue vector. -# for use by the progress bar inside of `tune_catalog()`. -summarize_catalog <- function(catalog, sep = " ") { - if (nrow(catalog) == 0) { - return("") +catalog_cleanup <- function() { + ids <- tune_env$progress_status_ids + catalog <- tune_env$progress_catalog + + if (length(ids) > 0 && !is.null(catalog) && nrow(catalog) > 0) { + if (cli::is_dynamic_tty()) { + # Update headers with final counts, then keep as permanent output + for (i in seq_len(nrow(catalog))) { + entry_id <- catalog$id[i] + entry <- ids[[entry_id]] + if (!is.null(entry)) { + header <- catalog_header(catalog$type[i], catalog$n[i]) + cli::cli_status_update(id = entry$header, msg = header) + cli::cli_status_clear(entry$header, result = "clear") + for (cont_id in entry$continuation) { + cli::cli_status_clear(cont_id, result = "clear") + } + } + } + } else { + # Non-dynamic: print compact summary, then clear silently + has_duplicates <- any(catalog$n > 1) + if (has_duplicates) { + parts <- vapply( + seq_len(nrow(catalog)), + function(i) { + color <- if (catalog$type[i] == "warning") { + cli::col_yellow + } else { + cli::col_red + } + color(paste0(catalog$type[i], " (x", catalog$n[i], ")")) + }, + character(1) + ) + cli::cli_inform(paste0( + "Issue totals: ", + paste(parts, collapse = ", ") + )) + } + suppressMessages( + for (entry in ids) { + for (cont_id in rev(entry$continuation)) { + try(cli::cli_status_clear(cont_id, result = "clear"), silent = TRUE) + } + try( + cli::cli_status_clear(entry$header, result = "clear"), + silent = TRUE + ) + } + ) + } + } + + rlang::env_bind(tune_env, progress_status_ids = list()) +} + +# The catalog heartbeat: a single progress bar that sits at the top of the +# progress area and advances once per model fit, so that a tuning run shows it's +# making progress even when no issues are caught. Gated on `uses_catalog()`, so +# it only appears in sequential, non-verbose, non-testing runs. +# +# `total` is the number of model fits across the whole run and `modulus` is the +# number per resample (the per-resample tick budget). We force an initial render +# at creation so the bar claims the top status-bar slot before any issue bar is +# created during the first resample; otherwise issues caught mid-resample would +# render above it. `progress_in_unit` counts ticks within the current resample so +# `catalog_progress_trueup()` can pad the bar up to the next multiple of +# `modulus` when upstream failures cause a resample to fit fewer models than the +# schedule allows for. +catalog_progress_init <- function(total, modulus = 1L) { + if (!uses_catalog()) { + return(invisible(NULL)) } - res <- dplyr::arrange(catalog, id) - res <- dplyr::mutate( - res, - color = dplyr::if_else( - type == "warning", - list(cli::col_yellow), - list(cli::col_red) - ) + rlang::env_bind(tune_env, progress_modulus = modulus, progress_in_unit = 0L) + + rlang::with_options( + cli::cli_progress_bar( + total = total, + format = paste( + "{cli::pb_spin} tuning {cli::pb_bar}", + "{cli::pb_current}/{cli::pb_total} {cli::pb_eta_str}" + ), + format_done = "{cli::col_green(cli::symbol$tick)} tuning complete [{cli::pb_elapsed}]", + clear = FALSE, + .envir = tune_env$progress_env + ), + cli.progress_show_after = 0 ) - res <- dplyr::rowwise(res) - res <- dplyr::mutate( - res, - msg = glue::glue("{color(cli::style_bold(lbls[id]))}: x{n}") + + cli::cli_progress_update( + set = 0, + force = TRUE, + .envir = tune_env$progress_env ) - res <- dplyr::ungroup(res) - res <- dplyr::pull(res, msg) - res <- glue::glue_collapse(res, sep = sep) - res + invisible(NULL) +} + +catalog_progress_tick <- function() { + if (!uses_catalog()) { + return(invisible(NULL)) + } + + # `force` so the displayed count keeps pace with the issue bars, which repaint + # on every occurrence; without it the throttled heartbeat lags behind and can + # show fewer fits than there are issues (#tune-heartbeat). + cli::cli_progress_update(force = TRUE, .envir = tune_env$progress_env) + tune_env$progress_in_unit <- tune_env$progress_in_unit + 1L + + invisible(NULL) +} + +catalog_progress_trueup <- function() { + if (!uses_catalog()) { + return(invisible(NULL)) + } + + pad <- tune_env$progress_modulus - tune_env$progress_in_unit + if (pad > 0) { + cli::cli_progress_update( + inc = pad, + force = TRUE, + .envir = tune_env$progress_env + ) + } + tune_env$progress_in_unit <- 0L + + invisible(NULL) +} + +# The number of model fits the grid loop performs per resample, used as the +# heartbeat's per-resample tick budget. The schedule depends only on the grid +# and workflow (not the split), so it's the same for every resample. Submodel +# parameters share a single fit, so a grid that only tunes submodel parameters +# counts as one fit per resample. +catalog_count_model_iters <- function(grid, workflow) { + if (is.null(grid) || nrow(grid) == 0) { + return(1L) + } + + sched <- schedule_grid(grid, workflow) + as.integer(sum(vapply( + sched$model_stage, + function(m) max(nrow(m), 1L), + integer(1) + ))) +} + +catalog_progress_done <- function() { + if (!uses_catalog()) { + return(invisible(NULL)) + } + + cli::cli_progress_done(.envir = tune_env$progress_env) + + invisible(NULL) } # catching and logging --------------------------------------------------------- @@ -237,8 +371,10 @@ summarize_catalog <- function(catalog, sep = " ") { tmp <- catcher(.expr) if (has_log_notes(tmp)) { + # log only the notes from this catch; `catalog_log()` increments counts, so + # passing the resample's accumulated `notes` would re-count earlier entries + catalog_log(append_log_notes(NULL, tmp, dots$location)) notes <- append_log_notes(notes, tmp, dots$location) - catalog_log(notes) } tmp <- remove_log_notes(tmp) assign("notes", notes, envir = parent.frame()) @@ -356,6 +492,15 @@ catalog_log <- function(x) { if (x_note %in% catalog$note) { idx <- match(x_note, catalog$note) catalog$n[idx] <- catalog$n[idx] + 1 + + if (uses_catalog() && cli::is_dynamic_tty()) { + entry_id <- catalog$id[idx] + ids <- tune_env$progress_status_ids[[entry_id]] + if (!is.null(ids)) { + header <- catalog_header(x_type, catalog$n[idx]) + cli::cli_status_update(id = ids$header, msg = header) + } + } } else { new_id <- nrow(catalog) + 1 catalog <- tibble::add_row( @@ -368,47 +513,66 @@ catalog_log <- function(x) { ) ) - # construct issue summary - color <- if (x_type == "warning") cli::col_yellow else cli::col_red - # pad by nchar(label) + nchar("warning") + additional spaces and symbols - pad <- nchar(new_id) + 14L - justify <- paste0("\n", strrep("\u00a0", pad)) - note <- gsub("\n", justify, x_note) - # pad `nchar("warning") - nchar("error")` spaces to the right of the `:` - if (x_type == "error") { - note <- paste0("\u00a0\u00a0", note) + if (uses_catalog()) { + header <- catalog_header(x_type, 1L) + body_lines <- catalog_body_lines(x_note) + + header_id <- cli::cli_status( + header, + msg_done = header, + .keep = TRUE, + .auto_close = FALSE + ) + cont_ids <- vapply( + body_lines, + function(line) { + cli::cli_status( + line, + msg_done = line, + .keep = TRUE, + .auto_close = FALSE + ) + }, + character(1) + ) + + tune_env$progress_status_ids[[new_id]] <- list( + header = header_id, + continuation = cont_ids + ) + } else { + color <- if (x_type == "warning") cli::col_yellow else cli::col_red + symbol <- if (x_type == "warning") "!" else cli::symbol$cross + header_text <- paste0(color(symbol), " ", color(x_type), " (x1):") + body_lines <- catalog_body_lines(x_note) + cli::cli_alert(paste0( + header_text, + "\n", + paste0(body_lines, collapse = "\n") + )) } - msg <- glue::glue( - "{color(cli::style_bold(lbls[new_id]))} | {color(x_type)}: {note}" - ) - cli::cli_alert(msg) } } rlang::env_bind(tune_env, progress_catalog = catalog) - if (uses_catalog()) { - rlang::env_bind( - tune_env$progress_env, - catalog_summary = summarize_catalog(catalog) - ) - if (!tune_env$progress_started) { - rlang::with_options( - cli::cli_progress_bar( - type = "custom", - format = "There were issues with some computations {catalog_summary}", - clear = FALSE, - .envir = tune_env$progress_env - ), - cli.progress_show_after = 0 - ) - rlang::env_bind(tune_env, progress_started = TRUE) - } + return(NULL) +} - cli::cli_progress_update(.envir = tune_env$progress_env) - } +catalog_header <- function(type, n) { + color <- if (type == "warning") cli::col_yellow else cli::col_red + symbol <- if (type == "warning") "!" else cli::symbol$cross + paste0(color(symbol), " ", color(paste0(type, " (x", n, "):"))) +} - return(NULL) +catalog_body_lines <- function(note) { + indent <- 2L + width <- cli::console_width() - indent + lines <- strsplit(note, "\n")[[1]] + wrapped <- unlist(lapply(lines, function(line) { + strwrap(line, width = width) + })) + paste0(strrep(" ", indent), wrapped) } remove_log_notes <- function(x) { diff --git a/R/loop_over_all_stages.R b/R/loop_over_all_stages.R index 4cf35eb4..40af94e8 100644 --- a/R/loop_over_all_stages.R +++ b/R/loop_over_all_stages.R @@ -127,6 +127,8 @@ grid_with_pre <- current_grid for (iter_model in seq_len(num_iterations_model)) { + catalog_progress_tick() + current_sched_model <- current_sched_pre$model_stage[[1]][iter_model, ] current_grid <- extend_grid(grid_with_pre, current_sched_model) @@ -525,6 +527,8 @@ } } + catalog_progress_trueup() + return_tbl } diff --git a/R/tune_grid_loop.R b/R/tune_grid_loop.R index 1e146bd1..a90f7336 100644 --- a/R/tune_grid_loop.R +++ b/R/tune_grid_loop.R @@ -108,9 +108,26 @@ tune_grid_loop <- function( inds <- vec_list_rowwise(inds) } + if (control$parallel_over == "everything") { + # each unit is already a single resample x candidate, so one fit per unit + catalog_progress_init(length(inds), modulus = 1L) + } else { + fits_per_resample <- if (uses_catalog()) { + catalog_count_model_iters(grid, workflow) + } else { + 1L + } + catalog_progress_init( + length(resamples) * fits_per_resample, + modulus = fits_per_resample + ) + } + cl <- loop_call(control$parallel_over, strategy, par_opt) res <- rlang::eval_bare(cl) + catalog_progress_done() + # ---------------------------------------------------------------------------- # Separate results into different components diff --git a/tests/testthat/_snaps/logging.md b/tests/testthat/_snaps/logging.md index c4a781dc..0a18f8f7 100644 --- a/tests/testthat/_snaps/logging.md +++ b/tests/testthat/_snaps/logging.md @@ -159,15 +159,9 @@ raise_error() })) Message - > A | warning: ope! yikes. - > B | error: AHHhH - ---- - - Code - catalog_summary_test - Output - A: x5 B: x5 + ! warning (x1): + x error (x1): + Issue totals: warning (x5), error (x5) # interactive logger works (fit_resamples, rlang warning + error) @@ -179,15 +173,9 @@ raise_error_rl() })) Message - > A | warning: ope! yikes. (but rlang) - > B | error: AHHhH (but rlang) - ---- - - Code - catalog_summary_test - Output - A: x5 B: x5 + ! warning (x1): + x error (x1): + Issue totals: warning (x5), error (x5) # interactive logger works (fit_resamples, multiline) @@ -196,15 +184,9 @@ Sale_Price ~ ., rsample::vfold_cv(modeldata::ames[, c(72, 40:45)], 5), control = control_resamples( extract = raise_multiline_conditions)) Message - > A | warning: hmmm what's happening - > B | error: aHHHksdjvndiuf - ---- - - Code - catalog_summary_test - Output - A: x5 B: x5 + ! warning (x1): + x error (x1): + Issue totals: warning (x5), error (x5) # interactive logger works (fit_resamples, occasional error) @@ -213,14 +195,8 @@ Sale_Price ~ ., rsample::vfold_cv(modeldata::ames[, c(72, 40:45)], 5), control = control_resamples( extract = later)) Message - > A | error: this errors now! ha! - ---- - - Code - catalog_summary_test - Output - A: x2 + x error (x1): + Issue totals: error (x2) # interactive logger works (fit_resamples, occasional errors) @@ -232,15 +208,9 @@ later() })) Message - > A | error: oh no - > B | error: this errors now! ha! - ---- - - Code - catalog_summary_test - Output - A: x1 B: x6 + x error (x1): + x error (x1): + Issue totals: error (x1), error (x6) # interactive logger works (fit_resamples, many distinct errors) @@ -249,18 +219,11 @@ Sale_Price ~ ., rsample::vfold_cv(modeldata::ames[, c(72, 40:45)], 5), control = control_resamples( extract = numbered)) Message - > A | error: error number 1 - > B | error: error number 2 - > C | error: error number 3 - > D | error: error number 4 - > E | error: error number 5 - ---- - - Code - catalog_summary_test - Output - A: x1 B: x1 C: x1 D: x1 E: x1 + x error (x1): + x error (x1): + x error (x1): + x error (x1): + x error (x1): # interactive logger works (tune grid, error) @@ -269,14 +232,8 @@ dist_power = tune()), Sale_Price ~ ., rsample::vfold_cv(modeldata::ames[, c( 72, 40:45)], 5), grid = 5, control = control_grid(extract = raise_error)) Message - > A | error: AHHhH - ---- - - Code - catalog_summary_test - Output - A: x75 + x error (x1): + Issue totals: error (x25) # interactive logger works (bayesian, error) @@ -285,12 +242,6 @@ dist_power = tune()), Sale_Price ~ ., rsample::vfold_cv(modeldata::ames[, c( 72, 40:45)], 5), initial = 5, iter = 5, control = control_bayes(extract = raise_error)) Message - > A | error: AHHhH - ---- - - Code - catalog_summary_test - Output - A: x100 + x error (x1): + Issue totals: error (x50) diff --git a/tests/testthat/_snaps/loop-over-all-stages-logging.md b/tests/testthat/_snaps/loop-over-all-stages-logging.md index 3e2fda61..dbb26167 100644 --- a/tests/testthat/_snaps/loop-over-all-stages-logging.md +++ b/tests/testthat/_snaps/loop-over-all-stages-logging.md @@ -5,7 +5,6 @@ dist_power = tune()), Sale_Price ~ ., folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: invalid type (list) for variable 'First_Flr_SF' Condition # model error doesn't stop grid @@ -14,7 +13,6 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: invalid type (list) for variable 'First_Flr_SF' Condition # prediction error doesn't stop grid @@ -23,7 +21,9 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: Assigned data `.ind` must be compatible with existing data. + x Existing data has 0 rows. + x Assigned data has 1465 rows. + ! Can't recycle input of size 1465 to size 0. Condition # capturing error correctly in notes @@ -32,7 +32,7 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: Error in `step_logging_helper()`: + ! testing error Condition # capturing warning correctly in notes @@ -41,7 +41,6 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | warning: testing warning # doesn't capturing message in notes @@ -56,19 +55,9 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE, extract = extract_error)) Message - > A | error: extract error # captures kknn R errors - Code - res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( - allow_par = FALSE)) - Message - > A | error: NA/NaN/Inf in foreign function call (arg 1) - Condition - -# captures xgboost C errors - Code res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) @@ -81,7 +70,7 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: Error in `step_logging_helper()`: + ! testing error Condition # emitter works with errors @@ -90,6 +79,6 @@ res_fit <- tune_grid(wf_spec, folds, grid = 2, control = control_grid( allow_par = FALSE)) Message - > A | error: Error in `step_logging_helper()`: + ! testing error Condition diff --git a/tests/testthat/_snaps/resample.new.md b/tests/testthat/_snaps/resample.new.md new file mode 100644 index 00000000..e1248437 --- /dev/null +++ b/tests/testthat/_snaps/resample.new.md @@ -0,0 +1,104 @@ +# failure in variables tidyselect specification is caught elegantly + + Code + result <- fit_resamples(workflow, folds, control = control) + Message + > x error (x1): + Can't select columns that don't exist. + x Column `foobar` doesn't exist. + Condition + Warning: + All models failed. Run `show_notes(.Last.tune.result)` for more information. + +--- + + Code + note + Output + [1] "Can't select columns that don't exist.\nx Column `foobar` doesn't exist." + +# classification models generate correct error message + + Code + result <- fit_resamples(log_mod, rec, folds, control = control) + Message + > x error (x1): + For a classification model, the outcome should be a , not a double + vector. + Condition + Warning: + All models failed. Run `show_notes(.Last.tune.result)` for more information. + +--- + + Code + note + Output + [1] "For a classification model, the outcome should be a , not a double vector." + +# `tune_grid()` falls back to `fit_resamples()` - formula + + Code + result <- tune_grid(lin_mod, mpg ~ ., folds) + Condition + Warning: + No tuning parameters have been detected, performance will be evaluated using the resamples with no tuning. + Did you want to assign any parameters with a value of `tune()`? + +# `tune_grid()` falls back to `fit_resamples()` - workflow variables + + Code + result <- tune_grid(wf, folds) + Condition + Warning: + No tuning parameters have been detected, performance will be evaluated using the resamples with no tuning. + Did you want to assign any parameters with a value of `tune()`? + +# `tune_grid()` ignores `grid` if there are no tuning parameters + + Code + result <- tune_grid(lin_mod, mpg ~ ., grid = data.frame(x = 1), folds) + Condition + Warning: + No tuning parameters have been detected, performance will be evaluated using the resamples with no tuning. + Did you want to assign any parameters with a value of `tune()`? + +# cannot autoplot `fit_resamples()` results + + Code + autoplot(result) + Condition + Error in `autoplot()`: + ! There is no `autoplot()` implementation for . + +# ellipses with fit_resamples + + Code + fit_resamples(lin_mod, mpg ~ ., folds, something = "wrong") + Condition + Warning: + The `...` are not used in this function but 1 object was passed: "something" + Output + # Resampling results + # 2-fold cross-validation + # A tibble: 2 x 4 + splits id .metrics .notes + + 1 Fold1 + 2 Fold2 + +# `fit_resamples()` when objects need tuning + + 2 arguments have been tagged for tuning in these components: model_spec and recipe. + i Please use one of the tuning functions (e.g. `tune_grid()`) to optimize them. + +--- + + 1 argument has been tagged for tuning in this component: model_spec. + i Please use one of the tuning functions (e.g. `tune_grid()`) to optimize them. + +--- + + 1 argument has been tagged for tuning in this component: recipe. + i Please use one of the tuning functions (e.g. `tune_grid()`) to optimize them. + diff --git a/tests/testthat/helper-logging.R b/tests/testthat/helper-logging.R index 3dff3888..4d05c17b 100644 --- a/tests/testthat/helper-logging.R +++ b/tests/testthat/helper-logging.R @@ -60,5 +60,5 @@ bake.step_logging_helper <- function(object, new_data, ...) { } catalog_lines <- function(lines) { - lines[grepl("^>", lines)] + lines[grepl("^(!|x|✖|✘)", lines) | grepl("^Issue totals:", lines)] } diff --git a/tests/testthat/helper-tune-package.R b/tests/testthat/helper-tune-package.R index 4c4710ad..f9b655e6 100644 --- a/tests/testthat/helper-tune-package.R +++ b/tests/testthat/helper-tune-package.R @@ -43,32 +43,17 @@ helper_objects_tune <- function() { # # ...we just want to see the unique issues, e.g. # -# > A | error: AHHhH catalog_lines <- function(lines) { - lines[grepl("^>", lines)] + lines[grepl("^(!|x|✖|✘)", lines) | grepl("^Issue totals:", lines)] } # Make a new binding to prevent infinite recursion when the original is mocked. initialize_catalog_ <- tune:::initialize_catalog -# Sets a new exit handler on `initialize_catalog()` that stores the summary -# of issues before it's cleared along with the progress bar. Together with -# the above, we can test the full catalog output. redefer_initialize_catalog <- function(test_env) { local({ function(control, env = rlang::caller_env(), workflow = NULL) { initialize_catalog_(control, env, workflow) - - withr::defer( - assign( - "catalog_summary_test", - tune:::tune_env$progress_env$catalog_summary, - test_env - ), - envir = env, - priority = "first" - ) - NULL } }) diff --git a/tests/testthat/test-logging.R b/tests/testthat/test-logging.R index ea599f19..f4b5cc3d 100644 --- a/tests/testthat/test-logging.R +++ b/tests/testthat/test-logging.R @@ -190,9 +190,6 @@ test_that("interactive logger works (fit_resamples, warning + error)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) test_that("interactive logger works (fit_resamples, rlang warning + error)", { @@ -236,9 +233,6 @@ test_that("interactive logger works (fit_resamples, rlang warning + error)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) @@ -280,9 +274,6 @@ test_that("interactive logger works (fit_resamples, multiline)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) test_that("interactive logger works (fit_resamples, occasional error)", { @@ -329,9 +320,6 @@ test_that("interactive logger works (fit_resamples, occasional error)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) test_that("interactive logger works (fit_resamples, occasional errors)", { @@ -400,9 +388,6 @@ test_that("interactive logger works (fit_resamples, occasional errors)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) @@ -451,9 +436,6 @@ test_that("interactive logger works (fit_resamples, many distinct errors)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) test_that("interactive logger works (tune grid, error)", { @@ -491,9 +473,6 @@ test_that("interactive logger works (tune grid, error)", { }, transform = catalog_lines ) - - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) }) test_that("interactive logger works (bayesian, error)", { @@ -533,7 +512,56 @@ test_that("interactive logger works (bayesian, error)", { }, transform = catalog_lines ) +}) + +test_that("catalog heartbeat helpers no-op when the catalog is inactive", { + expect_null(catalog_progress_init(10)) + expect_null(catalog_progress_tick()) + expect_null(catalog_progress_done()) +}) + +test_that("catalog heartbeat creates, advances, and tears down a bar", { + local_mocked_bindings(is_testing = function() FALSE) + + env <- rlang::current_env() + rlang::env_bind(tune_env, progress_env = env, progress_active = TRUE) + withr::defer( + rlang::env_bind(tune_env, progress_env = NULL, progress_active = FALSE) + ) + + n0 <- cli::cli_progress_num() + + catalog_progress_init(6, modulus = 3L) + expect_equal(cli::cli_progress_num(), n0 + 1) + + catalog_progress_tick() + catalog_progress_tick() + expect_equal(tune_env$progress_in_unit, 2L) + + # a resample that fit fewer models than the schedule allows still advances the + # bar by a full modulus, and resets the per-resample counter + catalog_progress_trueup() + expect_equal(tune_env$progress_in_unit, 0L) + + catalog_progress_done() + expect_equal(cli::cli_progress_num(), n0) +}) + +test_that("catalog_count_model_iters counts model fits per resample", { + skip_if_not_installed("kknn") + + expect_equal(catalog_count_model_iters(NULL, workflow()), 1L) + expect_equal(catalog_count_model_iters(tibble::tibble(), workflow()), 1L) + + wflow <- workflow() |> + add_formula(mpg ~ .) |> + add_model(parsnip::nearest_neighbor( + "regression", + "kknn", + dist_power = tune() + )) + grid <- tibble::tibble(dist_power = c(1, 1.5, 2)) - # `catalog_summary_test` written to this env via `redefer_initialize_catalog()` - expect_snapshot(catalog_summary_test) + # `dist_power` is not a submodel parameter, so each candidate is its own fit + expect_equal(catalog_count_model_iters(grid, wflow), 3L) })