From 401873fc7b470819aa5b24ec147f65924bf2c794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98topepo=E2=80=99?= Date: Thu, 23 Apr 2026 09:28:24 -0400 Subject: [PATCH 1/5] changes for #1186 --- .Rbuildignore | 1 + .gitignore | 1 + R/metric-selection.R | 23 +++++++++++++------- tests/testthat/_snaps/metric-args.md | 32 ++++++++++++++++++++++++++++ tests/testthat/test-metric-args.R | 17 +++++++++++++++ 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 541b3e261..a67e324a1 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -21,3 +21,4 @@ _cache$ ^\.vscode$ ^[.]?air[.]toml$ ^\.claude$ +^\.positai$ diff --git a/.gitignore b/.gitignore index 8f2f413a3..96f9b0b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tests/testthat/Rplots.pdf docs revdep tests/testthat/_snaps/notes.new.md +.positai diff --git a/R/metric-selection.R b/R/metric-selection.R index 8692a2da2..02d466745 100644 --- a/R/metric-selection.R +++ b/R/metric-selection.R @@ -328,23 +328,30 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { ) return(mtr_set) + } else { + if (!inherits(mtr_set, "metric_set")) { + cli::cli_abort( + "The {.arg metrics} argument should have class {.cls metric_set}, not {.cls {class(mtr_set)}}.", + call = call + ) + } } is_numeric_metric_set <- inherits(mtr_set, "numeric_metric_set") is_class_prob_metric_set <- inherits(mtr_set, "class_prob_metric_set") is_surv_metric_set <- inherits(mtr_set, c("survival_metric_set")) + is_qnt_metric_set <- inherits(mtr_set, c("quantile_metric_set")) - if ( - !is_numeric_metric_set && !is_class_prob_metric_set && !is_surv_metric_set - ) { + if (mode == "regression" && !is_numeric_metric_set) { cli::cli_abort( - "The {.arg metrics} argument should be the results of - {.fn yardstick::metric_set}.", + "The parsnip model has {.code mode} value of {.val {mode}}, + but the {.arg metrics} is a metric set for a + different model mode.", call = call ) } - if (mode == "regression" && !is_numeric_metric_set) { + if (mode == "classification" && !is_class_prob_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, but the {.arg metrics} is a metric set for a @@ -353,7 +360,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { ) } - if (mode == "classification" && !is_class_prob_metric_set) { + if (mode == "censored regression" && !is_surv_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, but the {.arg metrics} is a metric set for a @@ -362,7 +369,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { ) } - if (mode == "censored regression" && !is_surv_metric_set) { + if (mode == "quantile regression" && !is_qnt_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, but the {.arg metrics} is a metric set for a diff --git a/tests/testthat/_snaps/metric-args.md b/tests/testthat/_snaps/metric-args.md index 64b37ff64..0961747b3 100644 --- a/tests/testthat/_snaps/metric-args.md +++ b/tests/testthat/_snaps/metric-args.md @@ -7,6 +7,14 @@ - `rmse()`, a numeric metric | direction: minimize - `rsq()`, a numeric metric | direction: maximize +--- + + Code + check_metrics_arg(rmse, wflow) + Condition + Error: + ! The `metrics` argument should have class , not . + --- Code @@ -289,3 +297,27 @@ Error in `last_fit()`: ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. +--- + + Code + check_metrics_arg(NULL, wflow) + Output + A metric set, consisting of: + - `weighted_interval_score()`, a quantile metric | direction: minimize + +--- + + Code + check_metrics_arg(metric_set(rmse), wflow) + Condition + Error: + ! The parsnip model has `mode` value of "quantile regression", but the `metrics` is a metric set for a different model mode. + +--- + + Code + check_metrics_arg(metric_set(weighted_interval_score), wflow) + Output + A metric set, consisting of: + - `weighted_interval_score()`, a quantile metric | direction: minimize + diff --git a/tests/testthat/test-metric-args.R b/tests/testthat/test-metric-args.R index 20ea93719..190e7e766 100644 --- a/tests/testthat/test-metric-args.R +++ b/tests/testthat/test-metric-args.R @@ -21,12 +21,15 @@ test_that("metric inputs are checked for regression models", { metric_set(brier_survival_integrated, brier_survival, concordance_survival) met_reg <- metric_set(rmse) met_cls <- metric_set(brier_class) + met_qnt <- metric_set(weighted_interval_score) # ------------------------------------------------------------------------------ # check inputs expect_snapshot(check_metrics_arg(NULL, wflow)) + expect_snapshot(check_metrics_arg(rmse, wflow), error = TRUE) + expect_snapshot(check_metrics_arg(met_reg, wflow)) expect_snapshot(check_metrics_arg(met_cls, wflow), error = TRUE) expect_snapshot(check_metrics_arg(met_mix_int, wflow), error = TRUE) @@ -180,3 +183,17 @@ test_that("metric inputs are checked for censored regression models", { expect_snapshot(last_fit(wflow, split, metrics = met_cls), error = TRUE) expect_snapshot(last_fit(wflow, split, metrics = met_reg), error = TRUE) }) + +test_that("metric inputs are checked for censored regression models", { + wflow <- workflow( + y ~ x, + linear_reg(engine = "quantreg", mode = "quantile regression") + ) + + expect_snapshot(check_metrics_arg(NULL, wflow)) + + expect_snapshot(check_metrics_arg(metric_set(rmse), wflow), error = TRUE) + expect_snapshot( + check_metrics_arg(metric_set(weighted_interval_score), wflow) + ) +}) From a33fb1c4d11c1eef28239e0b7055da6da637ca33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98topepo=E2=80=99?= Date: Thu, 23 Apr 2026 09:32:03 -0400 Subject: [PATCH 2/5] news file entry --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 6cf444144..5b188176d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # tune (development version) +## Bug Fixes + +* Resampling and tuning would fail for quantile regression models if they passed a quantile regression metric (#1186) + # tune 2.1.0 * Model tuning has been enabled for quantile regression models. (#1125) From d21b253ea9f118c6be675b528ec08d375acd62c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98topepo=E2=80=99?= Date: Fri, 24 Apr 2026 08:30:07 -0400 Subject: [PATCH 3/5] fix test name --- tests/testthat/_snaps/metric-args.md | 2 +- tests/testthat/test-metric-args.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/_snaps/metric-args.md b/tests/testthat/_snaps/metric-args.md index 0961747b3..2dc6a2bfa 100644 --- a/tests/testthat/_snaps/metric-args.md +++ b/tests/testthat/_snaps/metric-args.md @@ -297,7 +297,7 @@ Error in `last_fit()`: ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. ---- +# metric inputs are checked for quantile regression models Code check_metrics_arg(NULL, wflow) diff --git a/tests/testthat/test-metric-args.R b/tests/testthat/test-metric-args.R index 190e7e766..0fc3dda2c 100644 --- a/tests/testthat/test-metric-args.R +++ b/tests/testthat/test-metric-args.R @@ -184,7 +184,7 @@ test_that("metric inputs are checked for censored regression models", { expect_snapshot(last_fit(wflow, split, metrics = met_reg), error = TRUE) }) -test_that("metric inputs are checked for censored regression models", { +test_that("metric inputs are checked for quantile regression models", { wflow <- workflow( y ~ x, linear_reg(engine = "quantreg", mode = "quantile regression") From 762e179277dd958848fe3a5a0d40ec8fc5c89c20 Mon Sep 17 00:00:00 2001 From: Max Kuhn Date: Sat, 25 Apr 2026 12:57:48 -0400 Subject: [PATCH 4/5] Update R/metric-selection.R Co-authored-by: Hannah Frick --- R/metric-selection.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/metric-selection.R b/R/metric-selection.R index 02d466745..5704b13e2 100644 --- a/R/metric-selection.R +++ b/R/metric-selection.R @@ -345,8 +345,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { if (mode == "regression" && !is_numeric_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but {.arg metrics} is a metric set for a different model mode.", call = call ) } From d62be19693dd5fe062f30b5fe42588efe695f9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98topepo=E2=80=99?= Date: Tue, 28 Apr 2026 15:01:28 -0400 Subject: [PATCH 5/5] changes based on reviewer comments --- R/metric-selection.R | 25 +++++------ tests/testthat/_snaps/censored-reg.md | 2 +- tests/testthat/_snaps/metric-args.md | 62 +++++++++++++-------------- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/R/metric-selection.R b/R/metric-selection.R index 5704b13e2..9a485c397 100644 --- a/R/metric-selection.R +++ b/R/metric-selection.R @@ -328,13 +328,13 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { ) return(mtr_set) - } else { - if (!inherits(mtr_set, "metric_set")) { - cli::cli_abort( - "The {.arg metrics} argument should have class {.cls metric_set}, not {.cls {class(mtr_set)}}.", - call = call - ) - } + } + + if (!inherits(mtr_set, "metric_set")) { + cli::cli_abort( + "The {.arg metrics} argument should have class {.cls metric_set}, not {.cls {class(mtr_set)}}.", + call = call + ) } is_numeric_metric_set <- inherits(mtr_set, "numeric_metric_set") @@ -345,7 +345,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { if (mode == "regression" && !is_numeric_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but {.arg metrics} is a metric set for a different model mode.", + but the metric set for a different model mode.", call = call ) } @@ -353,8 +353,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { if (mode == "classification" && !is_class_prob_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } @@ -362,8 +361,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { if (mode == "censored regression" && !is_surv_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } @@ -371,8 +369,7 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { if (mode == "quantile regression" && !is_qnt_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } diff --git a/tests/testthat/_snaps/censored-reg.md b/tests/testthat/_snaps/censored-reg.md index 765c67a60..e90a62ef3 100644 --- a/tests/testthat/_snaps/censored-reg.md +++ b/tests/testthat/_snaps/censored-reg.md @@ -12,7 +12,7 @@ tune_grid(spec, Surv(time, status) ~ ., resamples = rs, metrics = reg_mtr) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- diff --git a/tests/testthat/_snaps/metric-args.md b/tests/testthat/_snaps/metric-args.md index 2dc6a2bfa..c65d51cb8 100644 --- a/tests/testthat/_snaps/metric-args.md +++ b/tests/testthat/_snaps/metric-args.md @@ -29,7 +29,7 @@ check_metrics_arg(met_cls, wflow) Condition Error: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -37,7 +37,7 @@ check_metrics_arg(met_mix_int, wflow) Condition Error: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -45,7 +45,7 @@ fit_resamples(wflow, rs, metrics = met_cls) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -53,7 +53,7 @@ fit_resamples(wflow, rs, metrics = met_mix_int) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -61,7 +61,7 @@ tune_grid(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -69,7 +69,7 @@ tune_grid(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -77,7 +77,7 @@ tune_bayes(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -85,7 +85,7 @@ tune_bayes(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -93,7 +93,7 @@ last_fit(wflow, split, metrics = met_cls) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -101,7 +101,7 @@ last_fit(wflow, split, metrics = met_mix_int) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. # metric inputs are checked for classification models @@ -119,7 +119,7 @@ check_metrics_arg(met_reg, wflow) Condition Error: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -135,7 +135,7 @@ check_metrics_arg(met_mix_int, wflow) Condition Error: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -143,7 +143,7 @@ fit_resamples(wflow, rs, metrics = met_reg) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -151,7 +151,7 @@ fit_resamples(wflow, rs, metrics = met_mix_int) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -159,7 +159,7 @@ tune_grid(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -167,7 +167,7 @@ tune_grid(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -175,7 +175,7 @@ tune_bayes(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -183,7 +183,7 @@ tune_bayes(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -191,7 +191,7 @@ last_fit(wflow, split, metrics = met_reg) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -199,7 +199,7 @@ last_fit(wflow, split, metrics = met_mix_int) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. # metric inputs are checked for censored regression models @@ -215,7 +215,7 @@ check_metrics_arg(met_reg, wflow) Condition Error: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -223,7 +223,7 @@ check_metrics_arg(met_cls, wflow) Condition Error: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -239,7 +239,7 @@ fit_resamples(wflow, rs, metrics = met_cls) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -247,7 +247,7 @@ fit_resamples(wflow, rs, metrics = met_reg) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -255,7 +255,7 @@ tune_grid(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -263,7 +263,7 @@ tune_grid(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -271,7 +271,7 @@ tune_bayes(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -279,7 +279,7 @@ tune_bayes(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -287,7 +287,7 @@ last_fit(wflow, split, metrics = met_cls) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -295,7 +295,7 @@ last_fit(wflow, split, metrics = met_reg) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. # metric inputs are checked for quantile regression models @@ -311,7 +311,7 @@ check_metrics_arg(metric_set(rmse), wflow) Condition Error: - ! The parsnip model has `mode` value of "quantile regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "quantile regression", but the metric set for a different model mode. ---