diff --git a/.Rbuildignore b/.Rbuildignore index 541b3e26..a67e324a 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -21,3 +21,4 @@ _cache$ ^\.vscode$ ^[.]?air[.]toml$ ^\.claude$ +^\.positai$ diff --git a/.gitignore b/.gitignore index 8f2f413a..96f9b0b2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tests/testthat/Rplots.pdf docs revdep tests/testthat/_snaps/notes.new.md +.positai diff --git a/NEWS.md b/NEWS.md index 6cf44414..5b188176 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # tune (development version) +## Bug Fixes + +* Resampling and tuning would fail for quantile regression models if they passed a quantile regression metric (#1186) + # tune 2.1.0 * Model tuning has been enabled for quantile regression models. (#1125) diff --git a/R/metric-selection.R b/R/metric-selection.R index 8692a2da..9a485c39 100644 --- a/R/metric-selection.R +++ b/R/metric-selection.R @@ -330,43 +330,46 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) { return(mtr_set) } + if (!inherits(mtr_set, "metric_set")) { + cli::cli_abort( + "The {.arg metrics} argument should have class {.cls metric_set}, not {.cls {class(mtr_set)}}.", + call = call + ) + } + is_numeric_metric_set <- inherits(mtr_set, "numeric_metric_set") is_class_prob_metric_set <- inherits(mtr_set, "class_prob_metric_set") is_surv_metric_set <- inherits(mtr_set, c("survival_metric_set")) + is_qnt_metric_set <- inherits(mtr_set, c("quantile_metric_set")) - if ( - !is_numeric_metric_set && !is_class_prob_metric_set && !is_surv_metric_set - ) { + if (mode == "regression" && !is_numeric_metric_set) { cli::cli_abort( - "The {.arg metrics} argument should be the results of - {.fn yardstick::metric_set}.", + "The parsnip model has {.code mode} value of {.val {mode}}, + but the metric set for a different model mode.", call = call ) } - if (mode == "regression" && !is_numeric_metric_set) { + if (mode == "classification" && !is_class_prob_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } - if (mode == "classification" && !is_class_prob_metric_set) { + if (mode == "censored regression" && !is_surv_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } - if (mode == "censored regression" && !is_surv_metric_set) { + if (mode == "quantile regression" && !is_qnt_metric_set) { cli::cli_abort( "The parsnip model has {.code mode} value of {.val {mode}}, - but the {.arg metrics} is a metric set for a - different model mode.", + but the metric set for a different model mode.", call = call ) } diff --git a/tests/testthat/_snaps/censored-reg.md b/tests/testthat/_snaps/censored-reg.md index 765c67a6..e90a62ef 100644 --- a/tests/testthat/_snaps/censored-reg.md +++ b/tests/testthat/_snaps/censored-reg.md @@ -12,7 +12,7 @@ tune_grid(spec, Surv(time, status) ~ ., resamples = rs, metrics = reg_mtr) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- diff --git a/tests/testthat/_snaps/metric-args.md b/tests/testthat/_snaps/metric-args.md index 64b37ff6..c65d51cb 100644 --- a/tests/testthat/_snaps/metric-args.md +++ b/tests/testthat/_snaps/metric-args.md @@ -7,6 +7,14 @@ - `rmse()`, a numeric metric | direction: minimize - `rsq()`, a numeric metric | direction: maximize +--- + + Code + check_metrics_arg(rmse, wflow) + Condition + Error: + ! The `metrics` argument should have class , not . + --- Code @@ -21,7 +29,7 @@ check_metrics_arg(met_cls, wflow) Condition Error: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -29,7 +37,7 @@ check_metrics_arg(met_mix_int, wflow) Condition Error: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -37,7 +45,7 @@ fit_resamples(wflow, rs, metrics = met_cls) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -45,7 +53,7 @@ fit_resamples(wflow, rs, metrics = met_mix_int) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -53,7 +61,7 @@ tune_grid(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -61,7 +69,7 @@ tune_grid(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -69,7 +77,7 @@ tune_bayes(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -77,7 +85,7 @@ tune_bayes(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -85,7 +93,7 @@ last_fit(wflow, split, metrics = met_cls) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. --- @@ -93,7 +101,7 @@ last_fit(wflow, split, metrics = met_mix_int) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode. # metric inputs are checked for classification models @@ -111,7 +119,7 @@ check_metrics_arg(met_reg, wflow) Condition Error: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -127,7 +135,7 @@ check_metrics_arg(met_mix_int, wflow) Condition Error: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -135,7 +143,7 @@ fit_resamples(wflow, rs, metrics = met_reg) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -143,7 +151,7 @@ fit_resamples(wflow, rs, metrics = met_mix_int) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -151,7 +159,7 @@ tune_grid(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -159,7 +167,7 @@ tune_grid(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -167,7 +175,7 @@ tune_bayes(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -175,7 +183,7 @@ tune_bayes(wflow_tune, rs, metrics = met_mix_int) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -183,7 +191,7 @@ last_fit(wflow, split, metrics = met_reg) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. --- @@ -191,7 +199,7 @@ last_fit(wflow, split, metrics = met_mix_int) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode. # metric inputs are checked for censored regression models @@ -207,7 +215,7 @@ check_metrics_arg(met_reg, wflow) Condition Error: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -215,7 +223,7 @@ check_metrics_arg(met_cls, wflow) Condition Error: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -231,7 +239,7 @@ fit_resamples(wflow, rs, metrics = met_cls) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -239,7 +247,7 @@ fit_resamples(wflow, rs, metrics = met_reg) Condition Error in `fit_resamples()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -247,7 +255,7 @@ tune_grid(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -255,7 +263,7 @@ tune_grid(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_grid()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -263,7 +271,7 @@ tune_bayes(wflow_tune, rs, metrics = met_cls) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -271,7 +279,7 @@ tune_bayes(wflow_tune, rs, metrics = met_reg) Condition Error in `tune_bayes()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -279,7 +287,7 @@ last_fit(wflow, split, metrics = met_cls) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. --- @@ -287,5 +295,29 @@ last_fit(wflow, split, metrics = met_reg) Condition Error in `last_fit()`: - ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode. + ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode. + +# metric inputs are checked for quantile regression models + + Code + check_metrics_arg(NULL, wflow) + Output + A metric set, consisting of: + - `weighted_interval_score()`, a quantile metric | direction: minimize + +--- + + Code + check_metrics_arg(metric_set(rmse), wflow) + Condition + Error: + ! The parsnip model has `mode` value of "quantile regression", but the metric set for a different model mode. + +--- + + Code + check_metrics_arg(metric_set(weighted_interval_score), wflow) + Output + A metric set, consisting of: + - `weighted_interval_score()`, a quantile metric | direction: minimize diff --git a/tests/testthat/test-metric-args.R b/tests/testthat/test-metric-args.R index 20ea9371..0fc3dda2 100644 --- a/tests/testthat/test-metric-args.R +++ b/tests/testthat/test-metric-args.R @@ -21,12 +21,15 @@ test_that("metric inputs are checked for regression models", { metric_set(brier_survival_integrated, brier_survival, concordance_survival) met_reg <- metric_set(rmse) met_cls <- metric_set(brier_class) + met_qnt <- metric_set(weighted_interval_score) # ------------------------------------------------------------------------------ # check inputs expect_snapshot(check_metrics_arg(NULL, wflow)) + expect_snapshot(check_metrics_arg(rmse, wflow), error = TRUE) + expect_snapshot(check_metrics_arg(met_reg, wflow)) expect_snapshot(check_metrics_arg(met_cls, wflow), error = TRUE) expect_snapshot(check_metrics_arg(met_mix_int, wflow), error = TRUE) @@ -180,3 +183,17 @@ test_that("metric inputs are checked for censored regression models", { expect_snapshot(last_fit(wflow, split, metrics = met_cls), error = TRUE) expect_snapshot(last_fit(wflow, split, metrics = met_reg), error = TRUE) }) + +test_that("metric inputs are checked for quantile regression models", { + wflow <- workflow( + y ~ x, + linear_reg(engine = "quantreg", mode = "quantile regression") + ) + + expect_snapshot(check_metrics_arg(NULL, wflow)) + + expect_snapshot(check_metrics_arg(metric_set(rmse), wflow), error = TRUE) + expect_snapshot( + check_metrics_arg(metric_set(weighted_interval_score), wflow) + ) +})