OOB impurity importance

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: ranger
 Type: Package
 Title: A Fast Implementation of Random Forests
-Version: 0.12.3
-Date: 2020-05-08
+Version: 0.12.4
+Date: 2020-07-10
 Author: Marvin N. Wright [aut, cre], Stefan Wager [ctb], Philipp Probst [ctb]
 Maintainer: Marvin N. Wright <cran@wrig.de>
 Description: A fast implementation of Random Forests, particularly suited for high
@@ -17,6 +17,6 @@ LinkingTo: Rcpp, RcppEigen
 Depends: R (>= 3.1)
 Suggests: survival, testthat
 Encoding: UTF-8
-RoxygenNote: 7.0.2
+RoxygenNote: 7.1.0
 URL: https://github.com/imbs-hl/ranger
 BugReports: https://github.com/imbs-hl/ranger/issues

NEWS

@@ -1,4 +1,7 @@
 
+##### Version 0.12.4
+* Add OOB impurity importance
+
 ##### Version 0.12.3
 * Add ... argument to ranger()
 * Bug fixes

NEWS.md

@@ -1,4 +1,7 @@
 
+##### Version 0.12.4
+* Add OOB impurity importance
+
 ##### Version 0.12.3
 * Add ... argument to ranger()
 * Bug fixes

R/ranger.R

@@ -63,6 +63,7 @@
 ##' It is a modified version of the method by Sandri & Zuccolotto (2008), which is faster and more memory efficient. 
 ##' See Nembrini et al. (2018) for details.
 ##' This importance measure can be combined with the methods to estimate p-values in \code{\link{importance_pvalues}}.
+##' The 'impurity_oob' importance measure is based on a combination of inbag and out-of-bag observations and is also unbiased, see Zhou & Hooker (2019) or Loecher (2020).
 ##'
 ##' Regularization works by penalizing new variables by multiplying the splitting criterion by a factor, see Deng & Runger (2012) for details.  
 ##' If \code{regularization.usedepth=TRUE}, \eqn{f^d} is used, where \emph{f} is the regularization factor and \emph{d} the depth of the node.
@@ -89,7 +90,7 @@
 ##' @param data Training data of class \code{data.frame}, \code{matrix}, \code{dgCMatrix} (Matrix) or \code{gwaa.data} (GenABEL).
 ##' @param num.trees Number of trees.
 ##' @param mtry Number of variables to possibly split at in each node. Default is the (rounded down) square root of the number variables. Alternatively, a single argument function returning an integer, given the number of independent variables.
-##' @param importance Variable importance mode, one of 'none', 'impurity', 'impurity_corrected', 'permutation'. The 'impurity' measure is the Gini index for classification, the variance of the responses for regression and the sum of test statistics (see \code{splitrule}) for survival. 
+##' @param importance Variable importance mode, one of 'none', 'impurity', 'impurity_corrected', 'impurity_oob', 'permutation'. The 'impurity' measure is the Gini index for classification, the variance of the responses for regression and the sum of test statistics (see \code{splitrule}) for survival. 
 ##' @param write.forest Save \code{ranger.forest} object, required for prediction. Set to \code{FALSE} to reduce memory usage if no prediction intended.
 ##' @param probability Grow a probability forest as in Malley et al. (2012). 
 ##' @param min.node.size Minimal node size. Default 1 for classification, 5 for regression, 3 for survival, and 10 for probability.
@@ -200,6 +201,8 @@
 ##'   \item Sandri, M. & Zuccolotto, P. (2008). A bias correction algorithm for the Gini variable importance measure in classification trees. J Comput Graph Stat, 17:611-628. \url{https://doi.org/10.1198/106186008X344522}.
 ##'   \item Coppersmith D., Hong S. J., Hosking J. R. (1999). Partitioning nominal attributes in decision trees. Data Min Knowl Discov 3:197-217. \url{https://doi.org/10.1023/A:1009869804967}.
 ##'   \item Deng & Runger (2012). Feature selection via regularized trees. The 2012 International Joint Conference on Neural Networks (IJCNN), Brisbane, Australia. \url{https://doi.org/10.1109/IJCNN.2012.6252640}.
+##'   \item Zhou & Hooker (2019). Unbiased measurement of feature importance in tree-based methods. arXiv:1903.05179. \url{https://arxiv.org/abs/1903.05179}.
+##'   \item Loecher (2020). Unbiased variable importance for random forests. Commun Stat - Theory Methods. \url{https://doi.org/10.1080/03610926.2020.1764042}.
 ##'   }
 ##' @seealso \code{\link{predict.ranger}}
 ##' @useDynLib ranger, .registration = TRUE
@@ -572,6 +575,8 @@ ranger <- function(formula = NULL, data = NULL, num.trees = 500, mtry = NULL,
     importance.mode <- 1
   } else if (importance == "impurity_corrected" || importance == "impurity_unbiased") {
     importance.mode <- 5
+  } else if (importance == "impurity_oob") {
+    importance.mode <- 7
   } else if (importance == "permutation") {
     if (local.importance) {
       importance.mode <- 6

cpp_version/src/utility/ArgumentHandler.cpp

@@ -609,6 +609,7 @@ void ArgumentHandler::displayHelp() {
       << "                              TYPE = 5: Corrected node impurity: Bias-corrected version of node impurity importance."
       << std::endl;
   std::cout << "    " << "                              TYPE = 6: Local (casewise) permutation importance." << std::endl;
+  std::cout << "    " << "                              TYPE = 7: Out-of-bag impurity importance." << std::endl;
   std::cout << "    " << "                              (Default: 0)" << std::endl;
   std::cout << "    " << "--noreplace                   Sample without replacement." << std::endl;
   std::cout << "    "

cpp_version/src/version.h

@@ -1,3 +1,3 @@
 #ifndef RANGER_VERSION
-#define RANGER_VERSION "0.12.3"
+#define RANGER_VERSION "0.12.4"
 #endif

src/Forest.cpp

@@ -511,7 +511,7 @@ void Forest::grow() {
   std::vector<std::vector<double>> variable_importance_threads(num_threads);
 
   for (uint i = 0; i < num_threads; ++i) {
-    if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED) {
+    if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED || importance_mode == IMP_GINI_OOB) {
       variable_importance_threads[i].resize(num_independent_variables, 0);
     }
     threads.emplace_back(&Forest::growTreesInThread, this, i, &(variable_importance_threads[i]));
@@ -528,7 +528,7 @@ void Forest::grow() {
 #endif
 
   // Sum thread importances
-  if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED) {
+  if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED || importance_mode == IMP_GINI_OOB) {
     variable_importance.resize(num_independent_variables, 0);
     for (size_t i = 0; i < num_independent_variables; ++i) {
       for (uint j = 0; j < num_threads; ++j) {
@@ -541,7 +541,7 @@ void Forest::grow() {
 #endif
 
   // Divide importance by number of trees
-  if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED) {
+  if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED || importance_mode == IMP_GINI_OOB) {
     for (auto& v : variable_importance) {
       v /= num_trees;
     }

src/TreeClassification.cpp

@@ -196,6 +196,8 @@ bool TreeClassification::findBestSplit(size_t nodeID, std::vector<size_t>& possi
   // Compute gini index for this node and to variable importance if needed
   if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED) {
     addGiniImportance(nodeID, best_varID, best_decrease);
+  } else if (importance_mode == IMP_GINI_OOB) {
+    addGiniOobImportance(nodeID, best_varID, best_value);
   }
 
   // Regularization
@@ -536,6 +538,8 @@ bool TreeClassification::findBestSplitExtraTrees(size_t nodeID, std::vector<size
   // Compute gini index for this node and to variable importance if needed
   if (importance_mode == IMP_GINI || importance_mode == IMP_GINI_CORRECTED) {
     addGiniImportance(nodeID, best_varID, best_decrease);
+  } else if (importance_mode == IMP_GINI_OOB) {
+    addGiniOobImportance(nodeID, best_varID, best_value);
   }
 
   // Regularization
@@ -783,6 +787,62 @@ void TreeClassification::addGiniImportance(size_t nodeID, size_t varID, double d
   }
 }
 
+void TreeClassification::addGiniOobImportance(size_t nodeID, size_t varID, double split_value) {
+  size_t num_classes = class_values->size();
+  size_t num_samples_node = end_pos[nodeID] - start_pos[nodeID];
+
+  // Inbag
+  size_t n_left_inbag = 0;
+  std::vector<size_t> class_counts_all_inbag(num_classes);
+  std::vector<size_t> class_counts_left_inbag(num_classes);
+  for (size_t pos = start_pos[nodeID]; pos < end_pos[nodeID]; ++pos) {
+    size_t sampleID = sampleIDs[pos];
+    size_t classID = (*response_classIDs)[sampleID];
+
+    ++class_counts_all_inbag[classID];
+
+    if (data->get_x(sampleID, varID) <= split_value) {
+      ++class_counts_left_inbag[classID];
+      ++n_left_inbag;
+    }
+  }
+
+  // OOB
+  size_t n_left_oob = 0;
+  std::vector<size_t> class_counts_all_oob(num_classes);
+  std::vector<size_t> class_counts_left_oob(num_classes);
+  for (auto& sampleID : oob_sampleIDs) {
+    size_t classID = (*response_classIDs)[sampleID];
+    ++class_counts_all_oob[classID];
+
+    if (data->get_x(sampleID, varID) <= split_value) {
+      ++class_counts_left_oob[classID];
+      ++n_left_oob;
+    }
+  }
+  size_t n_oob = oob_sampleIDs.size();
+  size_t n_right_oob = n_oob - n_left_oob;
+
+  // Impurity in OOB observations
+  double sum_left = 0;
+  double sum_right = 0;
+  double sum_all = 0;
+  for (size_t j = 0; j < num_classes; ++j) {
+    size_t class_count_right_inbag = class_counts_all_inbag[j] - class_counts_left_inbag[j];
+    size_t class_count_right_oob = class_counts_all_oob[j] - class_counts_left_oob[j];
+
+    sum_left += (*class_weights)[j] * class_counts_left_inbag[j] * class_counts_left_oob[j];
+    sum_right += (*class_weights)[j] * class_count_right_inbag * class_count_right_oob;
+    sum_all += (*class_weights)[j] * class_counts_all_inbag[j] * class_counts_all_oob[j];
+  }
+
+  // Decrease of impurity
+  if (n_right_oob > 0 && n_left_oob > 0) {
+    double decrease = sum_right / (double) n_right_oob + sum_left / (double) n_left_oob - sum_all / (double) n_oob;
+    (*variable_importance)[varID] += decrease;
+  }
+}
+
 void TreeClassification::bootstrapClassWise() {
   // Number of samples is sum of sample fraction * number of samples
   size_t num_samples_inbag = 0;

src/TreeClassification.h

@@ -83,6 +83,7 @@ class TreeClassification: public Tree {
       double& best_decrease);
 
   void addGiniImportance(size_t nodeID, size_t varID, double decrease);
+  void addGiniOobImportance(size_t nodeID, size_t varID, double split_value);
 
   void bootstrapClassWise() override;
   void bootstrapWithoutReplacementClassWise() override;

src/globals.h

@@ -57,7 +57,8 @@ enum ImportanceMode {
   IMP_PERM_LIAW = 4,
   IMP_PERM_RAW = 3,
   IMP_GINI_CORRECTED = 5,
-  IMP_PERM_CASEWISE = 6
+  IMP_PERM_CASEWISE = 6,
+  IMP_GINI_OOB = 7
 };
 const uint MAX_IMP_MODE = 6;