From fc817de3815b81a247b53907880c9bd6c8e0660c Mon Sep 17 00:00:00 2001 From: Admin_mschuemi Date: Fri, 13 Jun 2025 03:13:28 -0400 Subject: [PATCH] Optimizing tidyCovariates for large data. Dixes #308 --- R/Normalization.R | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/R/Normalization.R b/R/Normalization.R index d9704f5..ab185bb 100644 --- a/R/Normalization.R +++ b/R/Normalization.R @@ -179,20 +179,27 @@ tidyCovariateData <- function(covariateData, deleteCovariateIds <- c(deleteCovariateIds, toDelete$covariateId) ParallelLogger::logInfo("Removing ", nrow(toDelete), " infrequent covariates") } - if (length(deleteCovariateIds) > 0) { - newCovariates <- newCovariates %>% - filter(!.data$covariateId %in% deleteCovariateIds) - } + # When performing both filtering by covariate IDs and normalization, it is *much* faster + # to apply the filtering to the maxValuePerCovariateId table, and let the inner join + # apply the filtering to the covariate table (instead of filtering the covariate table + # directly). if (normalize) { ParallelLogger::logInfo("Normalizing covariates") + if (length(deleteCovariateIds) > 0) { + covariateData$maxValuePerCovariateId <- covariateData$maxValuePerCovariateId %>% + filter(!.data$covariateId %in% deleteCovariateIds) + } newCovariates <- newCovariates %>% inner_join(covariateData$maxValuePerCovariateId, by = "covariateId") %>% mutate(covariateValue = .data$covariateValue / .data$maxValue) %>% select(-.data$maxValue) metaData$normFactors <- covariateData$maxValuePerCovariateId %>% collect() - } + } else if (length(deleteCovariateIds) > 0) { + newCovariates <- newCovariates %>% + filter(!.data$covariateId %in% deleteCovariateIds) + } newCovariateData$covariates <- newCovariates if (!is.null(covariateData$timeRef)) { newCovariateData$timeRef <- covariateData$timeRef