Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion R/GetCovariates.R
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@
#' @param minCharacterizationMean The minimum mean value for characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that have very low values. The default is 0.
#' @param minCharacterizationCount The minimum count value for characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that occur in very few cohort entries. The default is 0.
#' @param tempEmulationSchema Some database platforms like Oracle and Impala do not truly support
#' temp tables. To emulate temp tables, provide a schema with write
#' privileges where temp tables can be created.
Expand Down Expand Up @@ -147,6 +150,7 @@ getDbCovariateData <- function(connectionDetails = NULL,
targetTimeRefTable = NULL,
aggregated = FALSE,
minCharacterizationMean = 0,
minCharacterizationCount = 0,
tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
covariateCohortDatabaseSchema = NULL,
covariateCohortTable = NULL) {
Expand All @@ -172,7 +176,9 @@ getDbCovariateData <- function(connectionDetails = NULL,
}
errorMessages <- checkmate::makeAssertCollection()
minCharacterizationMean <- utils::type.convert(minCharacterizationMean, as.is = TRUE)
minCharacterizationCount <- utils::type.convert(minCharacterizationCount, as.is = TRUE)
checkmate::assertNumeric(x = minCharacterizationMean, lower = 0, upper = 1, add = errorMessages)
checkmate::assertIntegerish(x = minCharacterizationCount, lower = 0, len = 1, add = errorMessages)
checkmate::reportAssertions(collection = errorMessages)
if (!is.null(connectionDetails)) {
connection <- DatabaseConnector::connect(connectionDetails)
Expand Down Expand Up @@ -351,7 +357,8 @@ getDbCovariateData <- function(connectionDetails = NULL,
targetAnalysisRefTable = targetAnalysisRefTable,
targetTimeRefTable = targetTimeRefTable,
aggregated = aggregated,
minCharacterizationMean = minCharacterizationMean
minCharacterizationMean = minCharacterizationMean,
minCharacterizationCount = minCharacterizationCount
)
tempCovariateData <- do.call(eval(parse(text = fun)), args)
if (is.null(covariateData)) {
Expand Down
16 changes: 16 additions & 0 deletions R/GetCovariatesFromCohortAttributes.R
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@
#'
#' @param covariateSettings An object of type \code{covariateSettings} as created using the
#' \code{\link{createCohortAttrCovariateSettings}} function.
#' @param targetDatabaseSchema Optional target database schema. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param targetCovariateTable Optional target covariate table. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param targetCovariateContinuousTable Optional target continuous covariate table. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param targetCovariateRefTable Optional target covariate reference table. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param targetAnalysisRefTable Optional target analysis reference table. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param targetTimeRefTable Optional target time reference table. Ignored because cohort attribute covariates do not support exporting directly to target tables.
#' @param minCharacterizationMean The minimum mean value for characterization output. Ignored because aggregation is not supported for cohort attribute covariates.
#' @param minCharacterizationCount The minimum count value for characterization output. Ignored because aggregation is not supported for cohort attribute covariates.
#'
#' @template GetCovarParams
#'
Expand Down Expand Up @@ -66,7 +74,15 @@ getDbCohortAttrCovariatesData <- function(connection,
cdmVersion = "5",
rowIdField = "subject_id",
covariateSettings,
targetDatabaseSchema = NULL,
targetCovariateTable = NULL,
targetCovariateContinuousTable = NULL,
targetCovariateRefTable = NULL,
targetAnalysisRefTable = NULL,
targetTimeRefTable = NULL,
aggregated = FALSE,
minCharacterizationMean = 0,
minCharacterizationCount = 0,
tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")) {
if (aggregated) {
stop("Aggregation not implemented for covariates from cohort attributes.")
Expand Down
9 changes: 8 additions & 1 deletion R/GetCovariatesFromOtherCohorts.R
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
#' @param minCharacterizationMean The minimum mean value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that have very low values. The default is 0.
#' @param minCharacterizationCount The minimum count value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that occur in very few cohort entries. The default is 0.
#' @template GetCovarParams
#'
#' @export
Expand All @@ -57,6 +60,7 @@ getDbCohortBasedCovariatesData <- function(connection,
targetTimeRefTable = NULL,
aggregated = FALSE,
minCharacterizationMean = 0,
minCharacterizationCount = 0,
tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")) {
errorMessages <- checkmate::makeAssertCollection()
checkmate::assertClass(connection, "DatabaseConnectorConnection", add = errorMessages)
Expand All @@ -70,7 +74,9 @@ getDbCohortBasedCovariatesData <- function(connection,
checkmate::assertClass(covariateSettings, "covariateSettings", add = errorMessages)
checkmate::assertLogical(aggregated, len = 1, add = errorMessages)
minCharacterizationMean <- utils::type.convert(minCharacterizationMean, as.is = TRUE)
minCharacterizationCount <- utils::type.convert(minCharacterizationCount, as.is = TRUE)
checkmate::assertNumeric(x = minCharacterizationMean, lower = 0, upper = 1, add = errorMessages)
checkmate::assertIntegerish(x = minCharacterizationCount, lower = 0, len = 1, add = errorMessages)
checkmate::reportAssertions(collection = errorMessages)
if (!missing(cohortId)) {
warning("cohortId argument has been deprecated, please use cohortIds")
Expand Down Expand Up @@ -177,7 +183,8 @@ getDbCohortBasedCovariatesData <- function(connection,
targetAnalysisRefTable = targetAnalysisRefTable,
targetTimeRefTable = targetTimeRefTable,
aggregated = aggregated,
minCharacterizationMean = minCharacterizationMean
minCharacterizationMean = minCharacterizationMean,
minCharacterizationCount = minCharacterizationCount
)

sql <- "TRUNCATE TABLE #covariate_cohort_ref; DROP TABLE #covariate_cohort_ref;"
Expand Down
8 changes: 7 additions & 1 deletion R/GetDefaultCovariates.R
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
#' @param minCharacterizationMean The minimum mean value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that have very low values. The default is 0.
#' @param minCharacterizationCount The minimum count value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that occur in very few cohort entries. The default is 0.
#' @template GetCovarParams
#'
#' @examples
Expand Down Expand Up @@ -75,6 +78,7 @@ getDbDefaultCovariateData <- function(connection,
targetTimeRefTable = NULL,
aggregated = FALSE,
minCharacterizationMean = 0,
minCharacterizationCount = 0,
tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")) {
if (!is(covariateSettings, "covariateSettings")) {
stop("Covariate settings object not of type covariateSettings")
Expand All @@ -96,7 +100,9 @@ getDbDefaultCovariateData <- function(connection,
}
errorMessages <- checkmate::makeAssertCollection()
minCharacterizationMean <- utils::type.convert(minCharacterizationMean, as.is = TRUE)
minCharacterizationCount <- utils::type.convert(minCharacterizationCount, as.is = TRUE)
checkmate::assertNumeric(x = minCharacterizationMean, lower = 0, upper = 1, add = errorMessages)
checkmate::assertIntegerish(x = minCharacterizationCount, lower = 0, len = 1, add = errorMessages)
checkmate::reportAssertions(collection = errorMessages)


Expand All @@ -115,7 +121,7 @@ getDbDefaultCovariateData <- function(connection,
settings <- .toJson(covariateSettings)
rJava::J("org.ohdsi.featureExtraction.FeatureExtraction")$init(system.file("", package = "FeatureExtraction"))
json <- rJava::J("org.ohdsi.featureExtraction.FeatureExtraction")$createSql(
settings, aggregated, cohortTable, rowIdField, rJava::.jarray(as.character(cohortIds)), cdmDatabaseSchema, as.character(minCharacterizationMean)
settings, aggregated, cohortTable, rowIdField, rJava::.jarray(as.character(cohortIds)), cdmDatabaseSchema, as.character(minCharacterizationMean), as.character(minCharacterizationCount)
)
todo <- .fromJson(json)
if (length(todo$tempTables) != 0) {
Expand Down
4 changes: 4 additions & 0 deletions R/UnitTestHelperFunctions.R
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
#' @param minCharacterizationMean The minimum mean value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that have very low values. The default is 0.
#' @param minCharacterizationCount The minimum count value for binary characterization output. Values below this will be cut off from output. This
#' will help reduce the file size of the characterization output, but will remove information
#' on covariates that occur in very few cohort entries. The default is 0.
#' @param ... Additional arguments, not used.
#' @return
#' Returns an object of type \code{covariateData}, containing information on the covariates.
Expand All @@ -73,6 +76,7 @@
covariateSettings,
aggregated = FALSE,
minCharacterizationMean = 0,
minCharacterizationCount = 0,
...) {
writeLines("Constructing length of observation covariates")
if (covariateSettings$useLengthOfObs == FALSE) {
Expand Down
2 changes: 1 addition & 1 deletion inst/csv/jarChecksum.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
25224c0381cb8cf2d969e483400ae8b4caaaf3979df00b43c1197ce43c0cedbb
148c135b5461287933e45215d633318bd0a81778989061930bcde3604096ccd5
Binary file not shown.
41 changes: 36 additions & 5 deletions java/org/ohdsi/featureExtraction/FeatureExtraction.java
Original file line number Diff line number Diff line change
Expand Up @@ -515,11 +515,23 @@ else if (temporalSequence)
*/
public static String createSql(String settings, boolean aggregated, String cohortTable, String rowIdField, String[] cohortDefinitionIds,
String cdmDatabaseSchema, String minCharacterizationMean) {
return createSql(settings, aggregated, cohortTable, rowIdField, cohortDefinitionIds, cdmDatabaseSchema, minCharacterizationMean, "0");
}

/**
* Construct the SQL for creating and retrieving the features.
*
* @param minCharacterizationCount The minimum count value for characterization output. Values below this will be cut off from output.
* @return A JSON object.
*/
public static String createSql(String settings, boolean aggregated, String cohortTable, String rowIdField, String[] cohortDefinitionIds,
String cdmDatabaseSchema, String minCharacterizationMean, String minCharacterizationCount) {

long[] idsAsLongs = new long[cohortDefinitionIds.length];
for (int i = 0; i < cohortDefinitionIds.length; i++)
idsAsLongs[i] = Long.valueOf(cohortDefinitionIds[i]);
return createSql(settings, aggregated, cohortTable, rowIdField, idsAsLongs, cdmDatabaseSchema, Double.valueOf(minCharacterizationMean));
return createSql(settings, aggregated, cohortTable, rowIdField, idsAsLongs, cdmDatabaseSchema, Double.valueOf(minCharacterizationMean),
Integer.valueOf(minCharacterizationCount));
}


Expand Down Expand Up @@ -558,7 +570,7 @@ public static String createSql(String settings, boolean aggregated, String cohor
*/
public static String createSql(String settings, boolean aggregated, String cohortTable, String rowIdField, int cohortDefinitionId,
String cdmDatabaseSchema) {
return createSql(settings, aggregated, cohortTable, rowIdField, new long[]{cohortDefinitionId}, cdmDatabaseSchema, 0.0d);
return createSql(settings, aggregated, cohortTable, rowIdField, new long[]{cohortDefinitionId}, cdmDatabaseSchema, 0.0d, 0);
}

/**
Expand Down Expand Up @@ -596,6 +608,17 @@ public static String createSql(String settings, boolean aggregated, String cohor
*/
public static String createSql(String settings, boolean aggregated, String cohortTable, String rowIdField, long[] cohortDefinitionIds,
String cdmDatabaseSchema, double minCharacterizationMean) {
return createSql(settings, aggregated, cohortTable, rowIdField, cohortDefinitionIds, cdmDatabaseSchema, minCharacterizationMean, 0);
}

/**
* Construct the SQL for creating and retrieving the features.
*
* @param minCharacterizationCount The minimum count value for characterization output. Values below this will be cut off from output.
* @return A JSON object.
*/
public static String createSql(String settings, boolean aggregated, String cohortTable, String rowIdField, long[] cohortDefinitionIds,
String cdmDatabaseSchema, double minCharacterizationMean, int minCharacterizationCount) {

JSONObject jsonObject = new JSONObject(settings);

Expand Down Expand Up @@ -666,7 +689,8 @@ public static String createSql(String settings, boolean aggregated, String cohor
jsonWriter.key("sqlConstruction");
jsonWriter.value(createConstructionSql(jsonObject, idSetToName, temporal, temporalSequence, aggregated, cohortTable, rowIdField, cohortDefinitionIds, cdmDatabaseSchema));

String sqlQueryFeatures = createQuerySql(jsonObject, cohortTable, cohortDefinitionIds, aggregated, temporal, temporalSequence, minCharacterizationMean);
String sqlQueryFeatures = createQuerySql(jsonObject, cohortTable, cohortDefinitionIds, aggregated, temporal, temporalSequence, minCharacterizationMean,
minCharacterizationCount);
if (sqlQueryFeatures != null) {
jsonWriter.key("sqlQueryFeatures");
jsonWriter.value(sqlQueryFeatures);
Expand Down Expand Up @@ -743,7 +767,7 @@ private static Object createCleanupSql(JSONObject jsonObject, boolean temporal2)
}

private static String createQuerySql(JSONObject jsonObject, String cohortTable, long[] cohortDefinitionIds, boolean aggregated, boolean temporal, boolean temporalSequence,
double minCharacterizationMean) {
double minCharacterizationMean, int minCharacterizationCount) {
boolean temporalAnnual = isTemporalAnnual(jsonObject);
Stream<String> fields = Stream.<Stream<String>>of(
aggregated ? Stream.of("cohort_definition_id", "covariate_id", "sum_value") : Stream.of("row_id", "covariate_id", "covariate_value"),
Expand Down Expand Up @@ -785,8 +809,15 @@ else if (temporalAnnual)
if (aggregated) {
sql.append("\n) all_covariates\nINNER JOIN (\nSELECT cohort_definition_id, COUNT(*) AS total_count\nFROM @cohort_table {@cohort_definition_id != -1} ? {\nWHERE cohort_definition_id IN (@cohort_definition_id)}");
sql.append(" GROUP BY cohort_definition_id\n) total\n ON all_covariates.cohort_definition_id = total.cohort_definition_id");
List<String> filterConditions = new ArrayList<String>();
if (minCharacterizationMean != 0) {
sql.append(" WHERE all_covariates.sum_value / (1.0 * total.total_count) >= " + minCharacterizationMean);
filterConditions.add("all_covariates.sum_value / (1.0 * total.total_count) >= " + minCharacterizationMean);
}
if (minCharacterizationCount != 0) {
filterConditions.add("all_covariates.sum_value >= " + minCharacterizationCount);
}
if (!filterConditions.isEmpty()) {
sql.append(" WHERE " + filterConditions.stream().collect(Collectors.joining(" AND ")));
}
sql.append(";");
} else {
Expand Down
5 changes: 5 additions & 0 deletions man/dot-getDbLooCovariateData.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading