From 7e14387288957ea14dd8b4a4140040ec069db5fd Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Tue, 5 Nov 2024 12:01:11 -0800 Subject: [PATCH 01/35] attempt to patch count(*) overflow --- R/AggregateCovariates.R | 2 +- inst/settings/resultsDataModelSpecification.csv | 2 +- inst/sql/sql_server/ResultTables.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/AggregateCovariates.R b/R/AggregateCovariates.R index 8a3c220..6292b1d 100644 --- a/R/AggregateCovariates.R +++ b/R/AggregateCovariates.R @@ -260,7 +260,7 @@ computeTargetAggregateCovariateAnalyses <- function( message("Extracting target cohort counts") sql <- "select cohort_definition_id, - count(*) row_count, + count_big(*) row_count, count(distinct subject_id) person_count, min(datediff(day, cohort_start_date, cohort_end_date)) min_exposure_time, avg(datediff(day, cohort_start_date, cohort_end_date)) mean_exposure_time, diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 3dc0e59..ecd5d9e 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -113,7 +113,7 @@ cohort_counts,start_anchor,varchar(15),No,No,No,No,The start anchor cohort_counts,end_anchor,varchar(15),No,No,No,No,The end anchor cohort_counts,min_prior_observation,int,No,No,No,No,Minimum time observed before index cohort_counts,outcome_washout_days,int,No,No,No,No,Patients with outcome during washout are excluded -cohort_counts,row_count,int,Yes,No,No,No,The number of rows in each cohort +cohort_counts,row_count,bigint,Yes,No,No,No,The number of rows in each cohort cohort_counts,person_count,int,Yes,No,No,No,The number of distinct people in each cohort cohort_counts,min_exposure_time,bigint,No,No,No,No,Minimum exposure time across cohort cohort_counts,mean_exposure_time,bigint,No,No,No,No,Mean exposure time across cohort diff --git a/inst/sql/sql_server/ResultTables.sql b/inst/sql/sql_server/ResultTables.sql index edb85e2..5012558 100644 --- a/inst/sql/sql_server/ResultTables.sql +++ b/inst/sql/sql_server/ResultTables.sql @@ -155,7 +155,7 @@ CREATE TABLE @my_schema.@table_prefixcohort_counts( end_anchor varchar(15), min_prior_observation int, outcome_washout_days int, - row_count int NOT NULL, + row_count bigint NOT NULL, person_count int NOT NULL, min_exposure_time int, mean_exposure_time int, From 5c30f7404bfb3b285b515bd7275f5c59387a3fac Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 5 Nov 2024 16:12:18 -0500 Subject: [PATCH 02/35] Add migration instead of changing ResultTables.sql directly --- inst/sql/sql_server/ResultTables.sql | 2 +- .../migrations/Migration_2-v2_0_2_row_count_bigint.sql | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql diff --git a/inst/sql/sql_server/ResultTables.sql b/inst/sql/sql_server/ResultTables.sql index 5012558..edb85e2 100644 --- a/inst/sql/sql_server/ResultTables.sql +++ b/inst/sql/sql_server/ResultTables.sql @@ -155,7 +155,7 @@ CREATE TABLE @my_schema.@table_prefixcohort_counts( end_anchor varchar(15), min_prior_observation int, outcome_washout_days int, - row_count bigint NOT NULL, + row_count int NOT NULL, person_count int NOT NULL, min_exposure_time int, mean_exposure_time int, diff --git a/inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql b/inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql new file mode 100644 index 0000000..5d6f626 --- /dev/null +++ b/inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql @@ -0,0 +1,5 @@ +-- Database migrations for verion 2.0.2 +-- This migration updates the schema: + -- 1. To expand the row_count column to a bigint + +ALTER TABLE @database_schema.@table_prefixcohort_counts ALTER COLUMN row_count BIGINT; From 0bf7b0188055b7e01a067789fc5b3363b935cb67 Mon Sep 17 00:00:00 2001 From: OBrien Date: Sat, 15 Mar 2025 10:31:13 -0400 Subject: [PATCH 03/35] Cast datediff as bigint to prevent arithmetic overflow (occurred on VA's CDM) --- R/AggregateCovariates.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/AggregateCovariates.R b/R/AggregateCovariates.R index b537c9a..03d14b5 100644 --- a/R/AggregateCovariates.R +++ b/R/AggregateCovariates.R @@ -285,9 +285,9 @@ computeTargetAggregateCovariateAnalyses <- function( cohort_definition_id, count(*) row_count, count(distinct subject_id) person_count, - min(datediff(day, cohort_start_date, cohort_end_date)) min_exposure_time, - avg(datediff(day, cohort_start_date, cohort_end_date)) mean_exposure_time, - max(datediff(day, cohort_start_date, cohort_end_date)) max_exposure_time + min(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) min_exposure_time, + avg(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) mean_exposure_time, + max(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) max_exposure_time from (select * from #agg_cohorts_before union select * from #agg_cohorts_extras) temp group by cohort_definition_id;" From 3bd82d4450904471a4fa7863debdef5a1d34d7aa Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 11:43:18 -0400 Subject: [PATCH 04/35] Update CustomCovariates.R turning off progress bar --- R/CustomCovariates.R | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/R/CustomCovariates.R b/R/CustomCovariates.R index 2deee60..d422338 100644 --- a/R/CustomCovariates.R +++ b/R/CustomCovariates.R @@ -171,6 +171,8 @@ getDbDuringCovariateData <- function( getDomainSettings <- utils::read.csv(system.file("csv/PrespecAnalyses.csv", package = "Characterization")) + # not showing progress like FE + progressBar <- FALSE # create Tables sql <- "DROP TABLE IF EXISTS #cov_ref; @@ -187,7 +189,7 @@ getDbDuringCovariateData <- function( targetDialect = DatabaseConnector::dbms(connection), tempEmulationSchema = tempEmulationSchema ) - DatabaseConnector::executeSql(connection, sql = sql) + DatabaseConnector::executeSql(connection, sql = sql, progressBar = progressBar) sql <- "DROP TABLE IF EXISTS #analysis_ref; CREATE TABLE #analysis_ref( @@ -204,7 +206,7 @@ getDbDuringCovariateData <- function( targetDialect = DatabaseConnector::dbms(connection), tempEmulationSchema = tempEmulationSchema ) - DatabaseConnector::executeSql(connection, sql) + DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) # included covariates includedCovTable <- "" @@ -219,7 +221,8 @@ getDbDuringCovariateData <- function( tempTable = TRUE, data = data.frame(id = covariateSettings$includedCovariateIds), camelCaseToSnakeCase = TRUE, - tempEmulationSchema = tempEmulationSchema + tempEmulationSchema = tempEmulationSchema, + progressBar = progressBar ) } @@ -235,7 +238,8 @@ getDbDuringCovariateData <- function( tempTable = TRUE, data = data.frame(id = covariateSettings$includedCovariateConceptIds), camelCaseToSnakeCase = TRUE, - tempEmulationSchema = tempEmulationSchema + tempEmulationSchema = tempEmulationSchema, + progressBar = progressBar ) if (covariateSettings$addDescendantsToInclude) { @@ -262,7 +266,8 @@ getDbDuringCovariateData <- function( tempTable = TRUE, data = data.frame(id = covariateSettings$excludedCovariateConceptIds), camelCaseToSnakeCase = TRUE, - tempEmulationSchema = tempEmulationSchema + tempEmulationSchema = tempEmulationSchema, + progressBar = progressBar ) if (covariateSettings$addDescendantsToInclude) { @@ -325,7 +330,7 @@ getDbDuringCovariateData <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = TRUE + progressBar = progressBar ) time <- Sys.time() - start message(paste0("Execution took ", round(time, digits = 2), " ", units(time))) From 04168098908e35ba42723a1d05d64ae6c23e306a Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 11:51:13 -0400 Subject: [PATCH 05/35] Update DESCRIPTION trying to fix github actions --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index 8a260c4..30a4a3c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -28,6 +28,7 @@ Imports: rlang Suggests: devtools, + formatR, testthat, kableExtra, knitr, From 04b7762013f5eae47698a996fb0f4b83723a2e5d Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 12:26:13 -0400 Subject: [PATCH 06/35] updating vignettes updating vignettes to try and pass actions --- R/CustomCovariates.R | 2 +- vignettes/Specification.Rmd | 4 ++-- vignettes/UsingPackage.Rmd | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R/CustomCovariates.R b/R/CustomCovariates.R index d422338..ac7b146 100644 --- a/R/CustomCovariates.R +++ b/R/CustomCovariates.R @@ -171,7 +171,7 @@ getDbDuringCovariateData <- function( getDomainSettings <- utils::read.csv(system.file("csv/PrespecAnalyses.csv", package = "Characterization")) - # not showing progress like FE + # not showing progress progressBar <- FALSE # create Tables diff --git a/vignettes/Specification.Rmd b/vignettes/Specification.Rmd index c192764..6bfb559 100644 --- a/vignettes/Specification.Rmd +++ b/vignettes/Specification.Rmd @@ -13,8 +13,8 @@ header-includes: - \fancyfoot[CO,CE]{Characterization Package Version `r utils::packageVersion("Characterization")`} output: html_document: - number_sections: yes - toc: yes + number_sections: true + toc: true vignette: > %\VignetteIndexEntry{Specification} %\VignetteEngine{knitr::knitr} diff --git a/vignettes/UsingPackage.Rmd b/vignettes/UsingPackage.Rmd index df36d38..65d5feb 100644 --- a/vignettes/UsingPackage.Rmd +++ b/vignettes/UsingPackage.Rmd @@ -13,8 +13,8 @@ header-includes: - \renewcommand{\footrulewidth}{0.4pt} output: html_document: - number_sections: yes - toc: yes + number_sections: true + toc: true vignette: > %\VignetteIndexEntry{Using_Package} %\VignetteEngine{knitr::knitr} From bd093d2d80515a6af3c5906107f78364508c3a2c Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 12:29:42 -0400 Subject: [PATCH 07/35] Update R_CMD_check_Hades.yaml adding ubuntu latest as 20.04 depreciated --- .github/workflows/R_CMD_check_Hades.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml index fbe0e2b..7579757 100644 --- a/.github/workflows/R_CMD_check_Hades.yaml +++ b/.github/workflows/R_CMD_check_Hades.yaml @@ -22,7 +22,7 @@ jobs: config: - {os: windows-latest, r: 'release'} - {os: macOS-latest, r: 'release'} - - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + - {os: ubuntu-latest, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} env: GITHUB_PAT: ${{ secrets.GH_TOKEN }} From b95352850dbff01e509747983791f9bd760cff2d Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 12:55:51 -0400 Subject: [PATCH 08/35] preparing for 2.1.4 release preparing for 2.1.4 release --- DESCRIPTION | 4 ++-- NEWS.md | 6 ++++++ inst/settings/resultsDataModelSpecification.csv | 8 ++++---- inst/sql/sql_server/ResultTables.sql | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 30a4a3c..525b776 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 2.1.3 -Date: 2025-2-26 +Version: 2.1.4 +Date: 2025-5-07 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), person("Patrick", "Ryan", , "ryan@ohdsi.org", role = c("aut")), diff --git a/NEWS.md b/NEWS.md index eeadce2..ef8237a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +Characterization 2.1.4 +====================== +- fixed csv spec: made mean_exposure_time a float and specified that min_characterization_mean in covariate table must be non-null and is in the pk. +- added migration SQL to change mean_exposure_time to be a float +- changed Line 284 in AggregateCovariates.R to cast exposure_time summary values to bigint due to integer overflow in some dbms. + Characterization 2.1.3 ====================== - prepared for CRAN by adding examples, removing getwd(), replacing T/F with TRUE/FALSE and added example data inside package so no download required. diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 3dc0e59..28ee778 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -66,7 +66,7 @@ covariates,setting_id,varchar(30),Yes,Yes,No,No,The run identifier covariates,cohort_type,varchar(12),Yes,Yes,No,No,The cohort type covariates,target_cohort_id,int,Yes,Yes,No,No,The target cohort id covariates,outcome_cohort_id,int,Yes,Yes,No,No,The outcome cohort id -covariates,min_characterization_mean,float,No,Yes,No,No,Minimum fraction for feature extraction +covariates,min_characterization_mean,float,Yes,Yes,No,No,Minimum fraction for feature extraction covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id covariates,sum_value,int,Yes,No,No,No,The sum value covariates,average_value,float,No,No,No,No,The average value @@ -115,6 +115,6 @@ cohort_counts,min_prior_observation,int,No,No,No,No,Minimum time observed before cohort_counts,outcome_washout_days,int,No,No,No,No,Patients with outcome during washout are excluded cohort_counts,row_count,int,Yes,No,No,No,The number of rows in each cohort cohort_counts,person_count,int,Yes,No,No,No,The number of distinct people in each cohort -cohort_counts,min_exposure_time,bigint,No,No,No,No,Minimum exposure time across cohort -cohort_counts,mean_exposure_time,bigint,No,No,No,No,Mean exposure time across cohort -cohort_counts,max_exposure_time,bigint,No,No,No,No,Max exposure time across cohort +cohort_counts,min_exposure_time,bigint,No,No,No,No,Minimum exposure time across cohort in days +cohort_counts,mean_exposure_time,float,No,No,No,No,Mean exposure time across cohort in days +cohort_counts,max_exposure_time,bigint,No,No,No,No,Max exposure time across cohort in days diff --git a/inst/sql/sql_server/ResultTables.sql b/inst/sql/sql_server/ResultTables.sql index edb85e2..3b5acb2 100644 --- a/inst/sql/sql_server/ResultTables.sql +++ b/inst/sql/sql_server/ResultTables.sql @@ -94,7 +94,7 @@ CREATE TABLE @my_schema.@table_prefixcovariates ( covariate_id bigint NOT NULL, sum_value int NOT NULL, average_value float, - PRIMARY KEY (database_id, setting_id, target_cohort_id, outcome_cohort_id, cohort_type , covariate_id) + PRIMARY KEY (database_id, setting_id, target_cohort_id, outcome_cohort_id, cohort_type , covariate_id, min_characterization_mean) ); CREATE TABLE @my_schema.@table_prefixcovariates_continuous ( From c8c3eae425e17ea3880fc1a8c110e9956304ad27 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 15:58:33 -0400 Subject: [PATCH 09/35] minor bug and check warning fixes --- NEWS.md | 5 ++ R/CustomCovariates.R | 2 + R/Database.R | 36 ++++++++---- R/DechallengeRechallenge.R | 12 ++-- R/SaveLoad.R | 55 ++++++++++++++++++- R/TimeToEvent.R | 6 +- inst/sql/sql_server/ConceptCountsDuring.sql | 5 ++ inst/sql/sql_server/DomainConceptDuring.sql | 5 ++ .../sql_server/DomainConceptGroupDuring.sql | 5 ++ man/getDbDuringCovariateData.Rd | 2 + man/insertResultsToDatabase.Rd | 16 ++++-- tests/testthat/test-dechallengeRechallenge.R | 4 ++ tests/testthat/test-manualData.R | 2 + tests/testthat/test-runCharacterization.R | 5 +- tests/testthat/test-viewShiny.R | 3 +- 15 files changed, 134 insertions(+), 29 deletions(-) diff --git a/NEWS.md b/NEWS.md index ef8237a..0a5f6e3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,11 @@ Characterization 2.1.4 - fixed csv spec: made mean_exposure_time a float and specified that min_characterization_mean in covariate table must be non-null and is in the pk. - added migration SQL to change mean_exposure_time to be a float - changed Line 284 in AggregateCovariates.R to cast exposure_time summary values to bigint due to integer overflow in some dbms. +- added dummy sql code to prevent warnings about missing variables +- added code to save empty csv files when there are no rows as that way it is easier to see there are no results vs an error saving. +- removed progress bar from custom during features +- added option includedFiles in insertResultsToDatabase() where you can specify the csv files to upload to prevent warnings of missing csv files. +- made sure all connections are disconnected after use Characterization 2.1.3 ====================== diff --git a/R/CustomCovariates.R b/R/CustomCovariates.R index ac7b146..c317b21 100644 --- a/R/CustomCovariates.R +++ b/R/CustomCovariates.R @@ -147,6 +147,8 @@ createDuringCovariateSettings <- function( #' cohortTable = 'cohort' #' ) #' +#' DatabaseConnector::disconnect(connection) +#' #' @return #' A 'FeatureExtraction' covariateData object containing the during covariates based on user settings #' diff --git a/R/Database.R b/R/Database.R index 3e3640b..3588038 100644 --- a/R/Database.R +++ b/R/Database.R @@ -70,6 +70,7 @@ createSqliteDatabase <- function( #' @param resultsFolder The folder containing the csv results #' @param tablePrefix A prefix to append to the result tables for the characterization results #' @param csvTablePrefix The prefix added to the csv results - default is 'c_' +#' @param includedFiles Specify the csv files to upload or NULL to upload all in directory #' @family Database #' @return #' Returns the connection to the sqlite database @@ -79,13 +80,13 @@ createSqliteDatabase <- function( #' # generate results into resultsFolder #' conDet <- exampleOmopConnectionDetails() #' -#' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), +#' tteSet <- createTimeToEventSettings( +#' targetIds = c(1,2), #' outcomeIds = 3 -#' ) +#' ) #' #' cSet <- createCharacterizationSettings( -#' dechallengeRechallengeSettings = drSet +#' timeToEventSettings = tteSet #' ) #' #' runCharacterizationAnalyses( @@ -112,7 +113,8 @@ createSqliteDatabase <- function( #' insertResultsToDatabase( #' connectionDetails = charResultDbCD, #' schema = 'main', -#' resultsFolder = tempdir() +#' resultsFolder = tempdir(), +#' includedFiles = c('time_to_event') #' ) #' #' @@ -122,11 +124,16 @@ insertResultsToDatabase <- function( schema, resultsFolder, tablePrefix = "", - csvTablePrefix = "c_") { + csvTablePrefix = "c_", + includedFiles = NULL + ) { specLoc <- system.file("settings", "resultsDataModelSpecification.csv", package = "Characterization" ) specs <- utils::read.csv(specLoc) + if(!is.null(includedFiles)){ + specs <- specs[specs$table_name %in% includedFiles,] + } colnames(specs) <- SqlRender::snakeCaseToCamelCase(colnames(specs)) specs$tableName <- paste0(csvTablePrefix, specs$tableName) ResultModelManager::uploadResults( @@ -253,7 +260,8 @@ createCharacterizationTables <- function( ) DatabaseConnector::executeSql( connection = conn, - sql = sql + sql = sql, + progressBar = FALSE ) sql <- "DROP TABLE @my_schema.@table" @@ -269,7 +277,8 @@ createCharacterizationTables <- function( ) DatabaseConnector::executeSql( connection = conn, - sql = sql + sql = sql, + progressBar = FALSE ) } } @@ -288,10 +297,11 @@ createCharacterizationTables <- function( DatabaseConnector::executeSql( connection = conn, - sql = renderedSql + sql = renderedSql, + progressBar = FALSE ) - # add database migration here in the future + ## add database migration here in the future migrateDataModel( connectionDetails = connectionDetails, connection = conn, @@ -329,7 +339,11 @@ migrateDataModel <- function( connection <- DatabaseConnector::connect(connectionDetails = connectionDetails) on.exit(DatabaseConnector::disconnect(connection)) } - DatabaseConnector::executeSql(connection, updateVersionSql) + DatabaseConnector::executeSql( + connection = connection, + sql = updateVersionSql, + progressBar = FALSE + ) } diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index a662f10..5ba9f37 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -198,7 +198,8 @@ computeDechallengeRechallengeAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql + sql = sql, + progressBar = interactive() ) sql <- "select * from #challenge;" @@ -224,7 +225,8 @@ computeDechallengeRechallengeAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql, progressBar = FALSE, + sql = sql, + progressBar = FALSE, reportOverallTime = FALSE ) @@ -362,7 +364,8 @@ computeRechallengeFailCaseSeriesAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql + sql = sql, + progressBar = interactive() ) sql <- "select * from #fail_case_series;" @@ -388,7 +391,8 @@ computeRechallengeFailCaseSeriesAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql, progressBar = FALSE, + sql = sql, + progressBar = FALSE, reportOverallTime = FALSE ) diff --git a/R/SaveLoad.R b/R/SaveLoad.R index 44f9fd6..abb98c4 100644 --- a/R/SaveLoad.R +++ b/R/SaveLoad.R @@ -17,7 +17,15 @@ exportTimeToEventToCsv <- function( result, saveDirectory, - minCellCount = 0) { + minCellCount = 0 + ) { + + countN <- dplyr::pull( + dplyr::count(result$timeToEvent) + ) + + message("Writing ", countN, " rows to csv") + if (!dir.exists(saveDirectory)) { dir.create( path = saveDirectory, @@ -25,6 +33,19 @@ exportTimeToEventToCsv <- function( ) } + if(countN == 0){ + # save empty csv + dat <- as.data.frame(result$timeToEvent) + colnames(dat) <- SqlRender::camelCaseToSnakeCase( + string = colnames(dat) + ) + + readr::write_csv( + x = dat, + file = file.path(saveDirectory,"time_to_event.csv") + ) + } else{ + # save in batches Andromeda::batchApply( tbl = result$timeToEvent, fun = function(x) { @@ -59,6 +80,7 @@ exportTimeToEventToCsv <- function( ) } ) +} invisible( file.path( @@ -72,7 +94,9 @@ exportTimeToEventToCsv <- function( exportDechallengeRechallengeToCsv <- function( result, saveDirectory, - minCellCount = 0) { + minCellCount = 0 + ) { + countN <- dplyr::pull( dplyr::count(result$dechallengeRechallenge) ) @@ -82,6 +106,19 @@ exportDechallengeRechallengeToCsv <- function( dir.create(saveDirectory, recursive = TRUE) } + if(countN == 0){ + # save empty csv + dat <- as.data.frame(result$dechallengeRechallenge) + colnames(dat) <- SqlRender::camelCaseToSnakeCase( + string = colnames(dat) + ) + + readr::write_csv( + x = dat, + file = file.path(saveDirectory,"dechallenge_rechallenge.csv") + ) + } else{ + # export in batches Andromeda::batchApply( tbl = result$dechallengeRechallenge, fun = function(x) { @@ -174,6 +211,7 @@ exportDechallengeRechallengeToCsv <- function( ) } ) + } invisible( file.path( @@ -200,6 +238,18 @@ exportRechallengeFailCaseSeriesToCsv <- function( message("Writing ", countN, " rows to csv") + if(countN == 0){ + # save empty csv + dat <- as.data.frame(result$rechallengeFailCaseSeries) + colnames(dat) <- SqlRender::camelCaseToSnakeCase( + string = colnames(dat) + ) + readr::write_csv( + x = dat, + file = file.path(saveDirectory,"rechallenge_fail_case_series.csv") + ) + } else{ + # save in batches Andromeda::batchApply( tbl = result$rechallengeFailCaseSeries, fun = function(x) { @@ -229,6 +279,7 @@ exportRechallengeFailCaseSeriesToCsv <- function( ) } ) + } invisible( file.path( diff --git a/R/TimeToEvent.R b/R/TimeToEvent.R index 71c6969..2dee125 100644 --- a/R/TimeToEvent.R +++ b/R/TimeToEvent.R @@ -191,7 +191,8 @@ computeTimeToEventAnalyses <- function( DatabaseConnector::executeSql( connection = connection, - sql = sql + sql = sql, + progressBar = interactive() ) sql <- "select * from #two_tte_summary;" @@ -218,7 +219,8 @@ computeTimeToEventAnalyses <- function( DatabaseConnector::executeSql( connection = connection, - sql = sql, progressBar = FALSE, + sql = sql, + progressBar = FALSE, reportOverallTime = FALSE ) diff --git a/inst/sql/sql_server/ConceptCountsDuring.sql b/inst/sql/sql_server/ConceptCountsDuring.sql index 71b6dce..6c43b26 100644 --- a/inst/sql/sql_server/ConceptCountsDuring.sql +++ b/inst/sql/sql_server/ConceptCountsDuring.sql @@ -1,3 +1,8 @@ +-- adding line below to prevent warnings +IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL + DROP TABLE #fake; +SELECT "@row_id_field" as cname into #fake; + -- Feature construction {@aggregated} ? { IF OBJECT_ID('tempdb..#concept_count_data', 'U') IS NOT NULL diff --git a/inst/sql/sql_server/DomainConceptDuring.sql b/inst/sql/sql_server/DomainConceptDuring.sql index 03aeec3..603f5fb 100644 --- a/inst/sql/sql_server/DomainConceptDuring.sql +++ b/inst/sql/sql_server/DomainConceptDuring.sql @@ -1,3 +1,8 @@ +-- adding lines below to prevent warning +IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL + DROP TABLE #fake; +SELECT "@domain_end_date" as cname into #fake; + -- Feature construction SELECT CAST(@domain_concept_id AS BIGINT) * 1000 + @analysis_id AS covariate_id, diff --git a/inst/sql/sql_server/DomainConceptGroupDuring.sql b/inst/sql/sql_server/DomainConceptGroupDuring.sql index 68ebe73..6ec2efa 100644 --- a/inst/sql/sql_server/DomainConceptGroupDuring.sql +++ b/inst/sql/sql_server/DomainConceptGroupDuring.sql @@ -1,3 +1,8 @@ +-- add dummy code with all imputs to stop annoying warnings +IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL + DROP TABLE #fake; +SELECT "@domain_end_date" as cname into #fake; + IF OBJECT_ID('tempdb..#groups', 'U') IS NOT NULL DROP TABLE #groups; diff --git a/man/getDbDuringCovariateData.Rd b/man/getDbDuringCovariateData.Rd index 2f34740..1c6250e 100644 --- a/man/getDbDuringCovariateData.Rd +++ b/man/getDbDuringCovariateData.Rd @@ -75,6 +75,8 @@ duringData <- getDbDuringCovariateData( cohortTable = 'cohort' ) +DatabaseConnector::disconnect(connection) + } \seealso{ Other CovariateSetting: diff --git a/man/insertResultsToDatabase.Rd b/man/insertResultsToDatabase.Rd index 66e477e..e13d569 100644 --- a/man/insertResultsToDatabase.Rd +++ b/man/insertResultsToDatabase.Rd @@ -9,7 +9,8 @@ insertResultsToDatabase( schema, resultsFolder, tablePrefix = "", - csvTablePrefix = "c_" + csvTablePrefix = "c_", + includedFiles = NULL ) } \arguments{ @@ -22,6 +23,8 @@ insertResultsToDatabase( \item{tablePrefix}{A prefix to append to the result tables for the characterization results} \item{csvTablePrefix}{The prefix added to the csv results - default is 'c_'} + +\item{includedFiles}{Specify the csv files to upload or NULL to upload all in directory} } \value{ Returns the connection to the sqlite database @@ -37,13 +40,13 @@ Calls ResultModelManager uploadResults function to upload the csv files # generate results into resultsFolder conDet <- exampleOmopConnectionDetails() -drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), +tteSet <- createTimeToEventSettings( +targetIds = c(1,2), outcomeIds = 3 -) + ) cSet <- createCharacterizationSettings( - dechallengeRechallengeSettings = drSet + timeToEventSettings = tteSet ) runCharacterizationAnalyses( @@ -70,7 +73,8 @@ createCharacterizationTables( insertResultsToDatabase( connectionDetails = charResultDbCD, schema = 'main', - resultsFolder = tempdir() + resultsFolder = tempdir(), + includedFiles = c('time_to_event') ) diff --git a/tests/testthat/test-dechallengeRechallenge.R b/tests/testthat/test-dechallengeRechallenge.R index cd01401..89d303e 100644 --- a/tests/testthat/test-dechallengeRechallenge.R +++ b/tests/testthat/test-dechallengeRechallenge.R @@ -122,6 +122,9 @@ test_that("computeDechallengeRechallengeAnalyses", { dropTableIfExists = T, camelCaseToSnakeCase = F ) + + DatabaseConnector::disconnect(con) + res <- createDechallengeRechallengeSettings( targetIds = 1, outcomeIds = 2, @@ -199,6 +202,7 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { dropTableIfExists = T, camelCaseToSnakeCase = F ) + DatabaseConnector::disconnect(con) res <- createDechallengeRechallengeSettings( targetIds = 1, diff --git a/tests/testthat/test-manualData.R b/tests/testthat/test-manualData.R index a8728d8..c1466af 100644 --- a/tests/testthat/test-manualData.R +++ b/tests/testthat/test-manualData.R @@ -15,6 +15,7 @@ test_that("manual data runCharacterizationAnalyses", { server = manualData ) con <- DatabaseConnector::connect(connectionDetails = connectionDetails) + on.exit(DatabaseConnector::disconnect(con)) schema <- "main" # add persons - aggregate covs (age) @@ -312,6 +313,7 @@ test_that("manual data checking exclude count works", { server = manualData2 ) con <- DatabaseConnector::connect(connectionDetails = connectionDetails) + on.exit(DatabaseConnector::disconnect(con)) schema <- "main" # add persons - aggregate covs (age) diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index 3e45925..ff3ccea 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -157,11 +157,10 @@ test_that("runCharacterizationAnalyses", { file.exists(file.path(tempFolder, "result", "c_covariates_continuous.csv")) ) - # no results for dechal due to Eunomia - how to test? - testthat::expect_false( + testthat::expect_true( file.exists(file.path(tempFolder, "result", "c_dechallenge_rechallenge.csv")) ) - # testthat::expect_true( + #testthat::expect_true( # file.exists(file.path(tempFolder, "result", "rechallenge_fail_case_series.csv")) # ) testthat::expect_true( diff --git a/tests/testthat/test-viewShiny.R b/tests/testthat/test-viewShiny.R index d519ad3..c7f8cd2 100644 --- a/tests/testthat/test-viewShiny.R +++ b/tests/testthat/test-viewShiny.R @@ -95,7 +95,7 @@ test_that("prepareCharacterizationShiny works", { minCharacterizationMean = 0.01 ) - settings <- Characterization:::prepareCharacterizationShiny( + settings <- prepareCharacterizationShiny( resultFolder = file.path(resultLocation, "result"), cohortDefinitionSet = NULL, sqliteLocation = file.path(resultLocation, "sqliteCharacterization", "sqlite.sqlite") @@ -114,6 +114,7 @@ test_that("prepareCharacterizationShiny works", { ) ) conTest <- DatabaseConnector::connect(connectionDetailsTest) + on.exit(DatabaseConnector::disconnect(conTest)) tables <- tolower( DatabaseConnector::getTableNames( connection = conTest, From bd14fbd47e20d2e4a07487ab1b4da91879a8eae6 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 16:19:30 -0400 Subject: [PATCH 10/35] putting examples in own directories putting examples in own directories --- R/Database.R | 4 ++-- R/RunCharacterization.R | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/Database.R b/R/Database.R index 3588038..3e99a07 100644 --- a/R/Database.R +++ b/R/Database.R @@ -97,7 +97,7 @@ createSqliteDatabase <- function( #' outcomeTable = 'cohort', #' cdmDatabaseSchema = 'main', #' characterizationSettings = cSet, -#' outputDirectory = tempdir() +#' outputDirectory = file.path(tempdir(),'database') #' ) #' #' # create sqlite database @@ -113,7 +113,7 @@ createSqliteDatabase <- function( #' insertResultsToDatabase( #' connectionDetails = charResultDbCD, #' schema = 'main', -#' resultsFolder = tempdir(), +#' resultsFolder = file.path(tempdir(),'database'), #' includedFiles = c('time_to_event') #' ) #' diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 0f6f655..20c92b0 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -204,7 +204,7 @@ loadCharacterizationSettings <- function( #' outcomeTable = 'cohort', #' cdmDatabaseSchema = 'main', #' characterizationSettings = cSet, -#' outputDirectory = tempdir() +#' outputDirectory = file.path(tempdir(),'runChar') #' ) #' #' @export From 8cff3c69019e99ee945c997af2539839897f9b13 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 7 May 2025 16:51:29 -0400 Subject: [PATCH 11/35] fixing example doc fixing example doc --- man/insertResultsToDatabase.Rd | 4 ++-- man/runCharacterizationAnalyses.Rd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/man/insertResultsToDatabase.Rd b/man/insertResultsToDatabase.Rd index e13d569..6d055ff 100644 --- a/man/insertResultsToDatabase.Rd +++ b/man/insertResultsToDatabase.Rd @@ -57,7 +57,7 @@ runCharacterizationAnalyses( outcomeTable = 'cohort', cdmDatabaseSchema = 'main', characterizationSettings = cSet, - outputDirectory = tempdir() + outputDirectory = file.path(tempdir(),'database') ) # create sqlite database @@ -73,7 +73,7 @@ createCharacterizationTables( insertResultsToDatabase( connectionDetails = charResultDbCD, schema = 'main', - resultsFolder = tempdir(), + resultsFolder = file.path(tempdir(),'database'), includedFiles = c('time_to_event') ) diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index 8b8ebf5..08bbd07 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -97,7 +97,7 @@ runCharacterizationAnalyses( outcomeTable = 'cohort', cdmDatabaseSchema = 'main', characterizationSettings = cSet, - outputDirectory = tempdir() + outputDirectory = file.path(tempdir(),'runChar') ) } From 1e6dc250c03128e83ef649d8598be379599d1a3b Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 11:08:37 -0400 Subject: [PATCH 12/35] changing examples --- DESCRIPTION | 4 ++-- R/RunCharacterization.R | 4 ++-- R/ViewShiny.R | 26 +++++++++++++++++++------- man/runCharacterizationAnalyses.Rd | 4 ++-- man/viewCharacterization.Rd | 4 ++-- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 525b776..fd4ea49 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -16,9 +16,9 @@ BugReports: https://github.com/OHDSI/Characterization/issues Depends: R (>= 4.0.0) Imports: - Andromeda, + Andromeda (>= 1.0.0), DatabaseConnector (>= 6.3.1), - FeatureExtraction (>= 3.6.0), + FeatureExtraction (>= 3.8.0), SqlRender (>= 1.9.0), ParallelLogger (>= 3.0.0), ResultModelManager, diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 20c92b0..f1288c0 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -187,13 +187,13 @@ loadCharacterizationSettings <- function( #' #' conDet <- exampleOmopConnectionDetails() #' -#' drSet <- createDechallengeRechallengeSettings( +#' tteSet <- createTimeToEventSettings( #' targetIds = c(1,2), #' outcomeIds = 3 #' ) #' #' cSet <- createCharacterizationSettings( -#' dechallengeRechallengeSettings = drSet +#' timeToEventSettings = tteSet #' ) #' #' runCharacterizationAnalyses( diff --git a/R/ViewShiny.R b/R/ViewShiny.R index ed6d772..9c57250 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -15,13 +15,13 @@ #' #' conDet <- exampleOmopConnectionDetails() #' -#' drSet <- createDechallengeRechallengeSettings( +#' tteSet <- createTimeToEventSettings( #' targetIds = c(1,2), #' outcomeIds = 3 #' ) #' #' cSet <- createCharacterizationSettings( -#' dechallengeRechallengeSettings = drSet +#' timeToEventSettings = tteSet #' ) #' #' runCharacterizationAnalyses( @@ -48,12 +48,18 @@ viewCharacterization <- function( # check there are csv files in resultFolder if(length(dir(resultFolder, pattern = '.csv')) > 0 ){ - databaseSettings <- prepareCharacterizationShiny( - resultFolder = resultFolder, - cohortDefinitionSet = cohortDefinitionSet - ) + databaseSettings <- prepareCharacterizationShiny( + resultFolder = resultFolder, + cohortDefinitionSet = cohortDefinitionSet + ) + + if(length(databaseSettings) == 0){ + message('No actual results to view via shiny') + return(FALSE) + } else{ + viewChars(databaseSettings) + } - viewChars(databaseSettings) } else{ message('No csv results to view via shiny') return(FALSE) @@ -116,6 +122,12 @@ prepareCharacterizationShiny <- function( ) ) + + if(length(cohortIds) == 0){ + # if no cohortids then no results to view + return(invisible(list())) + } + DatabaseConnector::insertTable( connection = con, databaseSchema = "main", diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index 08bbd07..fb485b5 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -80,13 +80,13 @@ specified saveDirectory conDet <- exampleOmopConnectionDetails() -drSet <- createDechallengeRechallengeSettings( +tteSet <- createTimeToEventSettings( targetIds = c(1,2), outcomeIds = 3 ) cSet <- createCharacterizationSettings( - dechallengeRechallengeSettings = drSet + timeToEventSettings = tteSet ) runCharacterizationAnalyses( diff --git a/man/viewCharacterization.Rd b/man/viewCharacterization.Rd index 152a7e8..ec63824 100644 --- a/man/viewCharacterization.Rd +++ b/man/viewCharacterization.Rd @@ -24,13 +24,13 @@ Input is the output of ... conDet <- exampleOmopConnectionDetails() -drSet <- createDechallengeRechallengeSettings( +tteSet <- createTimeToEventSettings( targetIds = c(1,2), outcomeIds = 3 ) cSet <- createCharacterizationSettings( - dechallengeRechallengeSettings = drSet + timeToEventSettings = tteSet ) runCharacterizationAnalyses( From 3ef0b56808484d4cde6444cda01c522f875bb79e Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 11:19:26 -0400 Subject: [PATCH 13/35] Update ViewShiny.R --- R/ViewShiny.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 9c57250..ad1cf57 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -35,9 +35,12 @@ #' outputDirectory = file.path(tempdir(),'view') #' ) #' +#' # interactive shiny app +#' \dontrun{ #' viewCharacterization( #' resultFolder = file.path(tempdir(),'view') #' ) +#' } #' #' #' @export From 48885d4132943eb4d23cf26741fe407964a66021 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 11:23:41 -0400 Subject: [PATCH 14/35] update example doc and FE --- DESCRIPTION | 2 +- man/viewCharacterization.Rd | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index fd4ea49..8388689 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,7 @@ Depends: Imports: Andromeda (>= 1.0.0), DatabaseConnector (>= 6.3.1), - FeatureExtraction (>= 3.8.0), + FeatureExtraction (>= 3.9.0), SqlRender (>= 1.9.0), ParallelLogger (>= 3.0.0), ResultModelManager, diff --git a/man/viewCharacterization.Rd b/man/viewCharacterization.Rd index ec63824..10d7b78 100644 --- a/man/viewCharacterization.Rd +++ b/man/viewCharacterization.Rd @@ -44,9 +44,12 @@ runCharacterizationAnalyses( outputDirectory = file.path(tempdir(),'view') ) +# interactive shiny app +\dontrun{ viewCharacterization( resultFolder = file.path(tempdir(),'view') ) +} } From 6d44cee760ed89aecd54e2e7bb3794ac3a4ffde3 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 13:56:48 -0400 Subject: [PATCH 15/35] Update DESCRIPTION using the new FE that fixes the Andromeda issue --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 8388689..ef0f6ff 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,7 @@ Depends: Imports: Andromeda (>= 1.0.0), DatabaseConnector (>= 6.3.1), - FeatureExtraction (>= 3.9.0), + FeatureExtraction (>= 3.10.0), SqlRender (>= 1.9.0), ParallelLogger (>= 3.0.0), ResultModelManager, From 07c543cf013b6a5556c2d60e491e04cf0cdbda99 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 14:20:19 -0400 Subject: [PATCH 16/35] adding failed case series csv check adding failed case series csv test --- NEWS.md | 1 - tests/testthat/test-runCharacterization.R | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0a5f6e3..4d44ca7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,6 @@ Characterization 2.1.4 ====================== - fixed csv spec: made mean_exposure_time a float and specified that min_characterization_mean in covariate table must be non-null and is in the pk. -- added migration SQL to change mean_exposure_time to be a float - changed Line 284 in AggregateCovariates.R to cast exposure_time summary values to bigint due to integer overflow in some dbms. - added dummy sql code to prevent warnings about missing variables - added code to save empty csv files when there are no rows as that way it is easier to see there are no results vs an error saving. diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index ff3ccea..3663bd6 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -160,9 +160,9 @@ test_that("runCharacterizationAnalyses", { testthat::expect_true( file.exists(file.path(tempFolder, "result", "c_dechallenge_rechallenge.csv")) ) - #testthat::expect_true( - # file.exists(file.path(tempFolder, "result", "rechallenge_fail_case_series.csv")) - # ) + testthat::expect_true( + file.exists(file.path(tempFolder, "result", "c_rechallenge_fail_case_series.csv")) + ) testthat::expect_true( file.exists(file.path(tempFolder, "result", "c_time_to_event.csv")) ) From cc70d655c6010750fd8f536c173dce7f355605d7 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 8 May 2025 14:41:47 -0400 Subject: [PATCH 17/35] changing version ready for release changing version ready for release --- DESCRIPTION | 2 +- NEWS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ef0f6ff..0d26568 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 2.1.4 +Version: 2.2.0 Date: 2025-5-07 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), diff --git a/NEWS.md b/NEWS.md index 4d44ca7..96f8e79 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -Characterization 2.1.4 +Characterization 2.2.0 ====================== - fixed csv spec: made mean_exposure_time a float and specified that min_characterization_mean in covariate table must be non-null and is in the pk. - changed Line 284 in AggregateCovariates.R to cast exposure_time summary values to bigint due to integer overflow in some dbms. From 0a8f231a1b749e5feba301536bc434fd85276e38 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 26 Jun 2025 15:58:15 -0400 Subject: [PATCH 18/35] batch aggregate csv - changing aggregate csv to be done in batches - adding example results into testdata for testing the aggregate csv function --- R/RunCharacterization.R | 124 ++++++++++- tests/testthat/test-runCharacterization.R | 61 ++++++ .../cac_1_0_0_365_365/analysis_ref.csv | 11 + .../cac_1_0_0_365_365/cohort_counts.csv | 4 + .../cac_1_0_0_365_365/cohort_details.csv | 16 ++ .../cac_1_0_0_365_365/covariate_ref.csv | 93 +++++++++ .../cac_1_0_0_365_365/covariates.csv | 192 ++++++++++++++++++ .../covariates_continuous.csv | 10 + .../execution/cac_1_0_0_365_365/settings.csv | 40 ++++ .../testthat/testdata/execution/completed.csv | 7 + .../dr_1/dechallenge_rechallenge.csv | 2 + .../testthat/testdata/execution/execution.csv | 7 + tests/testthat/testdata/execution/log.txt | 6 + .../rfcs_1/rechallenge_fail_case_series.csv | 2 + .../testthat/testdata/execution/settings.rds | Bin 0 -> 786 bytes .../execution/tac_1_0/analysis_ref.csv | 4 + .../execution/tac_1_0/cohort_counts.csv | 9 + .../execution/tac_1_0/cohort_details.csv | 9 + .../execution/tac_1_0/covariate_ref.csv | 4 + .../testdata/execution/tac_1_0/covariates.csv | 9 + .../tac_1_0/covariates_continuous.csv | 5 + .../testdata/execution/tac_1_0/settings.csv | 21 ++ .../execution/tte_1/time_to_event.csv | 103 ++++++++++ 23 files changed, 731 insertions(+), 8 deletions(-) create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/analysis_ref.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_counts.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_details.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/covariate_ref.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates_continuous.csv create mode 100644 tests/testthat/testdata/execution/cac_1_0_0_365_365/settings.csv create mode 100644 tests/testthat/testdata/execution/completed.csv create mode 100644 tests/testthat/testdata/execution/dr_1/dechallenge_rechallenge.csv create mode 100644 tests/testthat/testdata/execution/execution.csv create mode 100644 tests/testthat/testdata/execution/log.txt create mode 100644 tests/testthat/testdata/execution/rfcs_1/rechallenge_fail_case_series.csv create mode 100644 tests/testthat/testdata/execution/settings.rds create mode 100644 tests/testthat/testdata/execution/tac_1_0/analysis_ref.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/cohort_counts.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/cohort_details.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/covariate_ref.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/covariates.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/covariates_continuous.csv create mode 100644 tests/testthat/testdata/execution/tac_1_0/settings.csv create mode 100644 tests/testthat/testdata/execution/tte_1/time_to_event.csv diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index f1288c0..8346ddb 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -361,7 +361,7 @@ runCharacterizationAnalyses <- function( ) # code to export all csvs into one file - aggregateCsvs( + aggregateCsvsBatch( outputFolder = outputDirectory, executionPath = executionPath, executionFolders = jobs$executionFolder, @@ -459,13 +459,6 @@ createJobs <- function( ) ) - # data.frame( - # functionName, - # settings # json, - # executionFolder, - # jobId - # ) - return(jobDf) } @@ -563,3 +556,118 @@ aggregateCsvs <- function( } } } + + +aggregateCsvsBatch <- function( + executionPath, + outputFolder, + executionFolders, # needed? + csvFilePrefix, + batchSize = 100000 + ) { + tables <- c( + "cohort_details.csv", "settings.csv", "covariates.csv", + "covariates_continuous.csv", "covariate_ref.csv", + "analysis_ref.csv", "cohort_counts.csv", + "time_to_event.csv", + "rechallenge_fail_case_series.csv", "dechallenge_rechallenge.csv" + ) + + colTypes <- c( + 'ciicc','ciiiicciiccc', 'didciiccd', + 'didddddddddciicc', 'dciicicc', + 'icciicccc', 'iiciicciicddddd', + '????????', + '?????????????????', '????????????????????' + ) + + # this makes sure results are recreated + firstTracker <- data.frame( + table = tables, + first = rep(TRUE, length(tables)) + ) + + csvTrackerFile <- file.path(outputFolder,'tracker.rds') + tracker <- list( + analysisRefTracker = c(), + covariateRefTracker = c(), + settingsTracker = c() + ) + saveRDS(tracker, csvTrackerFile) + + # create outputFolder + + folderNames <- dir(executionPath) + + # for each folder load covariates, covariates_continuous, + # covariate_ref and analysis_ref + for (folderName in folderNames) { + for (csvType in tables) { + loadPath <- file.path(executionPath, folderName, csvType) + savePath <- file.path(outputFolder, paste0(csvFilePrefix, csvType)) + if (file.exists(loadPath)) { + + firstTrackerCurrent <- firstTracker$first[firstTracker$table == csvType] + append <- file.exists(savePath) + + # code to save results in batches + processCsv <- function(x, pos){ + + tracker <- readRDS(csvTrackerFile) + + if (csvType == "analysis_ref.csv") { + x <- x %>% + dplyr::mutate( + unique_id = paste0(.data$setting_id, "-", .data$analysis_id) + ) %>% + dplyr::filter( # need to filter analysis_id and setting_id + !.data$unique_id %in% tracker$analysisRefTracker + ) %>% + dplyr::select(-"unique_id") + + tracker$analysisRefTracker <- unique(c(tracker$analysisRefTracker, paste0(x$setting_id, "-", x$analysis_id))) + } + if (csvType == "covariate_ref.csv") { # this could be problematic as may have differnet covariate_ids + x <- x %>% + dplyr::mutate( + unique_id = paste0(.data$setting_id, "-", .data$covariate_id) + ) %>% + dplyr::filter( # need to filter covariate_id and setting_id + !.data$unique_id %in% tracker$covariateRefTracker + ) %>% + dplyr::select(-"unique_id") + + tracker$covariateRefTracker <- unique(c(tracker$covariateRefTracker, paste0(x$setting_id, "-", x$covariate_id))) + } + if (csvType == "settings.csv") { + x <- x %>% + dplyr::filter( + !.data$setting_id %in% tracker$settingsTracker + ) + tracker$settingsTracker <- c(tracker$settingsTracker, unique(x$setting_id)) + } + + readr::write_csv( + x = x, + file = savePath, quote = "all", + #append = append & !firstTrackerCurrent & pos == 1 + append = append | pos != 1 + ) + + saveRDS(tracker,csvTrackerFile) + + } + + readr::read_csv_chunked( + file = loadPath, + callback = readr::SideEffectChunkCallback$new(processCsv), + chunk_size = batchSize, + col_types = colTypes[csvType == tables], + show_col_types = FALSE + ) + + firstTracker$first[firstTracker$table == csvType] <- FALSE + } + } + } +} diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index 3663bd6..88e533a 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -427,3 +427,64 @@ test_that("min cell count works", { testthat::expect_true(sum(is.na(res$min_value)) == length(res$min_value)) testthat::expect_true(sum(is.na(res$max_value)) == length(res$max_value)) }) + + + +test_that("checking the batch csv aggregation", { + + tempFolder <- tempfile("Characterization") + on.exit(unlink(tempFolder, recursive = TRUE), add = TRUE) + + executionPath <- testthat::test_path("testdata", "execution") + + if(!dir.exists(file.path(tempFolder,'aggCvs'))){ + dir.create(file.path(tempFolder,'aggCvs'), recursive = T) + } + if(!dir.exists(file.path(tempFolder,'aggCvs2'))){ + dir.create(file.path(tempFolder,'aggCvs2'), recursive = T) + } + if(!dir.exists(file.path(tempFolder,'aggCvs3'))){ + dir.create(file.path(tempFolder,'aggCvs3'), recursive = T) + } + +# checking the batch csv aggregation +Characterization:::aggregateCsvs( + executionPath = executionPath, + outputFolder = file.path(tempFolder,'aggCvs'), + csvFilePrefix = '' +) + +Characterization:::aggregateCsvsBatch( + executionPath = executionPath, + outputFolder = file.path(tempFolder,'aggCvs2'), + csvFilePrefix = '' +) + +Characterization:::aggregateCsvsBatch( + executionPath = executionPath, + outputFolder = file.path(tempFolder,'aggCvs3'), + csvFilePrefix = '', + batchSize = 1 +) + +#check files are the same using default batch +files <- dir(file.path(tempFolder,'aggCvs'), pattern = 'csv') +for(i in 1:length(files)){ + d1 <- readr::read_csv(file.path(tempFolder,'aggCvs', files[i]), + show_col_types = F) + d2 <- readr::read_csv(file.path(tempFolder,'aggCvs2', files[i]), + show_col_types = F) + testthat::expect_true(all.equal(d1,d2)) +} + +# when batchsize is 1 +files <- dir(file.path(tempFolder,'aggCvs'), pattern = 'csv') +for(i in 1:length(files)){ + d1 <- readr::read_csv(file.path(tempFolder,'aggCvs', files[i]), + show_col_types = F) + d2 <- readr::read_csv(file.path(tempFolder,'aggCvs3', files[i]), + show_col_types = F) + testthat::expect_true(all.equal(d1,d2)) +} + +}) diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/analysis_ref.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/analysis_ref.csv new file mode 100644 index 0000000..644a1a4 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/analysis_ref.csv @@ -0,0 +1,11 @@ +analysis_id,analysis_name,domain_id,start_day,end_day,is_binary,missing_means_zero,setting_id,database_id +1,DemographicsGender,Demographics,NA,NA,Y,NA,20250626154219589895388166,1 +4,DemographicsRace,Demographics,NA,NA,Y,NA,20250626154219589895388166,1 +2,DemographicsAge,Demographics,NA,NA,N,Y,20250626154219589895388166,1 +218,ConditionGroupEraDuring,Condition,NA,NA,Y,NA,20250626154219589895388166,1 +418,DrugGroupEraDuring,Drug,NA,NA,Y,NA,20250626154219589895388166,1 +505,ProcedureOccurrenceDuring,Procedure,NA,NA,Y,NA,20250626154219589895388166,1 +605,DeviceExposureDuring,Device,NA,NA,Y,NA,20250626154219589895388166,1 +713,MeasurementDuring,Measurement,NA,NA,Y,NA,20250626154219589895388166,1 +805,ObservationDuring,Observation,NA,NA,Y,NA,20250626154219589895388166,1 +927,VisitConceptCountDuring,Visit,NA,NA,N,Y,20250626154219589895388166,1 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_counts.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_counts.csv new file mode 100644 index 0000000..6cdfd74 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_counts.csv @@ -0,0 +1,4 @@ +target_cohort_id,outcome_cohort_id,cohort_type,risk_window_start,risk_window_end,start_anchor,end_anchor,min_prior_observation,outcome_washout_days,database_id,row_count,person_count,min_exposure_time,mean_exposure_time,max_exposure_time +1,3,Cases,1,365,cohort start,cohort start,0,0,1,115,115,0,0,0 +2,3,Cases,1,365,cohort start,cohort start,0,0,1, 35, 35,0,0,0 +4,3,Cases,1,365,cohort start,cohort start,0,0,1,150,150,0,0,0 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_details.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_details.csv new file mode 100644 index 0000000..fb27ae8 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/cohort_details.csv @@ -0,0 +1,16 @@ +setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +20250626154219589895388166,1,3,Cases,1 +20250626154219589895388166,2,3,Cases,1 +20250626154219589895388166,4,3,Cases,1 +20250626154219589895388166,1,3,CasesBefore,1 +20250626154219589895388166,2,3,CasesBefore,1 +20250626154219589895388166,4,3,CasesBefore,1 +20250626154219589895388166,1,3,CasesAfter,1 +20250626154219589895388166,2,3,CasesAfter,1 +20250626154219589895388166,4,3,CasesAfter,1 +20250626154219589895388166,1,3,CasesBetween,1 +20250626154219589895388166,2,3,CasesBetween,1 +20250626154219589895388166,4,3,CasesBetween,1 +20250626154219589895388166,1,3,Exclude,1 +20250626154219589895388166,2,3,Exclude,1 +20250626154219589895388166,4,3,Exclude,1 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariate_ref.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariate_ref.csv new file mode 100644 index 0000000..10d2b53 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariate_ref.csv @@ -0,0 +1,93 @@ +covariate_id,covariate_name,analysis_id,concept_id,value_as_concept_id,collisions,setting_id,database_id +8507001,gender = MALE,1,8507,NA,NA,20250626154219589895388166,1 +8532001,gender = FEMALE,1,8532,NA,NA,20250626154219589895388166,1 + 1002,age in years,2, 0,NA,NA,20250626154219589895388166,1 + 30753218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Esophagitis,218, 30753,NA,NA,20250626154219589895388166,1 + 78272218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sprain of wrist,218, 78272,NA,NA,20250626154219589895388166,1 + 80180218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Osteoarthritis,218, 80180,NA,NA,20250626154219589895388166,1 + 81893218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Ulcerative colitis,218, 81893,NA,NA,20250626154219589895388166,1 + 134438218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Contact dermatitis,218, 134438,NA,NA,20250626154219589895388166,1 + 195588218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Cystitis,218, 195588,NA,NA,20250626154219589895388166,1 + 198199218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Pyelonephritis,218, 198199,NA,NA,20250626154219589895388166,1 + 260139218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute bronchitis,218, 260139,NA,NA,20250626154219589895388166,1 + 378001218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Concussion with no loss of consciousness,218, 378001,NA,NA,20250626154219589895388166,1 + 439777218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Anemia,218, 439777,NA,NA,20250626154219589895388166,1 + 4001336218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Concussion injury of brain,218, 4001336,NA,NA,20250626154219589895388166,1 + 4112343218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute viral pharyngitis,218, 4112343,NA,NA,20250626154219589895388166,1 + 4113008218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of hand,218, 4113008,NA,NA,20250626154219589895388166,1 + 4116491218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Escherichia coli urinary tract infection,218, 4116491,NA,NA,20250626154219589895388166,1 + 4132546218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Traumatic brain injury,218, 4132546,NA,NA,20250626154219589895388166,1 + 4152936218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of thigh,218, 4152936,NA,NA,20250626154219589895388166,1 + 4155034218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of forearm,218, 4155034,NA,NA,20250626154219589895388166,1 + 4266809218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Diverticular disease,218, 4266809,NA,NA,20250626154219589895388166,1 + 4283893218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sinusitis,218, 4283893,NA,NA,20250626154219589895388166,1 + 4285898218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Polyp of colon,218, 4285898,NA,NA,20250626154219589895388166,1 + 4310024218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Angiodysplasia of stomach,218, 4310024,NA,NA,20250626154219589895388166,1 +40481087218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Viral sinusitis,218,40481087,NA,NA,20250626154219589895388166,1 + 28060218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Streptococcal sore throat,218, 28060,NA,NA,20250626154219589895388166,1 + 81151218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sprain of ankle,218, 81151,NA,NA,20250626154219589895388166,1 + 257012218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Chronic sinusitis,218, 257012,NA,NA,20250626154219589895388166,1 + 381316218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Cerebrovascular accident,218, 381316,NA,NA,20250626154219589895388166,1 + 4294548218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute bacterial sinusitis,218, 4294548,NA,NA,20250626154219589895388166,1 + 258780218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Emphysematous bronchitis,218, 258780,NA,NA,20250626154219589895388166,1 + 4094814218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Bullet wound,218, 4094814,NA,NA,20250626154219589895388166,1 + 4296205218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Second degree burn,218, 4296205,NA,NA,20250626154219589895388166,1 + 192671218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Gastrointestinal hemorrhage,218, 192671,NA,NA,20250626154219589895388166,1 + 4142905218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Fracture of rib,218, 4142905,NA,NA,20250626154219589895388166,1 + 738818418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Doxylamine,418, 738818,NA,NA,20250626154219589895388166,1 + 920293418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Nitrofurantoin,418, 920293,NA,NA,20250626154219589895388166,1 + 933724418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Phenazopyridine,418, 933724,NA,NA,20250626154219589895388166,1 + 975125418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Hydrocortisone,418, 975125,NA,NA,20250626154219589895388166,1 + 1118084418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: celecoxib,418, 1118084,NA,NA,20250626154219589895388166,1 + 1119510418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Dextromethorphan,418, 1119510,NA,NA,20250626154219589895388166,1 + 1125315418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Acetaminophen,418, 1125315,NA,NA,20250626154219589895388166,1 + 1150770418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Astemizole,418, 1150770,NA,NA,20250626154219589895388166,1 + 1177480418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Ibuprofen,418, 1177480,NA,NA,20250626154219589895388166,1 + 1713332418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Amoxicillin,418, 1713332,NA,NA,20250626154219589895388166,1 + 1759842418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Clavulanate,418, 1759842,NA,NA,20250626154219589895388166,1 + 1124300418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Diclofenac,418, 1124300,NA,NA,20250626154219589895388166,1 + 1729720418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Penicillin V,418, 1729720,NA,NA,20250626154219589895388166,1 + 1124957418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Oxycodone,418, 1124957,NA,NA,20250626154219589895388166,1 + 4037675505,procedure_occurrence during starting between cohort start and cohort end: Brief general examination,505, 4037675,NA,NA,20250626154219589895388166,1 + 4107731505,procedure_occurrence during starting between cohort start and cohort end: Subcutaneous immunotherapy,505, 4107731,NA,NA,20250626154219589895388166,1 + 4125906505,procedure_occurrence during starting between cohort start and cohort end: Suture open wound,505, 4125906,NA,NA,20250626154219589895388166,1 + 4187458505,procedure_occurrence during starting between cohort start and cohort end: Review of systems,505, 4187458,NA,NA,20250626154219589895388166,1 + 4191853505,procedure_occurrence during starting between cohort start and cohort end: Allergy screening test,505, 4191853,NA,NA,20250626154219589895388166,1 + 4293740505,procedure_occurrence during starting between cohort start and cohort end: Injection of tetanus antitoxin,505, 4293740,NA,NA,20250626154219589895388166,1 + 4326177505,procedure_occurrence during starting between cohort start and cohort end: Medication Reconciliation,505, 4326177,NA,NA,20250626154219589895388166,1 + 4035793505,procedure_occurrence during starting between cohort start and cohort end: Pulmonary rehabilitation,505, 4035793,NA,NA,20250626154219589895388166,1 + 4202451505,procedure_occurrence during starting between cohort start and cohort end: Percutaneous mechanical thrombectomy of portal vein using fluoroscopic guidance,505, 4202451,NA,NA,20250626154219589895388166,1 + 4010253505,procedure_occurrence during starting between cohort start and cohort end: Nasal sinus endoscopy,505, 4010253,NA,NA,20250626154219589895388166,1 + 4163872505,procedure_occurrence during starting between cohort start and cohort end: Plain chest X-ray,505, 4163872,NA,NA,20250626154219589895388166,1 + 4170947505,procedure_occurrence during starting between cohort start and cohort end: Bone immobilization,505, 4170947,NA,NA,20250626154219589895388166,1 + 3000876713,measurement during starting between cohort start and cohort end: Codfish IgE Ab [Units/volume] in Serum,713, 3000876,NA,NA,20250626154219589895388166,1 + 3000963713,measurement during starting between cohort start and cohort end: Hemoglobin,713, 3000963,NA,NA,20250626154219589895388166,1 + 3001247713,measurement during starting between cohort start and cohort end: Common Ragweed IgE Ab [Units/volume] in Serum,713, 3001247,NA,NA,20250626154219589895388166,1 + 3001488713,measurement during starting between cohort start and cohort end: Cow milk IgE Ab [Units/volume] in Serum,713, 3001488,NA,NA,20250626154219589895388166,1 + 3005136713,measurement during starting between cohort start and cohort end: Cladosporium herbarum IgE Ab [Units/volume] in Serum,713, 3005136,NA,NA,20250626154219589895388166,1 + 3006322713,measurement during starting between cohort start and cohort end: Oral temperature,713, 3006322,NA,NA,20250626154219589895388166,1 + 3006451713,measurement during starting between cohort start and cohort end: Walnut IgE Ab [Units/volume] in Serum,713, 3006451,NA,NA,20250626154219589895388166,1 + 3006734713,measurement during starting between cohort start and cohort end: White Oak IgE Ab [Units/volume] in Serum,713, 3006734,NA,NA,20250626154219589895388166,1 + 3009542713,measurement during starting between cohort start and cohort end: Hematocrit,713, 3009542,NA,NA,20250626154219589895388166,1 + 3011505713,measurement during starting between cohort start and cohort end: FEV1/FVC,713, 3011505,NA,NA,20250626154219589895388166,1 + 3012494713,measurement during starting between cohort start and cohort end: Peanut IgE Ab [Units/volume] in Serum,713, 3012494,NA,NA,20250626154219589895388166,1 + 3014599713,measurement during starting between cohort start and cohort end: Egg white IgE Ab [Units/volume] in Serum,713, 3014599,NA,NA,20250626154219589895388166,1 + 3015076713,measurement during starting between cohort start and cohort end: Soybean IgE Ab [Units/volume] in Serum,713, 3015076,NA,NA,20250626154219589895388166,1 + 3019406713,measurement during starting between cohort start and cohort end: Latex IgE Ab [Units/volume] in Serum,713, 3019406,NA,NA,20250626154219589895388166,1 + 3020655713,measurement during starting between cohort start and cohort end: Honey bee IgE Ab [Units/volume] in Serum,713, 3020655,NA,NA,20250626154219589895388166,1 + 3021226713,measurement during starting between cohort start and cohort end: Shrimp IgE Ab [Units/volume] in Serum,713, 3021226,NA,NA,20250626154219589895388166,1 + 3023430713,measurement during starting between cohort start and cohort end: Cat dander IgE Ab [Units/volume] in Serum,713, 3023430,NA,NA,20250626154219589895388166,1 + 3027231713,measurement during starting between cohort start and cohort end: Wheat IgE Ab [Units/volume] in Serum,713, 3027231,NA,NA,20250626154219589895388166,1 + 3036780713,measurement during starting between cohort start and cohort end: American house dust mite IgE Ab [Units/volume] in Serum,713, 3036780,NA,NA,20250626154219589895388166,1 + 4024958713,measurement during starting between cohort start and cohort end: Throat culture,713, 4024958,NA,NA,20250626154219589895388166,1 + 4052083713,measurement during starting between cohort start and cohort end: Measurement of respiratory function,713, 4052083,NA,NA,20250626154219589895388166,1 + 4133840713,measurement during starting between cohort start and cohort end: Spirometry,713, 4133840,NA,NA,20250626154219589895388166,1 +40769179713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Head and Neck [PhenX],713,40769179,NA,NA,20250626154219589895388166,1 +40769184713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Upper extremity - bilateral [PhenX],713,40769184,NA,NA,20250626154219589895388166,1 +40769189713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Trunk [PhenX],713,40769189,NA,NA,20250626154219589895388166,1 +40769194713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Lower extremity - bilateral [PhenX],713,40769194,NA,NA,20250626154219589895388166,1 + 3051031713,measurement during starting between cohort start and cohort end: History of Hospitalizations+Outpatient visits Narrative,713, 3051031,NA,NA,20250626154219589895388166,1 +40758406713,measurement during starting between cohort start and cohort end: HIV status,713,40758406,NA,NA,20250626154219589895388166,1 +40766240713,measurement during starting between cohort start and cohort end: Are you covered by health insurance or some other kind of health care plan [PhenX],713,40766240,NA,NA,20250626154219589895388166,1 +46235214713,measurement during starting between cohort start and cohort end: Sexual orientation,713,46235214,NA,NA,20250626154219589895388166,1 + 9201927,visit_occurrence concept count during day cohort start through cohort end concept_count relative to index: Inpatient Visit,927, 0,NA,NA,20250626154219589895388166,1 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates.csv new file mode 100644 index 0000000..46ac4fc --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates.csv @@ -0,0 +1,192 @@ +covariate_id,sum_value,average_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id,min_characterization_mean +8507001,50,0.4347826,20250626154219589895388166,1,3,Cases,1,0.01 +8532001,65,0.5652174,20250626154219589895388166,1,3,Cases,1,0.01 +8507001,17,0.4857143,20250626154219589895388166,2,3,Cases,1,0.01 +8532001,18,0.5142857,20250626154219589895388166,2,3,Cases,1,0.01 +8507001,67,0.4466667,20250626154219589895388166,4,3,Cases,1,0.01 +8532001,83,0.5533333,20250626154219589895388166,4,3,Cases,1,0.01 + 30753218, 23,0.20000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 78272218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 80180218,115,1.00000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 81893218, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 195588218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 260139218, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 378001218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 738818418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 920293418, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 933724418, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1118084418,115,1.00000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1119510418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1125315418, 7,0.06086957,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1177480418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1713332418, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1759842418, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3000876713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3001247713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3001488713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3005136713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006322713, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006451713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006734713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3012494713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3014599713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3015076713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3019406713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3020655713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3021226713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3023430713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3027231713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3036780713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4001336218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4024958713, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4052083713, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4112343218, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4125906505, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4132546218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4191853505, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4266809218, 44,0.38260870,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4283893218, 13,0.11304348,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4285898218, 28,0.24347826,20250626154219589895388166,1,3,CasesBefore,1,0.01 +40481087218, 12,0.10434783,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 28060218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 80180218, 35,1.00000000,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 81151218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 257012218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 260139218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 381316218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1124300418, 35,1.00000000,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1125315418, 4,0.11428571,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1177480418, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1713332418, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1729720418, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1759842418, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3000876713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3001247713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3001488713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3005136713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006322713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006451713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006734713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3011505713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3012494713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3014599713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3015076713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3019406713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3020655713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3021226713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3023430713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3027231713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3036780713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4024958713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4035793505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4107731505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4113008218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4125906505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4133840713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4191853505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4202451505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4266809218, 3,0.08571429,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4283893218, 6,0.17142857,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4285898218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4294548218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 +40481087218, 4,0.11428571,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 30753218, 23,0.15333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 78272218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 80180218,150,1.00000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 81151218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 81893218, 8,0.05333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 195588218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 260139218, 6,0.04000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 378001218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 738818418, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 920293418, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 933724418, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1118084418,115,0.76666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1119510418, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1124300418, 35,0.23333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1125315418, 11,0.07333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1177480418, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1713332418, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1759842418, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3000876713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3001247713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3001488713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3005136713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006322713, 9,0.06000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006451713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006734713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3011505713, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3012494713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3014599713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3015076713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3019406713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3020655713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3021226713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3023430713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3027231713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3036780713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4001336218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4024958713, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4052083713, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4107731505, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4112343218, 8,0.05333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4113008218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4125906505, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4132546218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4133840713, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4191853505, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4266809218, 47,0.31333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4283893218, 19,0.12666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4285898218, 30,0.20000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4294548218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 +40481087218, 16,0.10666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 28060218, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 260139218, 5,0.04347826,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1125315418, 6,0.05217391,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1177480418, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1713332418, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1729720418, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1759842418, 3,0.02608696,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 3006322713, 10,0.08695652,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 3011505713, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4024958713, 5,0.04347826,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4052083713, 3,0.02608696,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4112343218, 6,0.05217391,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4133840713, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4163872505, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4283893218, 10,0.08695652,20250626154219589895388166,1,3,CasesAfter,1,0.01 +40481087218, 9,0.07826087,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 260139218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 920293418, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 933724418, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 1125315418, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 3006322713, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4024958713, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4052083713, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4112343218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4116491218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4125906505, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4155034218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4283893218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 +40481087218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 28060218, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 260139218, 6,0.04000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1125315418, 8,0.05333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1177480418, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1713332418, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1729720418, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1759842418, 3,0.02000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 3006322713, 12,0.08000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 3011505713, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4024958713, 6,0.04000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4052083713, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4112343218, 8,0.05333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4133840713, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4163872505, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4283893218, 12,0.08000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 +40481087218, 11,0.07333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 192671218,115,1.00000000,20250626154219589895388166,1,3,CasesBetween,1,0.01 + 192671218, 35,1.00000000,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 3006322713, 1,0.02857143,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 4112343218, 1,0.02857143,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 192671218,150,1.00000000,20250626154219589895388166,4,3,CasesBetween,1,0.01 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates_continuous.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates_continuous.csv new file mode 100644 index 0000000..81f567f --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/covariates_continuous.csv @@ -0,0 +1,10 @@ +covariate_id,count_value,min_value,max_value,average_value,standard_deviation,median_value,p_10_value,p_25_value,p_75_value,p_90_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +1002,115,32,45,38.54783,3.190566,38,34,36,41,43,20250626154219589895388166,1,3,Cases,1 +1002, 35,32,46,38.68571,3.314597,39,35,36,41,43,20250626154219589895388166,2,3,Cases,1 +1002,150,32,46,38.58000,3.209194,39,34,36,41,43,20250626154219589895388166,4,3,Cases,1 +9201927,115,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,1,3,CasesAfter,1 +9201927, 35,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,2,3,CasesAfter,1 +9201927,150,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,4,3,CasesAfter,1 +9201927, 96,0,1,0.8347826,0,1,0,1,1,1,20250626154219589895388166,1,3,CasesBetween,1 +9201927, 31,0,1,0.8857143,0,1,0,1,1,1,20250626154219589895388166,2,3,CasesBetween,1 +9201927,127,0,1,0.8466667,0,1,0,1,1,1,20250626154219589895388166,4,3,CasesBetween,1 diff --git a/tests/testthat/testdata/execution/cac_1_0_0_365_365/settings.csv b/tests/testthat/testdata/execution/cac_1_0_0_365_365/settings.csv new file mode 100644 index 0000000..3f200d4 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_1_0_0_365_365/settings.csv @@ -0,0 +1,40 @@ +setting_id,min_prior_observation,outcome_washout_days,risk_window_start,risk_window_end,start_anchor,end_anchor,case_pre_target_duration,case_post_outcome_duration,covariate_setting_json,case_covariate_setting_json,database_id +20250626154219589895388166,0,0,1,365,cohort start,cohort start,365,365,"[ + { + ""temporal"": false, + ""temporalSequence"": false, + ""DemographicsGender"": true, + ""DemographicsAge"": true, + ""DemographicsRace"": true, + ""longTermStartDays"": -365, + ""mediumTermStartDays"": -180, + ""shortTermStartDays"": -30, + ""endDays"": 0, + ""includedCovariateConceptIds"": [], + ""addDescendantsToInclude"": false, + ""excludedCovariateConceptIds"": [], + ""addDescendantsToExclude"": false, + ""includedCovariateIds"": [], + ""attr_class"": ""covariateSettings"", + ""attr_fun"": ""getDbDefaultCovariateData"" + } +]","[ + { + ""temporal"": false, + ""temporalSequence"": false, + ""ConditionGroupEraDuring"": true, + ""DrugGroupEraDuring"": true, + ""ProcedureOccurrenceDuring"": true, + ""DeviceExposureDuring"": true, + ""MeasurementDuring"": true, + ""ObservationDuring"": true, + ""VisitConceptCountDuring"": true, + ""includedCovariateConceptIds"": [], + ""addDescendantsToInclude"": false, + ""excludedCovariateConceptIds"": [], + ""addDescendantsToExclude"": false, + ""includedCovariateIds"": [], + ""attr_class"": ""covariateSettings"", + ""attr_fun"": ""Characterization::getDbDuringCovariateData"" + } +]",1 diff --git a/tests/testthat/testdata/execution/completed.csv b/tests/testthat/testdata/execution/completed.csv new file mode 100644 index 0000000..399e14b --- /dev/null +++ b/tests/testthat/testdata/execution/completed.csv @@ -0,0 +1,7 @@ +run_date_time,job_id,start_time,end_time +2025-06-26T19:42:19Z,0,2025-06-26T19:42:19Z,2025-06-26T19:42:19Z +2025-06-26T19:42:17Z,tte_1,2025-06-26T19:42:19Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,dr_1,2025-06-26T19:42:20Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,rfcs_1,2025-06-26T19:42:20Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,tac_1_0,2025-06-26T19:42:20Z,2025-06-26T19:42:22Z +2025-06-26T19:42:17Z,cac_1_0_0_365_365,2025-06-26T19:42:22Z,2025-06-26T19:42:24Z diff --git a/tests/testthat/testdata/execution/dr_1/dechallenge_rechallenge.csv b/tests/testthat/testdata/execution/dr_1/dechallenge_rechallenge.csv new file mode 100644 index 0000000..f2ee8ca --- /dev/null +++ b/tests/testthat/testdata/execution/dr_1/dechallenge_rechallenge.csv @@ -0,0 +1,2 @@ +database_id,dechallenge_stop_interval,dechallenge_evaluation_window,target_cohort_definition_id,outcome_cohort_definition_id,num_exposure_eras,num_persons_exposed,num_cases,dechallenge_attempt,dechallenge_fail,dechallenge_success,rechallenge_attempt,rechallenge_fail,rechallenge_success,pct_dechallenge_attempt,pct_dechallenge_success,pct_dechallenge_fail,pct_rechallenge_attempt,pct_rechallenge_success,pct_rechallenge_fail +1,30,30,1,2,13,10,6,5,0,5,3,1,2,0.8333333,1,0,0.6,0.6666667,0.3333333 diff --git a/tests/testthat/testdata/execution/execution.csv b/tests/testthat/testdata/execution/execution.csv new file mode 100644 index 0000000..470eb5c --- /dev/null +++ b/tests/testthat/testdata/execution/execution.csv @@ -0,0 +1,7 @@ +run_date_time,job_id,start_time,end_time +2025-06-26T19:42:19Z,0,2025-06-26T19:42:19Z,2025-06-26T19:42:19Z +2025-06-26T19:42:17Z,tte_1,2025-06-26T19:42:19Z,2025-06-26T19:42:19Z +2025-06-26T19:42:17Z,dr_1,2025-06-26T19:42:20Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,rfcs_1,2025-06-26T19:42:20Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,tac_1_0,2025-06-26T19:42:20Z,2025-06-26T19:42:20Z +2025-06-26T19:42:17Z,cac_1_0_0_365_365,2025-06-26T19:42:22Z,2025-06-26T19:42:22Z diff --git a/tests/testthat/testdata/execution/log.txt b/tests/testthat/testdata/execution/log.txt new file mode 100644 index 0000000..9fe4b53 --- /dev/null +++ b/tests/testthat/testdata/execution/log.txt @@ -0,0 +1,6 @@ +2025-06-26 15:42:20 [Main thread] INFO FeatureExtraction Constructing features on server +2025-06-26 15:42:20 [Main thread] INFO FeatureExtraction Fetching data from server +2025-06-26 15:42:21 [Main thread] INFO FeatureExtraction Fetching data took 1.12 secs +2025-06-26 15:42:22 [Main thread] INFO FeatureExtraction Constructing features on server +2025-06-26 15:42:22 [Main thread] INFO FeatureExtraction Fetching data from server +2025-06-26 15:42:22 [Main thread] INFO FeatureExtraction Fetching data took 0.256 secs diff --git a/tests/testthat/testdata/execution/rfcs_1/rechallenge_fail_case_series.csv b/tests/testthat/testdata/execution/rfcs_1/rechallenge_fail_case_series.csv new file mode 100644 index 0000000..529f6b5 --- /dev/null +++ b/tests/testthat/testdata/execution/rfcs_1/rechallenge_fail_case_series.csv @@ -0,0 +1,2 @@ +database_id,dechallenge_stop_interval,dechallenge_evaluation_window,target_cohort_definition_id,outcome_cohort_definition_id,person_key,subject_id,dechallenge_exposure_number,dechallenge_exposure_start_date_offset,dechallenge_exposure_end_date_offset,dechallenge_outcome_number,dechallenge_outcome_start_date_offset,rechallenge_exposure_number,rechallenge_exposure_start_date_offset,rechallenge_exposure_end_date_offset,rechallenge_outcome_number,rechallenge_outcome_start_date_offset +1,30,30,1,2,1,7,1,0,31,1,5,2,120,151,2,143 diff --git a/tests/testthat/testdata/execution/settings.rds b/tests/testthat/testdata/execution/settings.rds new file mode 100644 index 0000000000000000000000000000000000000000..611ce5b130df8e7fcebc3472c4dbf3eeca8c380e GIT binary patch literal 786 zcmV+t1MU1DiwFP!000001IV71|H4^t|7i?2grd`0MwJ>fy z=Pb2I?Bdv^aK)qX2s{c;0N9DsI$2UWCLtzO^ZE66jz1sgeG4JfK+R?yHCFTzt-Kw+ zdj0~TdPfU&ZR;cs2)VNUk2dtLcJu>fe@3n!ucpFrJ5x7}Zw)oknn})Z1hHPUlO(Bf zO=1y%Jo96#XmKS8kHBaO@g&4?7N03Ql14D({RuER04XU8jLXj=*5et}MObUU?hLP< zh5Q*&(#4uZ*0e|(Sr;>eb3_*TYC(QkkdI0$Xj(XXX$3t#;zD|GLm8Envw21PwxVi} zt`B5{!w^^i1Ifo{OoEu;(3%e%b6=|!HsM#4`TSEwkDfRomrnBPcC9bluK78ax36e- zNbDkfM3QzqjdlCvEZ5}wGQKWu6W4O~EnC+Xtdo7e&G${^o3DAfKnbVDR>i&*x*UZz zY<;T|qA?e<18-;Y06r>U1gw7BgNO$Lk4Kcmr@(v=*0SkZPJ%3;?W!;FEYViTSug|< z4NONp{5fM{Cjy_UsLZq-n=>f0JkUrS@qjWCDj)nFpI||;)Xdj4jGar~#lG)BOmrY- zG9L1?G$hLa(>rJAPu&bVw=mouca0mx3Pjt}7umbkR}`yI&O$vbam78`BAHXkY`iZu z|7QKKsMysf;wNM5RB$!!3+ySO>z^@IsG!uccOf_dUkNxTL6pjb zD`(q!4cI6nJ^J&8^Xu4*Oamxq=)R*dm6kp|t_pGgIaeD+zCG)Wu)st=5A;h??BQWb z*`%oZF!t5}F3Jwma?Dw;ddWKxWfy{S;has$BX0>~HCI@V8RbrwO+n)i>0_4+9;<$s zJO(F>=&58?>vPXO^D|hw>p8EI2Be0^1jo=@WV{eCOp4U)cFXA|Mjp%aw1|Rv7pBr2 z+8fS#?iN)aIQuvV1Oy3{O3y0S|Hh)vqUF8@me{rw=q@R+v5Zpl_rLT5usVl*T;xaq Q_LG#q0oD+09#0DZ0JZvX$N&HU literal 0 HcmV?d00001 diff --git a/tests/testthat/testdata/execution/tac_1_0/analysis_ref.csv b/tests/testthat/testdata/execution/tac_1_0/analysis_ref.csv new file mode 100644 index 0000000..9b9c851 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/analysis_ref.csv @@ -0,0 +1,4 @@ +analysis_id,analysis_name,domain_id,start_day,end_day,is_binary,missing_means_zero,setting_id,database_id +1,DemographicsGender,Demographics,NA,NA,Y,NA,20250626154217500852269034,1 +4,DemographicsRace,Demographics,NA,NA,Y,NA,20250626154217500852269034,1 +2,DemographicsAge,Demographics,NA,NA,N,Y,20250626154217500852269034,1 diff --git a/tests/testthat/testdata/execution/tac_1_0/cohort_counts.csv b/tests/testthat/testdata/execution/tac_1_0/cohort_counts.csv new file mode 100644 index 0000000..9ac738a --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/cohort_counts.csv @@ -0,0 +1,9 @@ +target_cohort_id,outcome_cohort_id,cohort_type,risk_window_start,risk_window_end,start_anchor,end_anchor,min_prior_observation,outcome_washout_days,database_id,row_count,person_count,min_exposure_time,mean_exposure_time,max_exposure_time +1,0,Target,NA,NA,NA,NA,0,NA,1,549,549,0,0,0 +2,0,Target,NA,NA,NA,NA,0,NA,1,242,242,0,0,0 +3,0,Target,NA,NA,NA,NA,0,NA,1,150,150,1,1,1 +4,0,Target,NA,NA,NA,NA,0,NA,1,791,791,0,0,0 +1,0,Tall,NA,NA,NA,NA,0,NA,1,565,565,0,0,0 +2,0,Tall,NA,NA,NA,NA,0,NA,1,244,244,0,0,0 +3,0,Tall,NA,NA,NA,NA,0,NA,1,150,150,1,1,1 +4,0,Tall,NA,NA,NA,NA,0,NA,1,809,809,0,0,0 diff --git a/tests/testthat/testdata/execution/tac_1_0/cohort_details.csv b/tests/testthat/testdata/execution/tac_1_0/cohort_details.csv new file mode 100644 index 0000000..0c59153 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/cohort_details.csv @@ -0,0 +1,9 @@ +setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +20250626154217500852269034,1,0,Target,1 +20250626154217500852269034,2,0,Target,1 +20250626154217500852269034,3,0,Target,1 +20250626154217500852269034,4,0,Target,1 +20250626154217500852269034,1,0,Tall,1 +20250626154217500852269034,2,0,Tall,1 +20250626154217500852269034,3,0,Tall,1 +20250626154217500852269034,4,0,Tall,1 diff --git a/tests/testthat/testdata/execution/tac_1_0/covariate_ref.csv b/tests/testthat/testdata/execution/tac_1_0/covariate_ref.csv new file mode 100644 index 0000000..199b955 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/covariate_ref.csv @@ -0,0 +1,4 @@ +covariate_id,covariate_name,analysis_id,concept_id,value_as_concept_id,collisions,setting_id,database_id +8507001,gender = MALE,1,8507,NA,NA,20250626154217500852269034,1 +8532001,gender = FEMALE,1,8532,NA,NA,20250626154217500852269034,1 + 1002,age in years,2, 0,NA,NA,20250626154217500852269034,1 diff --git a/tests/testthat/testdata/execution/tac_1_0/covariates.csv b/tests/testthat/testdata/execution/tac_1_0/covariates.csv new file mode 100644 index 0000000..af21be1 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/covariates.csv @@ -0,0 +1,9 @@ +covariate_id,sum_value,average_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id,min_characterization_mean +8507001,261,0.4754098,20250626154217500852269034,1,0,Target,1,0.01 +8532001,288,0.5245902,20250626154217500852269034,1,0,Target,1,0.01 +8507001,116,0.4793388,20250626154217500852269034,2,0,Target,1,0.01 +8532001,126,0.5206612,20250626154217500852269034,2,0,Target,1,0.01 +8507001, 67,0.4466667,20250626154217500852269034,3,0,Target,1,0.01 +8532001, 83,0.5533333,20250626154217500852269034,3,0,Target,1,0.01 +8507001,377,0.4766119,20250626154217500852269034,4,0,Target,1,0.01 +8532001,414,0.5233881,20250626154217500852269034,4,0,Target,1,0.01 diff --git a/tests/testthat/testdata/execution/tac_1_0/covariates_continuous.csv b/tests/testthat/testdata/execution/tac_1_0/covariates_continuous.csv new file mode 100644 index 0000000..f76d916 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/covariates_continuous.csv @@ -0,0 +1,5 @@ +covariate_id,count_value,min_value,max_value,average_value,standard_deviation,median_value,p_10_value,p_25_value,p_75_value,p_90_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +1002,549,31,47,38.57741,3.257770,39,34,36,41,43,20250626154217500852269034,1,0,Target,1 +1002,242,32,46,38.54545,3.142534,38,35,36,41,43,20250626154217500852269034,2,0,Target,1 +1002,150,32,47,38.72667,3.195838,39,34,36,41,43,20250626154217500852269034,3,0,Target,1 +1002,791,31,47,38.56764,3.221001,38,34,36,41,43,20250626154217500852269034,4,0,Target,1 diff --git a/tests/testthat/testdata/execution/tac_1_0/settings.csv b/tests/testthat/testdata/execution/tac_1_0/settings.csv new file mode 100644 index 0000000..1c00e38 --- /dev/null +++ b/tests/testthat/testdata/execution/tac_1_0/settings.csv @@ -0,0 +1,21 @@ +setting_id,min_prior_observation,outcome_washout_days,risk_window_start,risk_window_end,start_anchor,end_anchor,case_pre_target_duration,case_post_outcome_duration,covariate_setting_json,case_covariate_setting_json,database_id +20250626154217500852269034,0,NA,NA,NA,NA,NA,NA,NA,"[ + { + ""temporal"": false, + ""temporalSequence"": false, + ""DemographicsGender"": true, + ""DemographicsAge"": true, + ""DemographicsRace"": true, + ""longTermStartDays"": -365, + ""mediumTermStartDays"": -180, + ""shortTermStartDays"": -30, + ""endDays"": 0, + ""includedCovariateConceptIds"": [], + ""addDescendantsToInclude"": false, + ""excludedCovariateConceptIds"": [], + ""addDescendantsToExclude"": false, + ""includedCovariateIds"": [], + ""attr_class"": ""covariateSettings"", + ""attr_fun"": ""getDbDefaultCovariateData"" + } +]",NA,1 diff --git a/tests/testthat/testdata/execution/tte_1/time_to_event.csv b/tests/testthat/testdata/execution/tte_1/time_to_event.csv new file mode 100644 index 0000000..682a803 --- /dev/null +++ b/tests/testthat/testdata/execution/tte_1/time_to_event.csv @@ -0,0 +1,103 @@ +database_id,target_cohort_definition_id,outcome_cohort_definition_id,outcome_type,target_outcome_type,time_to_event,num_events,time_scale +1,1,4,first,During first, 0,549,per 1-day +1,2,4,first,During first, 0,242,per 1-day +1,1,3,first,After last target end, 5, 1,per 1-day +1,1,3,first,After last target end, 6, 1,per 1-day +1,1,3,first,After last target end, 7, 1,per 1-day +1,1,3,first,After last target end, 8, 1,per 1-day +1,1,3,first,After last target end, 9, 4,per 1-day +1,1,3,first,After last target end, 10, 1,per 1-day +1,1,3,first,After last target end, 14, 2,per 1-day +1,1,3,first,After last target end, 15, 2,per 1-day +1,1,3,first,After last target end, 16, 2,per 1-day +1,1,3,first,After last target end, 17, 1,per 1-day +1,1,3,first,After last target end, 18, 1,per 1-day +1,1,3,first,After last target end, 19, 1,per 1-day +1,1,3,first,After last target end, 20, 2,per 1-day +1,1,3,first,After last target end, 23, 3,per 1-day +1,1,3,first,After last target end, 25, 3,per 1-day +1,1,3,first,After last target end, 26, 1,per 1-day +1,1,3,first,After last target end, 27, 2,per 1-day +1,1,3,first,After last target end, 28, 2,per 1-day +1,1,3,first,After last target end, 31, 2,per 1-day +1,1,3,first,After last target end, 32, 2,per 1-day +1,1,3,first,After last target end, 33, 2,per 1-day +1,1,3,first,After last target end, 34, 1,per 1-day +1,1,3,first,After last target end, 35, 2,per 1-day +1,1,3,first,After last target end, 37, 4,per 1-day +1,1,3,first,After last target end, 40, 1,per 1-day +1,1,3,first,After last target end, 42, 2,per 1-day +1,1,3,first,After last target end, 43, 2,per 1-day +1,1,3,first,After last target end, 44, 2,per 1-day +1,1,3,first,After last target end, 45, 1,per 1-day +1,1,3,first,After last target end, 46, 2,per 1-day +1,1,3,first,After last target end, 47, 2,per 1-day +1,1,3,first,After last target end, 48, 3,per 1-day +1,1,3,first,After last target end, 50, 2,per 1-day +1,1,3,first,After last target end, 51, 2,per 1-day +1,1,3,first,After last target end, 52, 1,per 1-day +1,1,3,first,After last target end, 53, 1,per 1-day +1,1,3,first,After last target end, 55, 1,per 1-day +1,1,3,first,After last target end, 56, 2,per 1-day +1,1,3,first,After last target end, 57, 1,per 1-day +1,1,3,first,After last target end, 58, 2,per 1-day +1,1,3,first,After last target end, 59, 2,per 1-day +1,1,3,first,After last target end, 61, 1,per 1-day +1,1,3,first,After last target end, 64, 5,per 1-day +1,1,3,first,After last target end, 65, 1,per 1-day +1,1,3,first,After last target end, 66, 2,per 1-day +1,1,3,first,After last target end, 68, 1,per 1-day +1,1,3,first,After last target end, 69, 3,per 1-day +1,1,3,first,After last target end, 70, 2,per 1-day +1,1,3,first,After last target end, 71, 1,per 1-day +1,1,3,first,After last target end, 74, 2,per 1-day +1,1,3,first,After last target end, 75, 2,per 1-day +1,1,3,first,After last target end, 76, 2,per 1-day +1,1,3,first,After last target end, 77, 3,per 1-day +1,1,3,first,After last target end, 79, 1,per 1-day +1,1,3,first,After last target end, 80, 4,per 1-day +1,1,3,first,After last target end, 81, 2,per 1-day +1,1,3,first,After last target end, 82, 3,per 1-day +1,1,3,first,After last target end, 84, 1,per 1-day +1,1,3,first,After last target end, 85, 1,per 1-day +1,1,3,first,After last target end, 87, 2,per 1-day +1,1,3,first,After last target end, 89, 3,per 1-day +1,2,3,first,After last target end, 5, 3,per 1-day +1,2,3,first,After last target end, 7, 1,per 1-day +1,2,3,first,After last target end, 10, 1,per 1-day +1,2,3,first,After last target end, 11, 1,per 1-day +1,2,3,first,After last target end, 16, 2,per 1-day +1,2,3,first,After last target end, 17, 1,per 1-day +1,2,3,first,After last target end, 20, 1,per 1-day +1,2,3,first,After last target end, 21, 1,per 1-day +1,2,3,first,After last target end, 24, 2,per 1-day +1,2,3,first,After last target end, 25, 1,per 1-day +1,2,3,first,After last target end, 27, 1,per 1-day +1,2,3,first,After last target end, 37, 1,per 1-day +1,2,3,first,After last target end, 41, 1,per 1-day +1,2,3,first,After last target end, 43, 1,per 1-day +1,2,3,first,After last target end, 44, 1,per 1-day +1,2,3,first,After last target end, 53, 2,per 1-day +1,2,3,first,After last target end, 57, 2,per 1-day +1,2,3,first,After last target end, 62, 1,per 1-day +1,2,3,first,After last target end, 64, 1,per 1-day +1,2,3,first,After last target end, 65, 1,per 1-day +1,2,3,first,After last target end, 69, 2,per 1-day +1,2,3,first,After last target end, 71, 1,per 1-day +1,2,3,first,After last target end, 73, 1,per 1-day +1,2,3,first,After last target end, 76, 1,per 1-day +1,2,3,first,After last target end, 77, 1,per 1-day +1,2,3,first,After last target end, 85, 1,per 1-day +1,2,3,first,After last target end, 88, 2,per 1-day +1,1,3,first,After last target end, 30, 31,per 30-day +1,1,3,first,After last target end, 60, 42,per 30-day +1,1,3,first,After last target end, 90, 42,per 30-day +1,1,4,first,During first, 0,549,per 30-day +1,2,3,first,After last target end, 30, 15,per 30-day +1,2,3,first,After last target end, 60, 8,per 30-day +1,2,3,first,After last target end, 90, 12,per 30-day +1,2,4,first,During first, 0,242,per 30-day +1,1,3,first,After last target end,365,115,per 365-day +1,1,4,first,During first, 0,549,per 365-day +1,2,3,first,After last target end,365, 35,per 365-day +1,2,4,first,During first, 0,242,per 365-day From cde8383b5af60577d08501d1a31ec6009fd759b4 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 26 Jun 2025 16:24:17 -0400 Subject: [PATCH 19/35] more csv aggregation updates - changing append logic to only append if it is not the first time writing to the file or not the first chunk - adding more tests to the csv aggregation --- R/RunCharacterization.R | 4 +- tests/testthat/test-runCharacterization.R | 16 ++ .../cac_2_0_0_365_365/analysis_ref.csv | 11 + .../cac_2_0_0_365_365/cohort_counts.csv | 4 + .../cac_2_0_0_365_365/cohort_details.csv | 16 ++ .../cac_2_0_0_365_365/covariate_ref.csv | 93 +++++++++ .../cac_2_0_0_365_365/covariates.csv | 192 ++++++++++++++++++ .../covariates_continuous.csv | 10 + .../execution/cac_2_0_0_365_365/settings.csv | 40 ++++ 9 files changed, 384 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/analysis_ref.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_counts.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_details.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/covariate_ref.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates_continuous.csv create mode 100644 tests/testthat/testdata/execution/cac_2_0_0_365_365/settings.csv diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 8346ddb..a6edd23 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -650,8 +650,8 @@ aggregateCsvsBatch <- function( readr::write_csv( x = x, file = savePath, quote = "all", - #append = append & !firstTrackerCurrent & pos == 1 - append = append | pos != 1 + append = !firstTrackerCurrent | pos != 1 + #append = append | pos != 1 ) saveRDS(tracker,csvTrackerFile) diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index 88e533a..cdde4a4 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -487,4 +487,20 @@ for(i in 1:length(files)){ testthat::expect_true(all.equal(d1,d2)) } +# make sure it still works if re-executed (no left over files) +Characterization:::aggregateCsvsBatch( + executionPath = executionPath, + outputFolder = file.path(tempFolder,'aggCvs3'), + csvFilePrefix = '', + batchSize = 1 +) +files <- dir(file.path(tempFolder,'aggCvs'), pattern = 'csv') +for(i in 1:length(files)){ + d1 <- readr::read_csv(file.path(tempFolder,'aggCvs', files[i]), + show_col_types = F) + d2 <- readr::read_csv(file.path(tempFolder,'aggCvs3', files[i]), + show_col_types = F) + testthat::expect_true(all.equal(d1,d2)) +} + }) diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/analysis_ref.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/analysis_ref.csv new file mode 100644 index 0000000..644a1a4 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/analysis_ref.csv @@ -0,0 +1,11 @@ +analysis_id,analysis_name,domain_id,start_day,end_day,is_binary,missing_means_zero,setting_id,database_id +1,DemographicsGender,Demographics,NA,NA,Y,NA,20250626154219589895388166,1 +4,DemographicsRace,Demographics,NA,NA,Y,NA,20250626154219589895388166,1 +2,DemographicsAge,Demographics,NA,NA,N,Y,20250626154219589895388166,1 +218,ConditionGroupEraDuring,Condition,NA,NA,Y,NA,20250626154219589895388166,1 +418,DrugGroupEraDuring,Drug,NA,NA,Y,NA,20250626154219589895388166,1 +505,ProcedureOccurrenceDuring,Procedure,NA,NA,Y,NA,20250626154219589895388166,1 +605,DeviceExposureDuring,Device,NA,NA,Y,NA,20250626154219589895388166,1 +713,MeasurementDuring,Measurement,NA,NA,Y,NA,20250626154219589895388166,1 +805,ObservationDuring,Observation,NA,NA,Y,NA,20250626154219589895388166,1 +927,VisitConceptCountDuring,Visit,NA,NA,N,Y,20250626154219589895388166,1 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_counts.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_counts.csv new file mode 100644 index 0000000..6cdfd74 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_counts.csv @@ -0,0 +1,4 @@ +target_cohort_id,outcome_cohort_id,cohort_type,risk_window_start,risk_window_end,start_anchor,end_anchor,min_prior_observation,outcome_washout_days,database_id,row_count,person_count,min_exposure_time,mean_exposure_time,max_exposure_time +1,3,Cases,1,365,cohort start,cohort start,0,0,1,115,115,0,0,0 +2,3,Cases,1,365,cohort start,cohort start,0,0,1, 35, 35,0,0,0 +4,3,Cases,1,365,cohort start,cohort start,0,0,1,150,150,0,0,0 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_details.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_details.csv new file mode 100644 index 0000000..fb27ae8 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/cohort_details.csv @@ -0,0 +1,16 @@ +setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +20250626154219589895388166,1,3,Cases,1 +20250626154219589895388166,2,3,Cases,1 +20250626154219589895388166,4,3,Cases,1 +20250626154219589895388166,1,3,CasesBefore,1 +20250626154219589895388166,2,3,CasesBefore,1 +20250626154219589895388166,4,3,CasesBefore,1 +20250626154219589895388166,1,3,CasesAfter,1 +20250626154219589895388166,2,3,CasesAfter,1 +20250626154219589895388166,4,3,CasesAfter,1 +20250626154219589895388166,1,3,CasesBetween,1 +20250626154219589895388166,2,3,CasesBetween,1 +20250626154219589895388166,4,3,CasesBetween,1 +20250626154219589895388166,1,3,Exclude,1 +20250626154219589895388166,2,3,Exclude,1 +20250626154219589895388166,4,3,Exclude,1 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariate_ref.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariate_ref.csv new file mode 100644 index 0000000..10d2b53 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariate_ref.csv @@ -0,0 +1,93 @@ +covariate_id,covariate_name,analysis_id,concept_id,value_as_concept_id,collisions,setting_id,database_id +8507001,gender = MALE,1,8507,NA,NA,20250626154219589895388166,1 +8532001,gender = FEMALE,1,8532,NA,NA,20250626154219589895388166,1 + 1002,age in years,2, 0,NA,NA,20250626154219589895388166,1 + 30753218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Esophagitis,218, 30753,NA,NA,20250626154219589895388166,1 + 78272218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sprain of wrist,218, 78272,NA,NA,20250626154219589895388166,1 + 80180218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Osteoarthritis,218, 80180,NA,NA,20250626154219589895388166,1 + 81893218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Ulcerative colitis,218, 81893,NA,NA,20250626154219589895388166,1 + 134438218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Contact dermatitis,218, 134438,NA,NA,20250626154219589895388166,1 + 195588218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Cystitis,218, 195588,NA,NA,20250626154219589895388166,1 + 198199218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Pyelonephritis,218, 198199,NA,NA,20250626154219589895388166,1 + 260139218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute bronchitis,218, 260139,NA,NA,20250626154219589895388166,1 + 378001218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Concussion with no loss of consciousness,218, 378001,NA,NA,20250626154219589895388166,1 + 439777218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Anemia,218, 439777,NA,NA,20250626154219589895388166,1 + 4001336218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Concussion injury of brain,218, 4001336,NA,NA,20250626154219589895388166,1 + 4112343218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute viral pharyngitis,218, 4112343,NA,NA,20250626154219589895388166,1 + 4113008218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of hand,218, 4113008,NA,NA,20250626154219589895388166,1 + 4116491218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Escherichia coli urinary tract infection,218, 4116491,NA,NA,20250626154219589895388166,1 + 4132546218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Traumatic brain injury,218, 4132546,NA,NA,20250626154219589895388166,1 + 4152936218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of thigh,218, 4152936,NA,NA,20250626154219589895388166,1 + 4155034218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Laceration of forearm,218, 4155034,NA,NA,20250626154219589895388166,1 + 4266809218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Diverticular disease,218, 4266809,NA,NA,20250626154219589895388166,1 + 4283893218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sinusitis,218, 4283893,NA,NA,20250626154219589895388166,1 + 4285898218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Polyp of colon,218, 4285898,NA,NA,20250626154219589895388166,1 + 4310024218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Angiodysplasia of stomach,218, 4310024,NA,NA,20250626154219589895388166,1 +40481087218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Viral sinusitis,218,40481087,NA,NA,20250626154219589895388166,1 + 28060218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Streptococcal sore throat,218, 28060,NA,NA,20250626154219589895388166,1 + 81151218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Sprain of ankle,218, 81151,NA,NA,20250626154219589895388166,1 + 257012218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Chronic sinusitis,218, 257012,NA,NA,20250626154219589895388166,1 + 381316218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Cerebrovascular accident,218, 381316,NA,NA,20250626154219589895388166,1 + 4294548218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Acute bacterial sinusitis,218, 4294548,NA,NA,20250626154219589895388166,1 + 258780218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Emphysematous bronchitis,218, 258780,NA,NA,20250626154219589895388166,1 + 4094814218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Bullet wound,218, 4094814,NA,NA,20250626154219589895388166,1 + 4296205218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Second degree burn,218, 4296205,NA,NA,20250626154219589895388166,1 + 192671218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Gastrointestinal hemorrhage,218, 192671,NA,NA,20250626154219589895388166,1 + 4142905218,condition_era group (ConditionGroupEraDuring) starting between cohort start and cohort end: Fracture of rib,218, 4142905,NA,NA,20250626154219589895388166,1 + 738818418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Doxylamine,418, 738818,NA,NA,20250626154219589895388166,1 + 920293418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Nitrofurantoin,418, 920293,NA,NA,20250626154219589895388166,1 + 933724418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Phenazopyridine,418, 933724,NA,NA,20250626154219589895388166,1 + 975125418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Hydrocortisone,418, 975125,NA,NA,20250626154219589895388166,1 + 1118084418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: celecoxib,418, 1118084,NA,NA,20250626154219589895388166,1 + 1119510418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Dextromethorphan,418, 1119510,NA,NA,20250626154219589895388166,1 + 1125315418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Acetaminophen,418, 1125315,NA,NA,20250626154219589895388166,1 + 1150770418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Astemizole,418, 1150770,NA,NA,20250626154219589895388166,1 + 1177480418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Ibuprofen,418, 1177480,NA,NA,20250626154219589895388166,1 + 1713332418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Amoxicillin,418, 1713332,NA,NA,20250626154219589895388166,1 + 1759842418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Clavulanate,418, 1759842,NA,NA,20250626154219589895388166,1 + 1124300418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Diclofenac,418, 1124300,NA,NA,20250626154219589895388166,1 + 1729720418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Penicillin V,418, 1729720,NA,NA,20250626154219589895388166,1 + 1124957418,drug_era group (DrugGroupEraDuring) starting between cohort start and cohort end: Oxycodone,418, 1124957,NA,NA,20250626154219589895388166,1 + 4037675505,procedure_occurrence during starting between cohort start and cohort end: Brief general examination,505, 4037675,NA,NA,20250626154219589895388166,1 + 4107731505,procedure_occurrence during starting between cohort start and cohort end: Subcutaneous immunotherapy,505, 4107731,NA,NA,20250626154219589895388166,1 + 4125906505,procedure_occurrence during starting between cohort start and cohort end: Suture open wound,505, 4125906,NA,NA,20250626154219589895388166,1 + 4187458505,procedure_occurrence during starting between cohort start and cohort end: Review of systems,505, 4187458,NA,NA,20250626154219589895388166,1 + 4191853505,procedure_occurrence during starting between cohort start and cohort end: Allergy screening test,505, 4191853,NA,NA,20250626154219589895388166,1 + 4293740505,procedure_occurrence during starting between cohort start and cohort end: Injection of tetanus antitoxin,505, 4293740,NA,NA,20250626154219589895388166,1 + 4326177505,procedure_occurrence during starting between cohort start and cohort end: Medication Reconciliation,505, 4326177,NA,NA,20250626154219589895388166,1 + 4035793505,procedure_occurrence during starting between cohort start and cohort end: Pulmonary rehabilitation,505, 4035793,NA,NA,20250626154219589895388166,1 + 4202451505,procedure_occurrence during starting between cohort start and cohort end: Percutaneous mechanical thrombectomy of portal vein using fluoroscopic guidance,505, 4202451,NA,NA,20250626154219589895388166,1 + 4010253505,procedure_occurrence during starting between cohort start and cohort end: Nasal sinus endoscopy,505, 4010253,NA,NA,20250626154219589895388166,1 + 4163872505,procedure_occurrence during starting between cohort start and cohort end: Plain chest X-ray,505, 4163872,NA,NA,20250626154219589895388166,1 + 4170947505,procedure_occurrence during starting between cohort start and cohort end: Bone immobilization,505, 4170947,NA,NA,20250626154219589895388166,1 + 3000876713,measurement during starting between cohort start and cohort end: Codfish IgE Ab [Units/volume] in Serum,713, 3000876,NA,NA,20250626154219589895388166,1 + 3000963713,measurement during starting between cohort start and cohort end: Hemoglobin,713, 3000963,NA,NA,20250626154219589895388166,1 + 3001247713,measurement during starting between cohort start and cohort end: Common Ragweed IgE Ab [Units/volume] in Serum,713, 3001247,NA,NA,20250626154219589895388166,1 + 3001488713,measurement during starting between cohort start and cohort end: Cow milk IgE Ab [Units/volume] in Serum,713, 3001488,NA,NA,20250626154219589895388166,1 + 3005136713,measurement during starting between cohort start and cohort end: Cladosporium herbarum IgE Ab [Units/volume] in Serum,713, 3005136,NA,NA,20250626154219589895388166,1 + 3006322713,measurement during starting between cohort start and cohort end: Oral temperature,713, 3006322,NA,NA,20250626154219589895388166,1 + 3006451713,measurement during starting between cohort start and cohort end: Walnut IgE Ab [Units/volume] in Serum,713, 3006451,NA,NA,20250626154219589895388166,1 + 3006734713,measurement during starting between cohort start and cohort end: White Oak IgE Ab [Units/volume] in Serum,713, 3006734,NA,NA,20250626154219589895388166,1 + 3009542713,measurement during starting between cohort start and cohort end: Hematocrit,713, 3009542,NA,NA,20250626154219589895388166,1 + 3011505713,measurement during starting between cohort start and cohort end: FEV1/FVC,713, 3011505,NA,NA,20250626154219589895388166,1 + 3012494713,measurement during starting between cohort start and cohort end: Peanut IgE Ab [Units/volume] in Serum,713, 3012494,NA,NA,20250626154219589895388166,1 + 3014599713,measurement during starting between cohort start and cohort end: Egg white IgE Ab [Units/volume] in Serum,713, 3014599,NA,NA,20250626154219589895388166,1 + 3015076713,measurement during starting between cohort start and cohort end: Soybean IgE Ab [Units/volume] in Serum,713, 3015076,NA,NA,20250626154219589895388166,1 + 3019406713,measurement during starting between cohort start and cohort end: Latex IgE Ab [Units/volume] in Serum,713, 3019406,NA,NA,20250626154219589895388166,1 + 3020655713,measurement during starting between cohort start and cohort end: Honey bee IgE Ab [Units/volume] in Serum,713, 3020655,NA,NA,20250626154219589895388166,1 + 3021226713,measurement during starting between cohort start and cohort end: Shrimp IgE Ab [Units/volume] in Serum,713, 3021226,NA,NA,20250626154219589895388166,1 + 3023430713,measurement during starting between cohort start and cohort end: Cat dander IgE Ab [Units/volume] in Serum,713, 3023430,NA,NA,20250626154219589895388166,1 + 3027231713,measurement during starting between cohort start and cohort end: Wheat IgE Ab [Units/volume] in Serum,713, 3027231,NA,NA,20250626154219589895388166,1 + 3036780713,measurement during starting between cohort start and cohort end: American house dust mite IgE Ab [Units/volume] in Serum,713, 3036780,NA,NA,20250626154219589895388166,1 + 4024958713,measurement during starting between cohort start and cohort end: Throat culture,713, 4024958,NA,NA,20250626154219589895388166,1 + 4052083713,measurement during starting between cohort start and cohort end: Measurement of respiratory function,713, 4052083,NA,NA,20250626154219589895388166,1 + 4133840713,measurement during starting between cohort start and cohort end: Spirometry,713, 4133840,NA,NA,20250626154219589895388166,1 +40769179713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Head and Neck [PhenX],713,40769179,NA,NA,20250626154219589895388166,1 +40769184713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Upper extremity - bilateral [PhenX],713,40769184,NA,NA,20250626154219589895388166,1 +40769189713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Trunk [PhenX],713,40769189,NA,NA,20250626154219589895388166,1 +40769194713,measurement during starting between cohort start and cohort end: Percentage area affected by eczema Lower extremity - bilateral [PhenX],713,40769194,NA,NA,20250626154219589895388166,1 + 3051031713,measurement during starting between cohort start and cohort end: History of Hospitalizations+Outpatient visits Narrative,713, 3051031,NA,NA,20250626154219589895388166,1 +40758406713,measurement during starting between cohort start and cohort end: HIV status,713,40758406,NA,NA,20250626154219589895388166,1 +40766240713,measurement during starting between cohort start and cohort end: Are you covered by health insurance or some other kind of health care plan [PhenX],713,40766240,NA,NA,20250626154219589895388166,1 +46235214713,measurement during starting between cohort start and cohort end: Sexual orientation,713,46235214,NA,NA,20250626154219589895388166,1 + 9201927,visit_occurrence concept count during day cohort start through cohort end concept_count relative to index: Inpatient Visit,927, 0,NA,NA,20250626154219589895388166,1 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates.csv new file mode 100644 index 0000000..46ac4fc --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates.csv @@ -0,0 +1,192 @@ +covariate_id,sum_value,average_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id,min_characterization_mean +8507001,50,0.4347826,20250626154219589895388166,1,3,Cases,1,0.01 +8532001,65,0.5652174,20250626154219589895388166,1,3,Cases,1,0.01 +8507001,17,0.4857143,20250626154219589895388166,2,3,Cases,1,0.01 +8532001,18,0.5142857,20250626154219589895388166,2,3,Cases,1,0.01 +8507001,67,0.4466667,20250626154219589895388166,4,3,Cases,1,0.01 +8532001,83,0.5533333,20250626154219589895388166,4,3,Cases,1,0.01 + 30753218, 23,0.20000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 78272218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 80180218,115,1.00000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 81893218, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 195588218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 260139218, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 378001218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 738818418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 920293418, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 933724418, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1118084418,115,1.00000000,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1119510418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1125315418, 7,0.06086957,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1177480418, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1713332418, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 1759842418, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3000876713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3001247713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3001488713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3005136713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006322713, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006451713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3006734713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3012494713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3014599713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3015076713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3019406713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3020655713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3021226713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3023430713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3027231713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 3036780713, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4001336218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4024958713, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4052083713, 4,0.03478261,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4112343218, 8,0.06956522,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4125906505, 3,0.02608696,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4132546218, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4191853505, 2,0.01739130,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4266809218, 44,0.38260870,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4283893218, 13,0.11304348,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 4285898218, 28,0.24347826,20250626154219589895388166,1,3,CasesBefore,1,0.01 +40481087218, 12,0.10434783,20250626154219589895388166,1,3,CasesBefore,1,0.01 + 28060218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 80180218, 35,1.00000000,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 81151218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 257012218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 260139218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 381316218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1124300418, 35,1.00000000,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1125315418, 4,0.11428571,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1177480418, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1713332418, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1729720418, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 1759842418, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3000876713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3001247713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3001488713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3005136713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006322713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006451713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3006734713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3011505713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3012494713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3014599713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3015076713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3019406713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3020655713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3021226713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3023430713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3027231713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 3036780713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4024958713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4035793505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4107731505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4113008218, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4125906505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4133840713, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4191853505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4202451505, 1,0.02857143,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4266809218, 3,0.08571429,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4283893218, 6,0.17142857,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4285898218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 4294548218, 2,0.05714286,20250626154219589895388166,2,3,CasesBefore,1,0.01 +40481087218, 4,0.11428571,20250626154219589895388166,2,3,CasesBefore,1,0.01 + 30753218, 23,0.15333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 78272218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 80180218,150,1.00000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 81151218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 81893218, 8,0.05333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 195588218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 260139218, 6,0.04000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 378001218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 738818418, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 920293418, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 933724418, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1118084418,115,0.76666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1119510418, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1124300418, 35,0.23333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1125315418, 11,0.07333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1177480418, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1713332418, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 1759842418, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3000876713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3001247713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3001488713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3005136713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006322713, 9,0.06000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006451713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3006734713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3011505713, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3012494713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3014599713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3015076713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3019406713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3020655713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3021226713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3023430713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3027231713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 3036780713, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4001336218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4024958713, 5,0.03333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4052083713, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4107731505, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4112343218, 8,0.05333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4113008218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4125906505, 4,0.02666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4132546218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4133840713, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4191853505, 3,0.02000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4266809218, 47,0.31333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4283893218, 19,0.12666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4285898218, 30,0.20000000,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 4294548218, 2,0.01333333,20250626154219589895388166,4,3,CasesBefore,1,0.01 +40481087218, 16,0.10666667,20250626154219589895388166,4,3,CasesBefore,1,0.01 + 28060218, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 260139218, 5,0.04347826,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1125315418, 6,0.05217391,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1177480418, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1713332418, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1729720418, 4,0.03478261,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 1759842418, 3,0.02608696,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 3006322713, 10,0.08695652,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 3011505713, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4024958713, 5,0.04347826,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4052083713, 3,0.02608696,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4112343218, 6,0.05217391,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4133840713, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4163872505, 2,0.01739130,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 4283893218, 10,0.08695652,20250626154219589895388166,1,3,CasesAfter,1,0.01 +40481087218, 9,0.07826087,20250626154219589895388166,1,3,CasesAfter,1,0.01 + 260139218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 920293418, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 933724418, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 1125315418, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 3006322713, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4024958713, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4052083713, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4112343218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4116491218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4125906505, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4155034218, 1,0.02857143,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 4283893218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 +40481087218, 2,0.05714286,20250626154219589895388166,2,3,CasesAfter,1,0.01 + 28060218, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 260139218, 6,0.04000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1125315418, 8,0.05333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1177480418, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1713332418, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1729720418, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 1759842418, 3,0.02000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 3006322713, 12,0.08000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 3011505713, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4024958713, 6,0.04000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4052083713, 4,0.02666667,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4112343218, 8,0.05333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4133840713, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4163872505, 2,0.01333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 4283893218, 12,0.08000000,20250626154219589895388166,4,3,CasesAfter,1,0.01 +40481087218, 11,0.07333333,20250626154219589895388166,4,3,CasesAfter,1,0.01 + 192671218,115,1.00000000,20250626154219589895388166,1,3,CasesBetween,1,0.01 + 192671218, 35,1.00000000,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 3006322713, 1,0.02857143,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 4112343218, 1,0.02857143,20250626154219589895388166,2,3,CasesBetween,1,0.01 + 192671218,150,1.00000000,20250626154219589895388166,4,3,CasesBetween,1,0.01 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates_continuous.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates_continuous.csv new file mode 100644 index 0000000..81f567f --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/covariates_continuous.csv @@ -0,0 +1,10 @@ +covariate_id,count_value,min_value,max_value,average_value,standard_deviation,median_value,p_10_value,p_25_value,p_75_value,p_90_value,setting_id,target_cohort_id,outcome_cohort_id,cohort_type,database_id +1002,115,32,45,38.54783,3.190566,38,34,36,41,43,20250626154219589895388166,1,3,Cases,1 +1002, 35,32,46,38.68571,3.314597,39,35,36,41,43,20250626154219589895388166,2,3,Cases,1 +1002,150,32,46,38.58000,3.209194,39,34,36,41,43,20250626154219589895388166,4,3,Cases,1 +9201927,115,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,1,3,CasesAfter,1 +9201927, 35,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,2,3,CasesAfter,1 +9201927,150,1,1,1.0000000,0,1,1,1,1,1,20250626154219589895388166,4,3,CasesAfter,1 +9201927, 96,0,1,0.8347826,0,1,0,1,1,1,20250626154219589895388166,1,3,CasesBetween,1 +9201927, 31,0,1,0.8857143,0,1,0,1,1,1,20250626154219589895388166,2,3,CasesBetween,1 +9201927,127,0,1,0.8466667,0,1,0,1,1,1,20250626154219589895388166,4,3,CasesBetween,1 diff --git a/tests/testthat/testdata/execution/cac_2_0_0_365_365/settings.csv b/tests/testthat/testdata/execution/cac_2_0_0_365_365/settings.csv new file mode 100644 index 0000000..3f200d4 --- /dev/null +++ b/tests/testthat/testdata/execution/cac_2_0_0_365_365/settings.csv @@ -0,0 +1,40 @@ +setting_id,min_prior_observation,outcome_washout_days,risk_window_start,risk_window_end,start_anchor,end_anchor,case_pre_target_duration,case_post_outcome_duration,covariate_setting_json,case_covariate_setting_json,database_id +20250626154219589895388166,0,0,1,365,cohort start,cohort start,365,365,"[ + { + ""temporal"": false, + ""temporalSequence"": false, + ""DemographicsGender"": true, + ""DemographicsAge"": true, + ""DemographicsRace"": true, + ""longTermStartDays"": -365, + ""mediumTermStartDays"": -180, + ""shortTermStartDays"": -30, + ""endDays"": 0, + ""includedCovariateConceptIds"": [], + ""addDescendantsToInclude"": false, + ""excludedCovariateConceptIds"": [], + ""addDescendantsToExclude"": false, + ""includedCovariateIds"": [], + ""attr_class"": ""covariateSettings"", + ""attr_fun"": ""getDbDefaultCovariateData"" + } +]","[ + { + ""temporal"": false, + ""temporalSequence"": false, + ""ConditionGroupEraDuring"": true, + ""DrugGroupEraDuring"": true, + ""ProcedureOccurrenceDuring"": true, + ""DeviceExposureDuring"": true, + ""MeasurementDuring"": true, + ""ObservationDuring"": true, + ""VisitConceptCountDuring"": true, + ""includedCovariateConceptIds"": [], + ""addDescendantsToInclude"": false, + ""excludedCovariateConceptIds"": [], + ""addDescendantsToExclude"": false, + ""includedCovariateIds"": [], + ""attr_class"": ""covariateSettings"", + ""attr_fun"": ""Characterization::getDbDuringCovariateData"" + } +]",1 From 8c616f90a34da57772a7a65781d9b6c279905964 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Mon, 30 Jun 2025 09:41:59 -0400 Subject: [PATCH 20/35] Fix inconsistent PK designation in RDMS --- inst/settings/resultsDataModelSpecification.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 28ee778..4672f1b 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -14,15 +14,15 @@ rechallenge_fail_case_series,target_cohort_definition_id,bigint,Yes,Yes,No,No,Th rechallenge_fail_case_series,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort rechallenge_fail_case_series,person_key,int,Yes,Yes,No,No,The dense rank for the patient (an identifier that is not the same as the database) rechallenge_fail_case_series,subject_id,bigint,No,No,No,No,The person identifier for the failed case series (optional) -rechallenge_fail_case_series,dechallenge_exposure_number,int,Yes,No,No,No,The number of times a dechallenge has occurred +rechallenge_fail_case_series,dechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a dechallenge has occurred rechallenge_fail_case_series,dechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the dechallenge start (number of days after index) rechallenge_fail_case_series,dechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the dechallenge end (number of days after index) -rechallenge_fail_case_series,dechallenge_outcome_number,int,Yes,No,No,No,The number of times an outcome has occurred during the dechallenge +rechallenge_fail_case_series,dechallenge_outcome_number,int,Yes,Yes,No,No,The number of times an outcome has occurred during the dechallenge rechallenge_fail_case_series,dechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) -rechallenge_fail_case_series,rechallenge_exposure_number,int,Yes,No,No,No,The number of times a rechallenge exposure has occurred +rechallenge_fail_case_series,rechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a rechallenge exposure has occurred rechallenge_fail_case_series,rechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the rechallenge start (number of days after index) rechallenge_fail_case_series,rechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the rechallenge end (number of days after index) -rechallenge_fail_case_series,rechallenge_outcome_number,int,Yes,No,No,No,The number of times the outcome has occurred during the rechallenge +rechallenge_fail_case_series,rechallenge_outcome_number,int,Yes,Yes,No,No,The number of times the outcome has occurred during the rechallenge rechallenge_fail_case_series,rechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) dechallenge_rechallenge,database_id,varchar(100),Yes,Yes,No,No,The database identifier dechallenge_rechallenge,dechallenge_stop_interval,int,Yes,Yes,No,No,The dechallenge stop interval @@ -66,7 +66,7 @@ covariates,setting_id,varchar(30),Yes,Yes,No,No,The run identifier covariates,cohort_type,varchar(12),Yes,Yes,No,No,The cohort type covariates,target_cohort_id,int,Yes,Yes,No,No,The target cohort id covariates,outcome_cohort_id,int,Yes,Yes,No,No,The outcome cohort id -covariates,min_characterization_mean,float,Yes,Yes,No,No,Minimum fraction for feature extraction +covariates,min_characterization_mean,float,Yes,No,No,No,Minimum fraction for feature extraction covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id covariates,sum_value,int,Yes,No,No,No,The sum value covariates,average_value,float,No,No,No,No,The average value From 3a90c7caf53e8797cf265969d526c6557bea4cda Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Mon, 30 Jun 2025 11:56:44 -0400 Subject: [PATCH 21/35] Restore PK for min_characterization_mean --- inst/settings/resultsDataModelSpecification.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 4672f1b..922aa84 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -66,7 +66,7 @@ covariates,setting_id,varchar(30),Yes,Yes,No,No,The run identifier covariates,cohort_type,varchar(12),Yes,Yes,No,No,The cohort type covariates,target_cohort_id,int,Yes,Yes,No,No,The target cohort id covariates,outcome_cohort_id,int,Yes,Yes,No,No,The outcome cohort id -covariates,min_characterization_mean,float,Yes,No,No,No,Minimum fraction for feature extraction +covariates,min_characterization_mean,float,Yes,Yes,No,No,Minimum fraction for feature extraction covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id covariates,sum_value,int,Yes,No,No,No,The sum value covariates,average_value,float,No,No,No,No,The average value From b3467535a40a0b2f55a6450480327eebfe1b85c5 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 28 Aug 2025 14:12:10 -0400 Subject: [PATCH 22/35] progressBar and complete csv updates - fixing progressBar to default to interactive() when possible or turned off when not possible to specify as an input. - added code to ensure all the result csv files are created --- DESCRIPTION | 2 +- R/AggregateCovariates.R | 22 +++++++++++-------- R/DechallengeRechallenge.R | 12 ++++++---- R/RunCharacterization.R | 9 ++++++++ R/TimeToEvent.R | 8 ++++--- man/computeDechallengeRechallengeAnalyses.Rd | 3 +++ ...omputeRechallengeFailCaseSeriesAnalyses.Rd | 3 +++ man/computeTimeToEventAnalyses.Rd | 3 +++ 8 files changed, 45 insertions(+), 17 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0d26568..926b57a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 2.2.0 +Version: 2.2.0.9999 Date: 2025-5-07 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), diff --git a/R/AggregateCovariates.R b/R/AggregateCovariates.R index 03d14b5..4a3bc89 100644 --- a/R/AggregateCovariates.R +++ b/R/AggregateCovariates.R @@ -202,6 +202,7 @@ computeTargetAggregateCovariateAnalyses <- function( outputFolder, minCharacterizationMean = 0, minCellCount = 0, + progressBar = interactive(), ...) { if(missing(outputFolder)){ @@ -251,7 +252,7 @@ computeTargetAggregateCovariateAnalyses <- function( tempTable = TRUE, dropTableIfExists = TRUE, createTable = TRUE, - progressBar = FALSE, + progressBar = progressBar, tempEmulationSchema = tempEmulationSchema ) @@ -273,7 +274,7 @@ computeTargetAggregateCovariateAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) completionTime <- Sys.time() - start @@ -299,7 +300,7 @@ computeTargetAggregateCovariateAnalyses <- function( counts <- DatabaseConnector::querySql( connection = connection, sql = sql, - snakeCaseToCamelCase = TRUE, + snakeCaseToCamelCase = TRUE ) message("Target Aggregate: Computing aggregate target covariate results") @@ -327,7 +328,8 @@ computeTargetAggregateCovariateAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql, progressBar = FALSE, + sql = sql, + progressBar = progressBar, reportOverallTime = FALSE ) @@ -363,6 +365,7 @@ computeCaseAggregateCovariateAnalyses <- function( outputFolder, minCharacterizationMean = 0, minCellCount = 0, + progressBar = interactive(), ...) { if(missing(outputFolder)){ @@ -443,7 +446,7 @@ computeCaseAggregateCovariateAnalyses <- function( tempTable = TRUE, dropTableIfExists = TRUE, createTable = TRUE, - progressBar = FALSE, + progressBar = progressBar, tempEmulationSchema = tempEmulationSchema ) @@ -469,7 +472,7 @@ computeCaseAggregateCovariateAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) @@ -495,7 +498,7 @@ computeCaseAggregateCovariateAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) } @@ -522,7 +525,7 @@ computeCaseAggregateCovariateAnalyses <- function( counts <- DatabaseConnector::querySql( connection = connection, sql = sql, - snakeCaseToCamelCase = TRUE, + snakeCaseToCamelCase = TRUE ) message("Case Aggregates: Computing aggregate before case covariate results") @@ -576,7 +579,8 @@ computeCaseAggregateCovariateAnalyses <- function( ) DatabaseConnector::executeSql( connection = connection, - sql = sql, progressBar = FALSE, + sql = sql, + progressBar = progressBar, reportOverallTime = FALSE ) diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index 5ba9f37..2cab633 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -97,6 +97,7 @@ createDechallengeRechallengeSettings <- function( #' @param databaseId An identifier for the database (string) #' @param outputFolder A directory to save the results as csv files #' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 +#' @param progressBar Whether to display a progress bar while the analysis is running #' @param ... extra inputs #' @family DechallengeRechallenge #' @@ -133,6 +134,7 @@ computeDechallengeRechallengeAnalyses <- function( databaseId = "database 1", outputFolder, minCellCount = 0, + progressBar = interactive(), ...) { if(missing(outputFolder)){ @@ -199,7 +201,7 @@ computeDechallengeRechallengeAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = interactive() + progressBar = progressBar ) sql <- "select * from #challenge;" @@ -226,7 +228,7 @@ computeDechallengeRechallengeAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) @@ -265,6 +267,7 @@ computeDechallengeRechallengeAnalyses <- function( #' @param showSubjectId if F then subject_ids are hidden (recommended if sharing results) #' @param outputFolder A directory to save the results as csv files #' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 +#' @param progressBar Whether to display a progress bar while the analysis is running #' @param ... extra inputs #' @family DechallengeRechallenge #' @@ -301,6 +304,7 @@ computeRechallengeFailCaseSeriesAnalyses <- function( showSubjectId = FALSE, outputFolder, minCellCount = 0, + progressBar = interactive(), ...) { if(missing(outputFolder)){ @@ -365,7 +369,7 @@ computeRechallengeFailCaseSeriesAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = interactive() + progressBar = progressBar ) sql <- "select * from #fail_case_series;" @@ -392,7 +396,7 @@ computeRechallengeFailCaseSeriesAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index a6edd23..cae3b8b 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -647,6 +647,8 @@ aggregateCsvsBatch <- function( tracker$settingsTracker <- c(tracker$settingsTracker, unique(x$setting_id)) } + # this does not work if the csv is empty - only + # works if the csv has rows. readr::write_csv( x = x, file = savePath, quote = "all", @@ -666,6 +668,13 @@ aggregateCsvsBatch <- function( show_col_types = FALSE ) + # readr::read_csv_chunked only works if the csv + # has 1 row or more. This code will copy the + # csv with no rows to we always get a complete set of csv files + if(!file.exists(savePath) & file.exists(loadPath)){ + file.copy(from = loadPath, to = savePath) + } + firstTracker$first[firstTracker$table == csvType] <- FALSE } } diff --git a/R/TimeToEvent.R b/R/TimeToEvent.R index 2dee125..9cad143 100644 --- a/R/TimeToEvent.R +++ b/R/TimeToEvent.R @@ -73,6 +73,7 @@ createTimeToEventSettings <- function( #' @param databaseId An identifier for the database (string) #' @param outputFolder A directory to save the results as csv files #' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 +#' @param progressBar Whether to display a progress bar while the analysis is running #' @param ... extra inputs #' @family TimeToEvent #' @@ -113,6 +114,7 @@ computeTimeToEventAnalyses <- function( databaseId = "database 1", outputFolder, minCellCount = 0, + progressBar = interactive(), ...) { if(missing(outputFolder)){ @@ -171,7 +173,7 @@ computeTimeToEventAnalyses <- function( createTable = TRUE, tempTable = TRUE, tempEmulationSchema = tempEmulationSchema, - progressBar = FALSE, + progressBar = progressBar, camelCaseToSnakeCase = TRUE ) @@ -192,7 +194,7 @@ computeTimeToEventAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = interactive() + progressBar = progressBar ) sql <- "select * from #two_tte_summary;" @@ -220,7 +222,7 @@ computeTimeToEventAnalyses <- function( DatabaseConnector::executeSql( connection = connection, sql = sql, - progressBar = FALSE, + progressBar = progressBar, reportOverallTime = FALSE ) diff --git a/man/computeDechallengeRechallengeAnalyses.Rd b/man/computeDechallengeRechallengeAnalyses.Rd index 2b173ff..8e4e4e1 100644 --- a/man/computeDechallengeRechallengeAnalyses.Rd +++ b/man/computeDechallengeRechallengeAnalyses.Rd @@ -15,6 +15,7 @@ computeDechallengeRechallengeAnalyses( databaseId = "database 1", outputFolder, minCellCount = 0, + progressBar = interactive(), ... ) } @@ -46,6 +47,8 @@ can be created} \item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} +\item{progressBar}{Whether to display a progress bar while the analysis is running} + \item{...}{extra inputs} } \value{ diff --git a/man/computeRechallengeFailCaseSeriesAnalyses.Rd b/man/computeRechallengeFailCaseSeriesAnalyses.Rd index 3b0bdd3..939d409 100644 --- a/man/computeRechallengeFailCaseSeriesAnalyses.Rd +++ b/man/computeRechallengeFailCaseSeriesAnalyses.Rd @@ -16,6 +16,7 @@ computeRechallengeFailCaseSeriesAnalyses( showSubjectId = FALSE, outputFolder, minCellCount = 0, + progressBar = interactive(), ... ) } @@ -49,6 +50,8 @@ can be created} \item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} +\item{progressBar}{Whether to display a progress bar while the analysis is running} + \item{...}{extra inputs} } \value{ diff --git a/man/computeTimeToEventAnalyses.Rd b/man/computeTimeToEventAnalyses.Rd index e4de389..a5784d8 100644 --- a/man/computeTimeToEventAnalyses.Rd +++ b/man/computeTimeToEventAnalyses.Rd @@ -16,6 +16,7 @@ computeTimeToEventAnalyses( databaseId = "database 1", outputFolder, minCellCount = 0, + progressBar = interactive(), ... ) } @@ -49,6 +50,8 @@ can be created} \item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} +\item{progressBar}{Whether to display a progress bar while the analysis is running} + \item{...}{extra inputs} } \value{ From c7285f7f67e6bcdb9ccb8dbe6153e662fae80cab Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 28 Aug 2025 14:34:59 -0400 Subject: [PATCH 23/35] Update R_CMD_check_Hades.yaml updating GHA with curl fix part 1 (thanks Anthony Sena!) --- .github/workflows/R_CMD_check_Hades.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml index 7579757..3023fd8 100644 --- a/.github/workflows/R_CMD_check_Hades.yaml +++ b/.github/workflows/R_CMD_check_Hades.yaml @@ -82,7 +82,7 @@ jobs: while read -r cmd do eval sudo $cmd - done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') + done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "22.04"))') - uses: r-lib/actions/setup-r-dependencies@v2 with: From 463794544175cca6ecd2a0952319d42e42b2b6c9 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 28 Aug 2025 14:41:01 -0400 Subject: [PATCH 24/35] Update to make person_count bigint --- R/AggregateCovariates.R | 2 +- inst/settings/resultsDataModelSpecification.csv | 2 +- ...count_bigint.sql => Migration_2-v2_2_-0_count_as_bigint.sql} | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) rename inst/sql/sql_server/migrations/{Migration_2-v2_0_2_row_count_bigint.sql => Migration_2-v2_2_-0_count_as_bigint.sql} (60%) diff --git a/R/AggregateCovariates.R b/R/AggregateCovariates.R index b718ec2..eca021a 100644 --- a/R/AggregateCovariates.R +++ b/R/AggregateCovariates.R @@ -285,7 +285,7 @@ computeTargetAggregateCovariateAnalyses <- function( sql <- "select cohort_definition_id, count_big(*) row_count, - count(distinct subject_id) person_count, + count_big(distinct subject_id) person_count, min(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) min_exposure_time, avg(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) mean_exposure_time, max(cast(datediff(day, cohort_start_date, cohort_end_date) as bigint)) max_exposure_time diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 83e2e26..7786bde 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -114,7 +114,7 @@ cohort_counts,end_anchor,varchar(15),No,No,No,No,The end anchor cohort_counts,min_prior_observation,int,No,No,No,No,Minimum time observed before index cohort_counts,outcome_washout_days,int,No,No,No,No,Patients with outcome during washout are excluded cohort_counts,row_count,bigint,Yes,No,No,No,The number of rows in each cohort -cohort_counts,person_count,int,Yes,No,No,No,The number of distinct people in each cohort +cohort_counts,person_count,bigint,Yes,No,No,No,The number of distinct people in each cohort cohort_counts,min_exposure_time,bigint,No,No,No,No,Minimum exposure time across cohort in days cohort_counts,mean_exposure_time,float,No,No,No,No,Mean exposure time across cohort in days cohort_counts,max_exposure_time,bigint,No,No,No,No,Max exposure time across cohort in days diff --git a/inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql b/inst/sql/sql_server/migrations/Migration_2-v2_2_-0_count_as_bigint.sql similarity index 60% rename from inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql rename to inst/sql/sql_server/migrations/Migration_2-v2_2_-0_count_as_bigint.sql index 5d6f626..8c21f14 100644 --- a/inst/sql/sql_server/migrations/Migration_2-v2_0_2_row_count_bigint.sql +++ b/inst/sql/sql_server/migrations/Migration_2-v2_2_-0_count_as_bigint.sql @@ -1,5 +1,7 @@ -- Database migrations for verion 2.0.2 -- This migration updates the schema: -- 1. To expand the row_count column to a bigint + -- 2. To expand the person_count column to a bigint ALTER TABLE @database_schema.@table_prefixcohort_counts ALTER COLUMN row_count BIGINT; +ALTER TABLE @database_schema.@table_prefixcohort_counts ALTER COLUMN person_count BIGINT; From a1b0ee3160c9ee0b56bb2f071a9d5817f51e3a27 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 28 Aug 2025 14:41:53 -0400 Subject: [PATCH 25/35] Rename migration file --- ...ount_as_bigint.sql => Migration_2-v2_2_-1_count_as_bigint.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename inst/sql/sql_server/migrations/{Migration_2-v2_2_-0_count_as_bigint.sql => Migration_2-v2_2_-1_count_as_bigint.sql} (100%) diff --git a/inst/sql/sql_server/migrations/Migration_2-v2_2_-0_count_as_bigint.sql b/inst/sql/sql_server/migrations/Migration_2-v2_2_-1_count_as_bigint.sql similarity index 100% rename from inst/sql/sql_server/migrations/Migration_2-v2_2_-0_count_as_bigint.sql rename to inst/sql/sql_server/migrations/Migration_2-v2_2_-1_count_as_bigint.sql From 78d11cbf85ae181359a28684177ac3c9a3e0fe0c Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 28 Aug 2025 14:43:28 -0400 Subject: [PATCH 26/35] do over --- ...count_as_bigint.sql => Migration_2-v2_2_0_count_as_bigint.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename inst/sql/sql_server/migrations/{Migration_2-v2_2_-1_count_as_bigint.sql => Migration_2-v2_2_0_count_as_bigint.sql} (100%) diff --git a/inst/sql/sql_server/migrations/Migration_2-v2_2_-1_count_as_bigint.sql b/inst/sql/sql_server/migrations/Migration_2-v2_2_0_count_as_bigint.sql similarity index 100% rename from inst/sql/sql_server/migrations/Migration_2-v2_2_-1_count_as_bigint.sql rename to inst/sql/sql_server/migrations/Migration_2-v2_2_0_count_as_bigint.sql From 740bab9ca586a80f606752e1f27bb31f985e9417 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 28 Aug 2025 14:43:50 -0400 Subject: [PATCH 27/35] Update DESCRIPTION --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 926b57a..0d26568 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 2.2.0.9999 +Version: 2.2.0 Date: 2025-5-07 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), From b017b9e10e96d3d3f9f9c2b4808f7605fa89f42e Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 28 Aug 2025 14:54:22 -0400 Subject: [PATCH 28/35] Update R_CMD_check_Hades.yaml --- .github/workflows/R_CMD_check_Hades.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml index 3023fd8..e16feaa 100644 --- a/.github/workflows/R_CMD_check_Hades.yaml +++ b/.github/workflows/R_CMD_check_Hades.yaml @@ -77,7 +77,10 @@ jobs: - name: Install system requirements if: runner.os == 'Linux' run: | + sudo apt-get install -y make + sudo apt-get install -y libcurl4-openssl-dev sudo apt-get install -y libssh-dev + sudo apt-get install -y libssl-dev Rscript -e 'install.packages("remotes")' while read -r cmd do From 695df040dd3635b54f9000fc5fc5d4ad94b6b3fa Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 28 Aug 2025 15:06:29 -0400 Subject: [PATCH 29/35] Fix migration file order --- ...count_as_bigint.sql => Migration_3-v2_2_0_count_as_bigint.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename inst/sql/sql_server/migrations/{Migration_2-v2_2_0_count_as_bigint.sql => Migration_3-v2_2_0_count_as_bigint.sql} (100%) diff --git a/inst/sql/sql_server/migrations/Migration_2-v2_2_0_count_as_bigint.sql b/inst/sql/sql_server/migrations/Migration_3-v2_2_0_count_as_bigint.sql similarity index 100% rename from inst/sql/sql_server/migrations/Migration_2-v2_2_0_count_as_bigint.sql rename to inst/sql/sql_server/migrations/Migration_3-v2_2_0_count_as_bigint.sql From a52467e1820091c466cd03977a54736794e32a9a Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 28 Aug 2025 15:16:17 -0400 Subject: [PATCH 30/35] Update R_CMD_check_Hades.yaml --- .github/workflows/R_CMD_check_Hades.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml index e16feaa..b1bbef3 100644 --- a/.github/workflows/R_CMD_check_Hades.yaml +++ b/.github/workflows/R_CMD_check_Hades.yaml @@ -22,7 +22,7 @@ jobs: config: - {os: windows-latest, r: 'release'} - {os: macOS-latest, r: 'release'} - - {os: ubuntu-latest, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + - {os: ubuntu-22.04, r: 'release', rtools: ''} env: GITHUB_PAT: ${{ secrets.GH_TOKEN }} From c617246b41c3207d46ad6824dea18ec5702badc6 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 2 Sep 2025 10:01:35 -0400 Subject: [PATCH 31/35] Single quotes for constants in SQL and use rlang::inform to report errors --- R/RunCharacterization.R | 2 +- inst/sql/sql_server/ConceptCountsDuring.sql | 2 +- inst/sql/sql_server/DomainConceptDuring.sql | 2 +- inst/sql/sql_server/DomainConceptGroupDuring.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index cae3b8b..6e250c5 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -421,7 +421,7 @@ runCharacterizationsInParallel <- function(x) { ) }, error = function(e) { - print(e) + rlang::inform(e) return(FALSE) } ) diff --git a/inst/sql/sql_server/ConceptCountsDuring.sql b/inst/sql/sql_server/ConceptCountsDuring.sql index 6c43b26..e10ebbd 100644 --- a/inst/sql/sql_server/ConceptCountsDuring.sql +++ b/inst/sql/sql_server/ConceptCountsDuring.sql @@ -1,7 +1,7 @@ -- adding line below to prevent warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT "@row_id_field" as cname into #fake; +SELECT '@row_id_field' as cname into #fake; -- Feature construction {@aggregated} ? { diff --git a/inst/sql/sql_server/DomainConceptDuring.sql b/inst/sql/sql_server/DomainConceptDuring.sql index 603f5fb..427d91c 100644 --- a/inst/sql/sql_server/DomainConceptDuring.sql +++ b/inst/sql/sql_server/DomainConceptDuring.sql @@ -1,7 +1,7 @@ -- adding lines below to prevent warning IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT "@domain_end_date" as cname into #fake; +SELECT '@domain_end_date' as cname into #fake; -- Feature construction SELECT diff --git a/inst/sql/sql_server/DomainConceptGroupDuring.sql b/inst/sql/sql_server/DomainConceptGroupDuring.sql index 6ec2efa..22407b0 100644 --- a/inst/sql/sql_server/DomainConceptGroupDuring.sql +++ b/inst/sql/sql_server/DomainConceptGroupDuring.sql @@ -1,7 +1,7 @@ -- add dummy code with all imputs to stop annoying warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT "@domain_end_date" as cname into #fake; +SELECT '@domain_end_date' as cname into #fake; IF OBJECT_ID('tempdb..#groups', 'U') IS NOT NULL DROP TABLE #groups; From 58b678379cad1eb74f16d9a8aabc1c52b788b67e Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 2 Sep 2025 10:47:29 -0400 Subject: [PATCH 32/35] preparing for release updating description and news for release --- DESCRIPTION | 2 +- NEWS.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0d26568..38742b1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model Version: 2.2.0 -Date: 2025-5-07 +Date: 2025-8-28 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), person("Patrick", "Ryan", , "ryan@ohdsi.org", role = c("aut")), diff --git a/NEWS.md b/NEWS.md index 96f8e79..274f112 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,8 @@ Characterization 2.2.0 - removed progress bar from custom during features - added option includedFiles in insertResultsToDatabase() where you can specify the csv files to upload to prevent warnings of missing csv files. - made sure all connections are disconnected after use +- fixed counts to use count_big (thanks Anthony Sena) to fix an issue where the number was bigger than an integer. +- added code to copy csv files in batches this is needed when the csv files are very large. Characterization 2.1.3 ====================== From 5517db5c6bbb96ee76d20ab068f427a19eb0cf14 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 2 Sep 2025 11:09:32 -0400 Subject: [PATCH 33/35] Update SQL syntax for SqlRender translation; properly emit error message --- R/RunCharacterization.R | 2 +- inst/sql/sql_server/ConceptCountsDuring.sql | 2 +- inst/sql/sql_server/DomainConceptDuring.sql | 2 +- inst/sql/sql_server/DomainConceptGroupDuring.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 6e250c5..14fd6b6 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -421,7 +421,7 @@ runCharacterizationsInParallel <- function(x) { ) }, error = function(e) { - rlang::inform(e) + rlang::inform(e$message) return(FALSE) } ) diff --git a/inst/sql/sql_server/ConceptCountsDuring.sql b/inst/sql/sql_server/ConceptCountsDuring.sql index e10ebbd..23d7344 100644 --- a/inst/sql/sql_server/ConceptCountsDuring.sql +++ b/inst/sql/sql_server/ConceptCountsDuring.sql @@ -1,7 +1,7 @@ -- adding line below to prevent warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@row_id_field' as cname into #fake; +SELECT '@row_id_field' as cname into #fake WHERE 1 = 1; -- Feature construction {@aggregated} ? { diff --git a/inst/sql/sql_server/DomainConceptDuring.sql b/inst/sql/sql_server/DomainConceptDuring.sql index 427d91c..669059e 100644 --- a/inst/sql/sql_server/DomainConceptDuring.sql +++ b/inst/sql/sql_server/DomainConceptDuring.sql @@ -1,7 +1,7 @@ -- adding lines below to prevent warning IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@domain_end_date' as cname into #fake; +SELECT '@domain_end_date' as cname into #fake WHERE 1=1; -- Feature construction SELECT diff --git a/inst/sql/sql_server/DomainConceptGroupDuring.sql b/inst/sql/sql_server/DomainConceptGroupDuring.sql index 22407b0..a8bf352 100644 --- a/inst/sql/sql_server/DomainConceptGroupDuring.sql +++ b/inst/sql/sql_server/DomainConceptGroupDuring.sql @@ -1,7 +1,7 @@ -- add dummy code with all imputs to stop annoying warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@domain_end_date' as cname into #fake; +SELECT '@domain_end_date' as cname into #fake WHERE 1 = 1; IF OBJECT_ID('tempdb..#groups', 'U') IS NOT NULL DROP TABLE #groups; From 614f426973dc21d558dc698e3370063a9f91184d Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 2 Sep 2025 11:40:56 -0400 Subject: [PATCH 34/35] One more SQL fix --- inst/sql/sql_server/ConceptCountsDuring.sql | 2 +- inst/sql/sql_server/DomainConceptDuring.sql | 2 +- inst/sql/sql_server/DomainConceptGroupDuring.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inst/sql/sql_server/ConceptCountsDuring.sql b/inst/sql/sql_server/ConceptCountsDuring.sql index 23d7344..e266de8 100644 --- a/inst/sql/sql_server/ConceptCountsDuring.sql +++ b/inst/sql/sql_server/ConceptCountsDuring.sql @@ -1,7 +1,7 @@ -- adding line below to prevent warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@row_id_field' as cname into #fake WHERE 1 = 1; +SELECT '@row_id_field' as cname into #fake FROM @cdm_database_schema.@domain_table WHERE 1 = 0; -- Feature construction {@aggregated} ? { diff --git a/inst/sql/sql_server/DomainConceptDuring.sql b/inst/sql/sql_server/DomainConceptDuring.sql index 669059e..ae6681f 100644 --- a/inst/sql/sql_server/DomainConceptDuring.sql +++ b/inst/sql/sql_server/DomainConceptDuring.sql @@ -1,7 +1,7 @@ -- adding lines below to prevent warning IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@domain_end_date' as cname into #fake WHERE 1=1; +SELECT '@domain_end_date' as cname into #fake FROM @cdm_database_schema.@domain_table WHERE 1 = 0; -- Feature construction SELECT diff --git a/inst/sql/sql_server/DomainConceptGroupDuring.sql b/inst/sql/sql_server/DomainConceptGroupDuring.sql index a8bf352..1fb9b1d 100644 --- a/inst/sql/sql_server/DomainConceptGroupDuring.sql +++ b/inst/sql/sql_server/DomainConceptGroupDuring.sql @@ -1,7 +1,7 @@ -- add dummy code with all imputs to stop annoying warnings IF OBJECT_ID('tempdb..#fake', 'U') IS NOT NULL DROP TABLE #fake; -SELECT '@domain_end_date' as cname into #fake WHERE 1 = 1; +SELECT '@domain_end_date' as cname into #fake FROM @cdm_database_schema.@domain_table WHERE 1 = 0; IF OBJECT_ID('tempdb..#groups', 'U') IS NOT NULL DROP TABLE #groups; From a4eeacb42cfab1db1040919306f05bf54e5dfbd4 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 9 Sep 2025 08:59:40 -0400 Subject: [PATCH 35/35] Update ViewShiny.R updating code that extracts ids in prepare for shiny to work with DatabaseConnector v7 --- R/ViewShiny.R | 54 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/R/ViewShiny.R b/R/ViewShiny.R index ad1cf57..d5eb4ff 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -116,12 +116,36 @@ prepareCharacterizationShiny <- function( if (!"cg_cohort_definition" %in% tables) { cohortIds <- unique( c( - DatabaseConnector::querySql(con, paste0("select distinct TARGET_COHORT_ID from ", tablePrefix, csvTablePrefix, "cohort_details where COHORT_TYPE = 'Target';"))$TARGET_COHORT_ID, - DatabaseConnector::querySql(con, paste0("select distinct OUTCOME_COHORT_ID from ", tablePrefix, csvTablePrefix, "cohort_details where COHORT_TYPE = 'TnO';"))$OUTCOME_COHORT_ID, - DatabaseConnector::querySql(con, paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"))$TARGET_COHORT_DEFINITION_ID, - DatabaseConnector::querySql(con, paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"))$OUTCOME_COHORT_DEFINITION_ID, - DatabaseConnector::querySql(con, paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"))$TARGET_COHORT_DEFINITION_ID, - DatabaseConnector::querySql(con, paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"))$OUTCOME_COHORT_DEFINITION_ID + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_COHORT_ID from ", tablePrefix, csvTablePrefix, "cohort_details where COHORT_TYPE = 'Target';"), + snakeCaseToCamelCase = TRUE + )$targetCohortId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_COHORT_ID from ", tablePrefix, csvTablePrefix, "cohort_details where COHORT_TYPE = 'TnO';"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$targetCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), + snakeCaseToCamelCase = TRUE + )$targetCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortDefinitionId ) ) @@ -146,9 +170,21 @@ prepareCharacterizationShiny <- function( if (!"database_meta_data" %in% tables) { dbIds <- unique( c( - DatabaseConnector::querySql(con, paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "analysis_ref;"))$DATABASE_ID, - DatabaseConnector::querySql(con, paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "dechallenge_rechallenge;"))$DATABASE_ID, - DatabaseConnector::querySql(con, paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"))$DATABASE_ID + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "analysis_ref;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "dechallenge_rechallenge;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$databaseId ) )