test(dwh-migration): setup test suite to validate DWH migration

.Rprofile

@@ -1,7 +1,9 @@
 options(test.run_snapshot = FALSE)
 options(test.run_data_validation = FALSE)
+options(test.run_dwh_migration = FALSE)
 
 message("ℹ️ test: snapshot and data validation tests are default set to 'RUN = FALSE'")
 message("ℹ️ Run options(test.run_snapshot = TRUE) in the console to activate the snaphot tests.")
 message("ℹ️ Run options(test.run_data_validation = TRUE) in the console to activate the data validation tests.")
+message("ℹ️ Run options(test.run_dwh_migration = TRUE) in the console to activate the data warehouse migration tests.")
 message("ℹ️ See the testing README for more information.")

tests/testthat/README.md

@@ -16,7 +16,7 @@ Since untracked local snapshot files persist across git branch switches, you can
 
 ### 1. Set up a database connection
 
-Create a database connection with the variable name 'test_con'. If this variable is not found, the snapshot tests will be skipped.
+Create a database connection with the variable name `test_con`. If this variable is not found, the snapshot tests will be skipped.
 
 ``` r
 test_con <- connect_watina()
@@ -26,13 +26,19 @@ test_con <- connect_watina()
 
 *When reviewing a PR, switch to the stable main branch (`git switch main`).*
 
-Run the snapshot tests:
+Run the standard snapshot tests:
 
 ``` r
 devtools::test(filter = "get")
 ```
 
-This will create local snapshot files that are stored in tests/testthat/\_snaps.
+Or, if you are verifying a data warehouse migration, run the migration tests:
+
+``` r
+devtools::test(filter = "dwh-migration")
+```
+
+The tests will create local snapshot files that are stored in `tests/testthat/_snaps`.
 
 ### 3. Modify your code
 
@@ -43,7 +49,11 @@ Switch to your feature branch if you're reviewing a PR or apply your changes whe
 Execute the test code with the applied changes:
 
 ``` r
+# standard snapshot tests
 devtools::test(filter = "get")
+
+# data warehouse migration tests
+devtools::test(filter = "dwh-migration")
 ```
 
 **If the tests pass**: The data generated by the modified code is a perfect structural match with your baseline.
@@ -56,6 +66,7 @@ Testing behavior is managed dynamically through R session `options()`. When you
 
 - **test.run_snapshot** (*default is FALSE*): Toggle to test database query changes. When FALSE, database snapshot tests are skipped.
 - **test.run_data_validation** (*default is FALSE*): Toggle to execute extensive end-to-end table checks (e.g., retrieving all historical rows). Keep FALSE unless verifying a major data warehouse migration.
+- **test.run_dwh_migration** (*default is FALSE*): Toggle to execute tests specifically written to evaluate data warehouse changes. These tests include a static validation set with locations that are unlikely to have data changes in the future. Keep FALSE unless verifying a data warehouse change.
 
 You can flip these switches directly in your RStudio console at any time. Never modify the default settings in the `.Rprofile`!
 
@@ -66,9 +77,13 @@ options(test.run_snapshot = TRUE)
 # Activate extensive data validation tests
 options(test.run_data_validation = TRUE)
 
+# Activate data warehouse migration tests
+options(test.run_dwh_migration = TRUE)
+
 # Return to default mode
 options(test.run_snapshot = FALSE)
 options(test.run_data_validation = FALSE)
+options(test.run_dwh_migration = FALSE)
 ```
 
-💡 Under the hood: When you run `devtools::test()`, `setup.R` automatically reads these console options. If no active database connection (`test_con`) is found in your Global Environment, database tests will always automatically skip for safety (e.g., on GitHub Actions).
+💡 Under the hood: When you run `devtools::test()`, `setup.R` automatically reads these console options and maps them to internal skip flags (e.g. converting `test.run_snapshot = FALSE` into `test.skip_snapshot = TRUE`). If no active database connection (`test_con`) is found in your Global Environment, database tests will always automatically skip for safety (e.g., on GitHub Actions).

tests/testthat/helper-get.R → tests/testthat/helper.R

@@ -1,3 +1,4 @@
+# Watina connection for testing ------------------------------------------------
 fetch_watina_connection <- function() {
   skip_if_not(
     exists("test_con", envir = .GlobalEnv),
@@ -10,30 +11,13 @@ fetch_watina_connection <- function() {
   return(get("test_con", envir = .GlobalEnv))
 }
 
+# Clean up snapshot files ------------------------------------------------------
 create_file_name <- function(function_name, test_name) {
   file_end <- "data.csv"
 
   paste0(function_name, "_", test_name, "_", file_end)
 }
 
-announce_files <- function(file_names, function_name) {
-  for (f in file_names) {
-    announce_snapshot_file(name = create_file_name(function_name, f))
-  }
-}
-
-announce_files_locs <- function(file_names) {
-  announce_files(file_names, "locs")
-}
-
-announce_files_xg3 <- function(file_names) {
-  announce_files(file_names, "xg3")
-}
-
-announce_files_chem <- function(file_names) {
-  announce_files(file_names, "chem")
-}
-
 clean_up_table <- function(table) {
   floor_0 <- c(
     "x",
@@ -59,7 +43,9 @@ clean_up_table <- function(table) {
     arrange(
       pick(
         ends_with("_code"),
+        ends_with("_id"),
         ends_with("year"),
+        ends_with("date"),
         ends_with("_ost"),
         ends_with("_lcl"),
         ends_with("_variable")
@@ -69,13 +55,34 @@ clean_up_table <- function(table) {
   return(table)
 }
 
-write_file <- function(path, table) {
-  write.csv(clean_up_table(table), path, row.names = FALSE)
+# Announce snapshot files ------------------------------------------------------
+# Assure snapshot files are not deleted when tests are skipped
+announce_files <- function(file_names, function_name) {
+  for (f in file_names) {
+    announce_snapshot_file(name = create_file_name(function_name, f))
+  }
+}
+
+announce_files_locs <- function(file_names) {
+  announce_files(file_names, "locs")
+}
+
+announce_files_xg3 <- function(file_names) {
+  announce_files(file_names, "xg3")
+}
+
+announce_files_chem <- function(file_names) {
+  announce_files(file_names, "chem")
+}
+
+announce_files_migration <- function(file_names) {
+  announce_files(file_names, "dwh_migration")
 }
 
+# Expect snapshot tests --------------------------------------------------------
 expect_data <- function(data, function_name, test_name) {
   path <- tempfile(fileext = ".csv")
-  write_file(path, data)
+  write.csv(clean_up_table(data), path, row.names = FALSE)
   expect_snapshot_file(path, create_file_name(function_name, test_name))
 }
 
@@ -90,3 +97,63 @@ expect_xg3 <- function(xg3, test_name) {
 expect_chem <- function(chem, test_name) {
   expect_data(chem, "chem", test_name)
 }
+
+expect_migration <- function(data, test_name) {
+  expect_data(data, "dwh_migration", test_name)
+}
+
+# Settings for DWH migration tests ---------------------------------------------
+# Selected locations for DWH migration testing:
+#   - location is locked (Status = 'afgesloten')
+#   - with a sufficient amount of observations (COUNT(peilpunten) >= 1000)
+dwh_test_locations <- c(
+  "BLAP028",
+  "BOBP020",
+  "BGNP008",
+  "BGNP009",
+  "BGNP011",
+  "BGNP012",
+  "BGNP020",
+  "BGNP021",
+  "BGNP025",
+  "BGNP028",
+  "BGNP031",
+  "BGNP035",
+  "BGNP036",
+  "CABP005",
+  "BRSP005",
+  "DUNS002",
+  "DUNS003",
+  "DURP004",
+  "GRMP013",
+  "GWZP010",
+  "IJSP032",
+  "KAMP002",
+  "KAMP041",
+  "KAMP206",
+  "KAMS021",
+  "KALP144",
+  "KALP154",
+  "KRGP038",
+  "KRGP039",
+  "POLP002",
+  "POLP006",
+  "KBRP128",
+  "KBRS003",
+  "KBRS007",
+  "KBRS009",
+  "UKPP025",
+  "VLBP021",
+  "VLBP023",
+  "VLBS004",
+  "ZSCP122",
+  "ZSCP132",
+  "ZSCP133",
+  "ZWAS232"
+)
+
+# DWH W0002_10_Watina has only validity VLD
+dwh_test_validity <- c("VLD")
+# Include all loc_types
+dwh_test_types <- c("P", "S", "R", "N", "W", "D", "L", "B")
+

tests/testthat/setup.R

@@ -4,6 +4,10 @@ withr::local_options(
     test.skip_data_validation = !(getOption(
       "test.run_data_validation",
       default = FALSE
+    )),
+    test.skip_dwh_migration = !(getOption(
+      "test.run_dwh_migration",
+      default = FALSE
     ))
   ),
   .local_envir = testthat::teardown_env()