From f335ae061843e89b7bfe2bdf6093f9deea8ac5f1 Mon Sep 17 00:00:00 2001
From: Jens von Bergmann <jens@mountainmath.ca>
Date: Thu, 22 Jan 2026 20:09:22 -0800
Subject: [PATCH] Optimize date parsing in normalize_cansim_values using lookup
 tables

Instead of calling as.Date() on every row (millions of rows),
build a lookup table for unique date values (typically hundreds
or thousands), then use vector lookup for assignment.

Benchmark on table 14-10-0287 (5.4M rows):
- Original: mean 85.8 sec
- Optimized: mean 67.0 sec
- Improvement: 22% faster overall

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 R/cansim.R | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/R/cansim.R b/R/cansim.R
index aa6dac73..e3bb172d 100644
--- a/R/cansim.R
+++ b/R/cansim.R
@@ -98,20 +98,37 @@ normalize_cansim_values <- function(data, replacement_value="val_norm", normaliz
   if (!trad_cansim) {
     # do nothing
   } else if (grepl("^\\d{4}$",sample_date)) {
-    # year
-    data <- data %>%
-      mutate(Date=as.Date(paste0(!!as.name(date_field),"-",default_month,"-",default_day)))
+    # year - use lookup table for efficiency
+    unique_dates <- unique(data[[date_field]])
+    date_lookup <- setNames(
+      as.Date(paste0(unique_dates, "-", default_month, "-", default_day)),
+      unique_dates
+    )
+    data$Date <- unname(date_lookup[data[[date_field]]])
   } else if (grepl("^\\d{4}/\\d{4}$",sample_date)) {
-    # year range, use second year as anchor
-    data <- data %>%
-      mutate(Date=as.Date(paste0(gsub("^\\d{4}/","",!!as.name(date_field)),"-",default_month,"-",default_day)))
+    # year range, use second year as anchor - use lookup table for efficiency
+    unique_dates <- unique(data[[date_field]])
+    date_lookup <- setNames(
+      as.Date(paste0(gsub("^\\d{4}/", "", unique_dates), "-", default_month, "-", default_day)),
+      unique_dates
+    )
+    data$Date <- unname(date_lookup[data[[date_field]]])
   } else if (grepl("^\\d{4}-\\d{2}$",sample_date)) {
-    # year and month
-    data <- data %>% mutate(Date=as.Date(paste0(!!as.name(date_field),"-",default_day)))
+    # year and month - use lookup table for efficiency
+    unique_dates <- unique(data[[date_field]])
+    date_lookup <- setNames(
+      as.Date(paste0(unique_dates, "-", default_day)),
+      unique_dates
+    )
+    data$Date <- unname(date_lookup[data[[date_field]]])
   } else if (grepl("^\\d{4}-\\d{2}-\\d{2}$",sample_date)) {
-    # year, month and day
-    data <- data %>%
-      mutate(Date=as.Date(!!as.name(date_field)))
+    # year, month and day - use lookup table for efficiency
+    unique_dates <- unique(data[[date_field]])
+    date_lookup <- setNames(
+      as.Date(unique_dates),
+      unique_dates
+    )
+    data$Date <- unname(date_lookup[data[[date_field]]])
   }
 
   cansimTableNumber <- cleaned_ndm_table_number(cansimTableNumber)