From bb960a1645aa57a60e94b96b4ffe8ab50d9bf384 Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Tue, 16 Sep 2025 10:48:06 -0700 Subject: [PATCH 01/13] better connection error handling --- DESCRIPTION | 2 +- R/cansim.R | 8 ++++++++ R/cansim_vectors.R | 14 ++++++++++++++ README.md | 4 ++-- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index f6fd0b04..cb0e1cbf 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: cansim Type: Package Title: Accessing Statistics Canada Data Table and Vectors -Version: 0.4.4 +Version: 0.4.5 Authors@R: c( person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut","cre")), person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut"))) diff --git a/R/cansim.R b/R/cansim.R index fbef3204..aa6dac73 100644 --- a/R/cansim.R +++ b/R/cansim.R @@ -929,6 +929,10 @@ get_cansim_table_url <- function(cansimTableNumber, language = "en"){ l <- cleaned_ndm_language(language) %>% substr(1,2) url=paste0("https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV/",naked_ndm_table_number(cansimTableNumber),"/",l) response <- httr::GET(url) + if (is.null(response)){return(response)} + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } @@ -976,6 +980,10 @@ get_cansim_changed_tables <- function(start_date,end_date=NULL){ lapply(function(date){ url=paste0("https://www150.statcan.gc.ca/t1/wds/rest/getChangedCubeList/",strftime(date,"%Y-%m-%d")) response <- httr::GET(url) + if (is.null(response)){return(response)} + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } diff --git a/R/cansim_vectors.R b/R/cansim_vectors.R index 4945d21c..0af34c30 100644 --- a/R/cansim_vectors.R +++ b/R/cansim_vectors.R @@ -207,6 +207,9 @@ get_cansim_vector<-function(vectors, start_time = as.Date("1800-01-01"), end_tim timeout = timeout) } if (is.null(response)) return(response) + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } @@ -317,6 +320,9 @@ get_cansim_vector_for_latest_periods<-function(vectors, periods=NULL, message(paste0("Accessing CANSIM NDM vectors from Statistics Canada",addition)) response <- post_with_timeout_retry(url, body=vectors_string, timeout = timeout) if (is.null(response)) return(response) + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } @@ -457,6 +463,10 @@ get_cansim_data_for_table_coord_periods<-function(tableCoordinates, periods=NULL } message(paste0("Accessing CANSIM NDM coordinates from Statistics Canada",addition)) response <- post_with_timeout_retry(url, body=body_string, timeout = timeout) + if (is.null(response)) {return(response)} + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } @@ -564,6 +574,10 @@ get_cansim_vector_info <- function(vectors){ url="https://www150.statcan.gc.ca/t1/wds/rest/getSeriesInfoFromVector" vectors_string=paste0("[",paste(purrr::map(as.character(vectors),function(x)paste0('{"vectorId":',x,'}')),collapse = ", "),"]") response <- post_with_timeout_retry(url, body=vectors_string) + if (is.null(response)){return(response)} + if (is.null(response$status_code)) { + stop("Problem downloading data.\n",response$error,call.=FALSE) + } if (response$status_code!=200) { stop("Problem downloading data, status code ",response$status_code,"\n",httr::content(response),call.=FALSE) } diff --git a/README.md b/README.md index 0e73634b..70008548 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ If you want to get in touch, we are pretty good at responding via email or via t If you wish to cite the `cansim` package in your work: - von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.4. DOI: 10.32614/CRAN.package.cansim + von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.5. DOI: 10.32614/CRAN.package.cansim A BibTeX entry for LaTeX users is @@ -241,7 +241,7 @@ A BibTeX entry for LaTeX users is title = {cansim: functions and convenience tools for accessing Statistics Canada data tables}, year = {2025}, doi = {10.32614/CRAN.package.cansim}, - note = {R package version 0.4.4}, + note = {R package version 0.4.5}, url = {https://mountainmath.github.io/cansim/} } ``` From 1bef2594dae4aace32bc919f53e8d0b61cee6c46 Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Tue, 16 Sep 2025 10:48:53 -0700 Subject: [PATCH 02/13] new and cran comments --- NEWS.md | 4 ++++ cran-comments.md | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/NEWS.md b/NEWS.md index b312c8b1..176e2b9d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# cansim 0.4.5 +## Minor changes +* better connection error handling + # cansim 0.4.4 ## Minor changes * fix a problem with metadata parsing does not work properly for table names diff --git a/cran-comments.md b/cran-comments.md index 56481467..6b59b4a9 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -151,3 +151,8 @@ There were no ERRORs or WARNINGs or NOTEs. * fix a problem with metadata parsing does not work properly for table names * make documentations more consistent wrt default langauge names * add convenience functions for setting cache paths for data accessed via get_cansim_connection + +# cansim 0.4.5 +## Minor changes +* better connection error handling + From adb42bc83bc15d974a21c132986965d2e97a12e0 Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 08:18:38 -0800 Subject: [PATCH 03/13] more improvements on error handling --- R/cansim_helpers.R | 4 ++++ R/cansim_tables_list.R | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/R/cansim_helpers.R b/R/cansim_helpers.R index 2a433aa9..0282dc62 100644 --- a/R/cansim_helpers.R +++ b/R/cansim_helpers.R @@ -279,6 +279,10 @@ get_cansim_code_set <- function(code_set=c("scalar", "frequency", "symbol", "sta if (refresh | !file.exists(path)) { url='https://www150.statcan.gc.ca/t1/wds/rest/getCodeSets' r<-get_with_timeout_retry(url) + if (is.null(r)||is.null(r$status_code)){ + warning("Problem downloading code sets.") + return(NULL) + } if (r$status_code==200) { content <- httr::content(r) saveRDS(content,path) diff --git a/R/cansim_tables_list.R b/R/cansim_tables_list.R index 9c184a82..66ac7835 100644 --- a/R/cansim_tables_list.R +++ b/R/cansim_tables_list.R @@ -89,6 +89,10 @@ list_cansim_cubes <- function(lite=FALSE,refresh=FALSE,quiet=FALSE){ if (!quiet) message("Retrieving cube information from StatCan servers...") url=ifelse(lite,"https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite","https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList") r<-get_with_timeout_retry(url,retry=0,warn_only=TRUE) + if (is.null(r)||is.null(r$status_code)){ + warning("Could not retrieve cube list from StatCan servers.") + return(NULL) + } if (r$status_code==200) { content <- httr::content(r) From 1288fb1c8ed32a5bb5f719e0a52c8fc88adc451f Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 08:45:03 -0800 Subject: [PATCH 04/13] fail more gracefully when statcan servers are down and cube list can't be accessed --- R/cansim_tables_list.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/R/cansim_tables_list.R b/R/cansim_tables_list.R index 66ac7835..24fb237c 100644 --- a/R/cansim_tables_list.R +++ b/R/cansim_tables_list.R @@ -177,7 +177,11 @@ list_cansim_cubes <- function(lite=FALSE,refresh=FALSE,quiet=FALSE){ #' #' @export search_cansim_cubes <- function(search_term, refresh=FALSE){ - list_cansim_cubes(refresh = refresh) %>% + cube_list <- list_cansim_cubes(refresh = refresh) + if (is.null(cube_list)) { + stop("Could not retrieve cube list from StatCan servers.",call.=FALSE) + } + cube_list %>% filter(grepl(search_term,.data$cubeTitleEn,ignore.case = TRUE) | grepl(search_term,.data$cubeTitleFr,ignore.case = TRUE) | grepl(search_term,.data$surveyEn,ignore.case = TRUE) | From 6eeb3d4156589befd61e42a6d693c1293815c95c Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 09:23:26 -0800 Subject: [PATCH 05/13] tidyselect tweaks --- R/cansim_vectors.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/cansim_vectors.R b/R/cansim_vectors.R index 0af34c30..eb2a534b 100644 --- a/R/cansim_vectors.R +++ b/R/cansim_vectors.R @@ -417,7 +417,7 @@ get_cansim_data_for_table_coord_periods<-function(tableCoordinates, periods=NULL if ("list" %in% class(tableCoordinates)) { tableCoordinates <- tibble::enframe(tableCoordinates) %>% setNames(c("cansimTableNumber","COORDINATE")) %>% - tidyr::unnest_longer(.data$COORDINATE) + tidyr::unnest_longer("COORDINATE") } tableCoordinates <- tableCoordinates %>% mutate(cansimTableNumber=naked_ndm_table_number(.data$cansimTableNumber)) %>% From 99a4db3c281eeb1044e826249a47d6cf6c96066b Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 11:06:50 -0800 Subject: [PATCH 06/13] fix doi badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70008548..56637364 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![CRAN status](https://www.r-pkg.org/badges/version/cansim)](https://CRAN.R-project.org/package=cansim) [![CRAN_Downloads_Badge](https://cranlogs.r-pkg.org/badges/cansim)](https://cranlogs.r-pkg.org/badges/cansim) [![R-CMD-check](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml) -[![DOI](https://img.shields.io/badge/doi-10.32614/CRAN.package.cansim-#d2b24a.svg)](https://doi.org/10.32614/CRAN.package.cansim) +[![DOI](https://img.shields.io/badge/doi-10.32614-d2b24a.svg](https://doi.org/10.32614/CRAN.package.cansim) cansim logo From 6cf8f57ad8aa32209f64d99e9f7266569424806b Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 11:11:41 -0800 Subject: [PATCH 07/13] fix dio badge --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 56637364..465fd4e6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![CRAN status](https://www.r-pkg.org/badges/version/cansim)](https://CRAN.R-project.org/package=cansim) [![CRAN_Downloads_Badge](https://cranlogs.r-pkg.org/badges/cansim)](https://cranlogs.r-pkg.org/badges/cansim) [![R-CMD-check](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml) -[![DOI](https://img.shields.io/badge/doi-10.32614-d2b24a.svg](https://doi.org/10.32614/CRAN.package.cansim) +[![DOI](https://img.shields.io/badge/ DOI-10.32614/CRAN.package.cansim-d2b24a.svg)](https://doi.org/10.32614/CRAN.package.cansim) cansim logo @@ -231,7 +231,7 @@ If you want to get in touch, we are pretty good at responding via email or via t If you wish to cite the `cansim` package in your work: - von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.5. DOI: 10.32614/CRAN.package.cansim + von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.4. DOI: 10.32614/CRAN.package.cansim A BibTeX entry for LaTeX users is @@ -241,7 +241,7 @@ A BibTeX entry for LaTeX users is title = {cansim: functions and convenience tools for accessing Statistics Canada data tables}, year = {2025}, doi = {10.32614/CRAN.package.cansim}, - note = {R package version 0.4.5}, + note = {R package version 0.4.4}, url = {https://mountainmath.github.io/cansim/} } ``` From e3b4e4bdd37adc02aad696ab759c02d1940e65c0 Mon Sep 17 00:00:00 2001 From: Jens von Bergmann Date: Sat, 15 Nov 2025 11:24:25 -0800 Subject: [PATCH 08/13] toggle visibility of main logo on pkgdown site --- README.md | 7 +- docs/404.html | 2 +- docs/LICENSE-text.html | 2 +- docs/LICENSE.html | 2 +- docs/articles/cansim.html | 2 +- docs/articles/index.html | 2 +- docs/articles/listing_cansim_tables.html | 2 +- .../figure-html/unnamed-chunk-4-1.png | Bin 157608 -> 157621 bytes .../articles/partial_table_data_download.html | 2 +- docs/articles/retrieving_cansim_vectors.html | 30 +++--- .../figure-html/unnamed-chunk-4-1.png | Bin 89244 -> 89360 bytes docs/articles/working_with_hierarchies.html | 4 +- docs/articles/working_with_large_tables.html | 86 +++++++++--------- .../figure-html/unnamed-chunk-10-1.png | Bin 98839 -> 98844 bytes docs/authors.html | 2 +- docs/index.html | 17 ++-- docs/news/index.html | 11 ++- docs/pkgdown.yml | 4 +- .../add_cansim_vectors_to_template.html | 4 +- .../add_provincial_abbreviations.html | 2 +- docs/reference/cansim_old_to_new.html | 2 +- .../cansim_repartition_cached_table.html | 2 +- docs/reference/categories_for_level.html | 2 +- docs/reference/collect_and_normalize.html | 2 +- docs/reference/correspondence.html | 2 +- docs/reference/create_index.html | 2 +- docs/reference/csv2arrow.html | 2 +- docs/reference/csv2sqlite.html | 2 +- docs/reference/disconnect_cansim_sqlite.html | 2 +- .../fold_in_metadata_for_columns.html | 2 +- docs/reference/get_cansim.html | 2 +- docs/reference/get_cansim_changed_tables.html | 2 +- docs/reference/get_cansim_code_set.html | 2 +- .../get_cansim_column_categories.html | 2 +- docs/reference/get_cansim_column_list.html | 2 +- docs/reference/get_cansim_connection.html | 2 +- docs/reference/get_cansim_cube_metadata.html | 2 +- ...t_cansim_data_for_table_coord_periods.html | 2 +- .../get_cansim_key_release_schedule.html | 2 +- .../get_cansim_series_info_cube_coord.html | 2 +- docs/reference/get_cansim_sqlite.html | 2 +- docs/reference/get_cansim_table_info.html | 2 +- .../get_cansim_table_last_release_date.html | 2 +- docs/reference/get_cansim_table_notes.html | 2 +- docs/reference/get_cansim_table_overview.html | 2 +- .../get_cansim_table_short_notes.html | 2 +- docs/reference/get_cansim_table_subject.html | 2 +- docs/reference/get_cansim_table_survey.html | 2 +- docs/reference/get_cansim_table_template.html | 2 +- docs/reference/get_cansim_table_url.html | 2 +- docs/reference/get_cansim_vector.html | 2 +- .../get_cansim_vector_for_latest_periods.html | 2 +- docs/reference/get_cansim_vector_info.html | 2 +- .../get_deduped_column_level_data.html | 2 +- docs/reference/index.html | 2 +- docs/reference/list_cansim_cached_tables.html | 2 +- docs/reference/list_cansim_cubes.html | 2 +- .../list_cansim_sqlite_cached_tables.html | 2 +- docs/reference/list_cansim_tables.html | 2 +- docs/reference/normalize_cansim_values.html | 2 +- docs/reference/parse_metadata.html | 2 +- .../remove_cansim_cached_tables.html | 2 +- .../remove_cansim_sqlite_cached_table.html | 2 +- docs/reference/search_cansim_cubes.html | 2 +- docs/reference/search_cansim_tables.html | 2 +- docs/reference/set_cansim_cache_path.html | 2 +- docs/reference/show_cansim_cache_path.html | 5 +- docs/reference/view_cansim_webpage.html | 2 +- docs/search.json | 2 +- 69 files changed, 148 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 465fd4e6..a03f0625 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,12 @@ [![DOI](https://img.shields.io/badge/ DOI-10.32614/CRAN.package.cansim-d2b24a.svg)](https://doi.org/10.32614/CRAN.package.cansim) -cansim logo + + An R package to retrieve and work with public Statistics Canada data tables. diff --git a/docs/404.html b/docs/404.html index c73f2820..d0ec05d7 100644 --- a/docs/404.html +++ b/docs/404.html @@ -38,7 +38,7 @@ cansim - 0.4.4 + 0.4.5