diff --git a/.github/workflows/check-full.yaml b/.github/workflows/check-full.yaml index 7a47123a..8c667fdb 100644 --- a/.github/workflows/check-full.yaml +++ b/.github/workflows/check-full.yaml @@ -6,6 +6,9 @@ on: - cron: '0 0 * * 0' name: R-CMD-check (full) + +permissions: read-all + jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} @@ -27,7 +30,7 @@ jobs: R_KEEP_PKG_SOURCE: yes steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: r-lib/actions/setup-pandoc@v2 @@ -35,7 +38,6 @@ jobs: with: r-version: ${{ matrix.config.r }} http-user-agent: ${{ matrix.config.http-user-agent }} - use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2 with: @@ -44,4 +46,5 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: - upload-snapshots: true \ No newline at end of file + upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml index b60da2fb..45f18b19 100644 --- a/.github/workflows/check-standard.yaml +++ b/.github/workflows/check-standard.yaml @@ -8,6 +8,8 @@ on: name: R-CMD-check (standard) +permissions: read-all + jobs: R-CMD-check: runs-on: ubuntu-latest @@ -15,15 +17,16 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: r-lib/actions/setup-r@v2 - with: - use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::rcmdcheck needs: check - - uses: r-lib/actions/check-r-package@v2 \ No newline at end of file + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/DESCRIPTION b/DESCRIPTION index ea2c8b76..44620fe4 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: eurostat Title: Tools for Eurostat Open Data -Version: 4.0.0 -Date: 2023-12-19 +Version: 4.1.0.9007 +Date: 2026-04-29 Authors@R: c( person("Leo", "Lahti", , "leo.lahti@iki.fi", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-5537-637X")), @@ -10,6 +10,8 @@ Authors@R: c( person("Markus", "Kainu", role = "aut"), person("Przemyslaw", "Biecek", role = "aut"), person("Daniel", "Antal", role = "ctb"), + person("Daniel", "Loos", role = "ctb", + comment = c(ORCID = "0000-0002-4024-4443")), person("Diego", "Hernangomez", role = "ctb", comment = c(ORCID = "0000-0001-8457-4658")), person("Joona", "Lehtomaki", role = "ctb"), @@ -18,8 +20,9 @@ Authors@R: c( person("Paul", "Rougieux", role = "ctb"), person("Anna", "Vasylytsya", role = "ctb"), person("Oliver", "Reiter", role = "ctb"), - person("Pyry", "Kantanen", role = "ctb", + person("Pyry", "Kantanen", role = "aut", comment = c(ORCID = "0000-0003-2853-2765")), + person("Akewak", "Jeba", role = "ctb"), person("Enrico", "Spinielli", , "enrico.spinielli@gmail.com", role = "ctb", comment = c(ORCID = "0000-0001-8584-9131")) ) @@ -38,13 +41,14 @@ Imports: curl, digest, dplyr, - httr2 (>= 0.2.3), + httr2 (>= 1.1.2), + purrr, ISOweek, jsonlite, lubridate, + R.utils, rappdirs, readr, - RefManageR, regions, rlang, stringi, @@ -52,13 +56,14 @@ Imports: tibble, tidyr (>= 1.0.0), xml2, - data.table (>= 1.14.8) + data.table (>= 1.18.2.1) Suggests: - giscoR, + giscoR (>= 1.1.0), knitr, rmarkdown, sf, - testthat (>= 3.0.0) + testthat (>= 3.0.0), + lifecycle VignetteBuilder: knitr Config/Needs/website: ggplot2, tmap, styler, sessioninfo, @@ -71,6 +76,6 @@ MailingList: rOpenGov NeedsCompilation: no Repository: CRAN Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.3 X-schema.org-isPartOf: http://ropengov.org/ X-schema.org-keywords: ropengov diff --git a/LICENSE b/LICENSE index 1de69c39..0325ecb1 100755 --- a/LICENSE +++ b/LICENSE @@ -1,2 +1,2 @@ -YEAR: 2014-2022 +YEAR: 2014-2024 COPYRIGHT HOLDER: Leo Lahti, Janne Huovari, Markus Kainu, Przemyslaw Biecek \ No newline at end of file diff --git a/NAMESPACE b/NAMESPACE index 2ecf819b..61d87b39 100755 --- a/NAMESPACE +++ b/NAMESPACE @@ -14,6 +14,9 @@ export(get_eurostat_folder) export(get_eurostat_geospatial) export(get_eurostat_interactive) export(get_eurostat_json) +export(get_eurostat_link) +export(get_eurostat_local) +export(get_eurostat_sdmx) export(get_eurostat_toc) export(grepEurostatTOC) export(harmonize_country_code) @@ -30,8 +33,7 @@ export(set_eurostat_cache_dir) export(validate_geo_code) export(validate_nuts_regions) importFrom(ISOweek,ISOweek2date) -importFrom(RefManageR,BibEntry) -importFrom(RefManageR,toBiblatex) +importFrom(R.utils,isUrl) importFrom(classInt,classIntervals) importFrom(countrycode,countrycode) importFrom(curl,curl_download) @@ -40,6 +42,7 @@ importFrom(data.table,.SD) importFrom(data.table,fread) importFrom(data.table,melt) importFrom(data.table,setDT) +importFrom(data.table,setcolorder) importFrom(digest,digest) importFrom(dplyr,"%>%") importFrom(dplyr,case_when) @@ -52,9 +55,13 @@ importFrom(httr2,req_error) importFrom(httr2,req_perform) importFrom(httr2,req_proxy) importFrom(httr2,req_retry) +importFrom(httr2,req_url_path_append) +importFrom(httr2,req_url_query) importFrom(httr2,req_user_agent) importFrom(httr2,request) importFrom(httr2,resp_body_json) +importFrom(httr2,resp_body_raw) +importFrom(httr2,resp_check_status) importFrom(httr2,resp_content_type) importFrom(httr2,resp_is_error) importFrom(httr2,url_build) @@ -66,15 +73,19 @@ importFrom(lubridate,dmy) importFrom(lubridate,month) importFrom(lubridate,year) importFrom(lubridate,ymd) +importFrom(purrr,map_dfr) importFrom(readr,col_character) importFrom(readr,cols) +importFrom(readr,read_csv) importFrom(readr,read_tsv) importFrom(regions,recode_nuts) importFrom(regions,validate_geo_code) importFrom(regions,validate_nuts_regions) importFrom(rlang,"!!") +importFrom(rlang,.data) importFrom(rlang,sym) importFrom(stats,na.omit) +importFrom(stats,setNames) importFrom(stringi,stri_extract_first_regex) importFrom(stringi,stri_replace_all_fixed) importFrom(stringi,stri_replace_all_regex) @@ -89,8 +100,14 @@ importFrom(utils,capture.output) importFrom(utils,download.file) importFrom(utils,hasName) importFrom(utils,menu) +importFrom(utils,modifyList) importFrom(utils,person) +importFrom(utils,read.csv) +importFrom(utils,read.delim) importFrom(utils,toBibtex) importFrom(xml2,read_xml) +importFrom(xml2,xml_attr) importFrom(xml2,xml_find_all) +importFrom(xml2,xml_find_first) +importFrom(xml2,xml_ns) importFrom(xml2,xml_text) diff --git a/NEWS.md b/NEWS.md index 22c8436a..4360a9d9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,33 @@ +# eurostat 4.1.0.9006 + +## Minor updates + +* Removed experimental functions from the codebase, fixed tests that were pointing at datasets renamed by Eurostat + +# eurostat 4.1.0.9005 + +## Deprecated and defunct + +* Removed RefManageR dependency from the package due to it being unmaintained + +# eurostat 4.1.0.9004 + +## Major updates + +* Add new function `get_eurostat_sdmx()` for implementing functionalities that are specifically targeted for the new SDMX API. It is accompanied by other helper functions that are related to fetching metadata from the SDMX API: `build_api_base_uri()`, `build_agencyID()`, `data_filtering_on_dimension()`, `get_codelist_id()` and `label_eurostat_sdmx()` +* Add new function `get_eurostat_local()` for reading local SDMX-CSV (.csv) and compressed SDMX-CSV (.csv.gz) files downloaded manually from the Eurostat website. This function relies only on `data.table` functions as it is expected that locally stored files may be larger and require more efficient handling with regards to RAM and processing time constrains. `tidy_eurostat_sdmx()` and `parse_filename()` are helper functions related to this function. +* New `legacy_data_format()` function and `legacy.data.output` attribute in `get_eurostat()` and `get_eurostat_local()` transforms modern API naming conventions into the format that was used in the old API and eurostat package versions 3.8.* and prior. It is a helper function to other functions. +* Add internal functions for retrieving SDMX metadata + +## Minor updates + +* `get_eurostat_geospatial()` documentation mentions that data from 2024 is now available (fixes issue #327 by @yhsong1013) +* Fix mentions to Eurostat base URLs in `check_access_to_data()`, `get_eurostat()`, `get_eurostat_json()`, `get_eurostat_sdmx()`, `label_eurostat()` and `search_eurostat()` (fixes issue #326 by @olivierchantrel) + +## Deprecated and defunct + +* Remove deprecated parameter `make_valid` from `get_eurostat_geospatial()` + # eurostat 4.0.0 ## Major updates @@ -70,7 +100,7 @@ downloading geospatial data (PR #264, thanks to @dieghernan): ## Major updates -* Updated `get_eurostat()` and its assorted functions to download data from the new dissemination API (related to issues #251, #243). See Eurostat web page Transition - from Eurostat Bulk Download to API for a list of differences between old and new data sources: https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API +* Updated `get_eurostat()` and its assorted functions to download data from the new dissemination API (related to issues #251, #243). See Eurostat web page Transition - from Eurostat Bulk Download to API for a list of differences between old and new data sources: https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload * Added new temporary functions for downloading and handling data from the new dissemination API: `get_eurostat_raw2`, `tidy_eurostat2`, `convert_time_col2`, `eurotime2date2`, `eurotime2num2` and `label_eurostat2`. When the old bulk download facilities are decommissioned, these functions will replace the old functions with old naming schemes (without the 2s at the end). * `tidy_eurostat2` function is now able to handle multiple time frequencies in one call: For example, you can download annual, quarterly, and monthly data simply by using a vector c("A", "Q", "M") in select_time instead of using these singular frequencies in separate calls. The function will also return multiple time series in one dataset if select_time is NULL (as it is by default). If the dataset contains multiple time series and these are explicitly downloaded / no select_time parameter is given, a message will be printed. * `eurotime2num` can now handle monthly and weekly data as well. @@ -82,7 +112,7 @@ downloading geospatial data (PR #264, thanks to @dieghernan): # eurostat 3.7.13 (2023-02-01) -* Updated `get_eurostat_json()` to migrate from JSON web service to API Statistics (addressed in issues #243, #251). Please note that the output from JSON API is now slightly different than before: the datasets now contain a freq column to indicate the frequency with which data has been collected, for example annually "A", monthly "M" or quarterly "Q". See Eurostat - Data browser online help website for more information: https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics +* Updated `get_eurostat_json()` to migrate from JSON web service to API Statistics (addressed in issues #243, #251). Please note that the output from JSON API is now slightly different than before: the datasets now contain a freq column to indicate the frequency with which data has been collected, for example annually "A", monthly "M" or quarterly "Q". See Eurostat - Data browser online help website for more information: https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/json * Minor fixes in `get_bibentry()` and `get_eurostat_geospatial()` # eurostat 3.7.12 (2022-06-28) diff --git a/R/check_access_to_data.R b/R/check_access_to_data.R index f5e26776..dedba1ad 100755 --- a/R/check_access_to_data.R +++ b/R/check_access_to_data.R @@ -1,4 +1,4 @@ -#' @title Check access to ec.europe.eu +#' @title Check access to ec.europa.eu #' @description Check if R has access to resources at http://ec.europa.eu #' @author Markus Kainu #' @return a logical. @@ -9,12 +9,14 @@ #' #' @importFrom curl curl_download #' @importFrom utils download.file +#' @importFrom R.utils isUrl #' #' @export check_access_to_data <- function() { temp <- tempfile() http_url <- "https://ec.europa.eu/eurostat/cache/GISCO/distribution/v2/nuts/geojson/NUTS_RG_60M_2006_4326_LEVL_0.geojson" + stopifnot(R.utils::isUrl(http_url)) # If unix use curl::curl_download to test connection # If windows use download.file with default method 'wininet' if (.Platform$OS.type == "unix") { diff --git a/R/data_countries.R b/R/data_countries.R index 7d09d677..6af43ffa 100755 --- a/R/data_countries.R +++ b/R/data_countries.R @@ -2,9 +2,16 @@ #' @description Countries and country codes in EU, #' Euro area, EFTA and EU candidate countries. #' @format A data_frame: -#' * **code**: Country code in the Eurostat database. +#' * **code**: Country code in the Eurostat database (two-letter ISO code (ISO 3166 alpha-2) except in the case of Greece where EL is used). #' * **name**: Country name in English. -#' * **label**: Country name in the Eurostat database. +#' * **label**: Country name in the Eurostat database +#' * **name_fr**: Country name in French +#' * **name_de**: Country name in German +#' * **country_language**: Country name in national language(s). +#' +#' @details +#' Country codes are two-letter ISO codes (ISO 3166 alpha-2) except in the case +#' of Greece where EL is used instead of the standard ISO code. #' #' @family datasets #' @source , diff --git a/R/data_spatial.R b/R/data_spatial.R index a1aacf8a..bc415e4b 100755 --- a/R/data_spatial.R +++ b/R/data_spatial.R @@ -1,63 +1,49 @@ #' Geospatial data of Europe from GISCO in 1:60 million scale from -#' year 2016 +#' year 2024 #' #' @description Geospatial data of Europe from GISCO in 1:60 million scale -#' from year 2016 +#' from year 2024 #' #' @family datasets #' @family geospatial #' @format sf object #' @docType data -#' @name eurostat_geodata_60_2016 -#' -#' @details -#' The dataset contains 2016 observations (rows) and 12 variables (columns). -#' -#' The object contains the following columns: -#' * **id**: JSON id code, the same as **NUTS_ID**. See **NUTS_ID** below for -#' further clarification. -#' * **LEVL_CODE**: NUTS level code: 0 (national level), 1 (major -#' socio-economic regions), 2 (basic regions for the application of regional -#' policies) or 3 (small regions). -#' * **NUTS_ID**: NUTS ID code, consisting of country code and numbers (1 for -#' NUTS 1, 2 for NUTS 2 and 3 for NUTS 3) -#' * **CNTR_CODE**: Country code: two-letter ISO code (ISO 3166 alpha-2), except -#' in the case of Greece (EL). -#' * **NAME_LATN**: NUTS name in local language, transliterated to Latin script -#' * **NUTS_NAME**: NUTS name in local language, in local script. -#' * **MOUNT_TYPE**: Mountain typology for NUTS 3 regions. -#' * 1: "where more than 50 % of the surface is covered by topographic -#' mountain areas" -#' * 2: "in which more than 50 % of the regional population lives in -#' topographic mountain areas" -#' * 3: "where more than 50 % of the surface is covered by topographic -#' mountain areas and where more than 50 % of the regional population lives -#' in these mountain areas" -#' * 4: non-mountain region / other region -#' * 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 and -#' non-EU countries) -#' * **URBN_TYPE**: Urban-rural typology for NUTS 3 regions. -#' * 1: predominantly urban region -#' * 2: intermediate region -#' * 3: predominantly rural region -#' * 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 -#' regions) -#' * **COAST_TYPE**: Coastal typology for NUTS 3 regions. -#' * 1: coastal (on coast) -#' * 2: coastal (>= 50% of population living within 50km of the coastline) -#' * 3: non-coastal region -#' * 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 -#' regions) -#' * **FID**: Same as NUTS_ID. -#' * **geo**: Same as NUTS_ID, added for for easier joins with dplyr. However, -#' it is recommended to use other identical fields for this purpose. -#' * **geometry**: geospatial information. -#' -#' Dataset updated: 2023-06-29. For a more recent version, please use -#' [giscoR::gisco_get_nuts()] function. +#' @name eurostat_geodata_60_2024 +#' +#' @format The dataset contains 2016 observations (rows) and 12 variables (columns): +#' \describe{ +#' \item{id}{JSON id code, the same as NUTS_ID. See NUTS_ID below for further clarification.} +#' \item{LEVL_CODE}{NUTS level code: 0 (national level), 1 (major socio-economic regions), 2 (basic regions for the application of regional policies) or 3 (small regions).} +#' \item{NUTS_ID}{NUTS ID code, consisting of country code and numbers (1 for NUTS 1, 2 for NUTS 2 and 3 for NUTS 3)} +#' \item{CNTR_CODE}{Country code: two-letter ISO code (ISO 3166 alpha-2), except in the case of Greece (EL).} +#' \item{NAME_LATN}{NUTS name in local language, transliterated to Latin script} +#' \item{NUTS_NAME}{NUTS name in local language, in local script.} +#' \item{MOUNT_TYPE}{Mountain typology for NUTS 3 regions. +#' 1: "where more than 50 % of the surface is covered by topographic mountain areas" +#' 2: "in which more than 50 % of the regional population lives in topographic mountain areas" +#' 3: "where more than 50 % of the surface is covered by topographic mountain areas and where more than 50 % of the regional population lives in these mountain areas" +#' 4: non-mountain region / other region +#' 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 and non-EU countries)} +#' \item{URBN_TYPE}{Urban-rural typology for NUTS 3 regions. +#' 1: predominantly urban region +#' 2: intermediate region +#' 3: predominantly rural region +#' 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 regions)} +#' \item{COAST_TYPE}{Coastal typology for NUTS 3 regions. +#' 1: coastal (on coast) +#' 2: coastal (>= 50% of population living within 50km of the coastline) +#' 3: non-coastal region +#' 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 regions)} +#' \item{FID}{Same as NUTS_ID} +#' \item{geo}{Same as NUTS_ID} +#' \item{geometry}{geospatial information} +#' } +#' +#' Dataset updated: 2023-06-29. For a possibly more recent version, please use +#' \CRANpkg{giscoR} function `gisco_get_nuts()` function. #' #' @source -#' Data source: Eurostat via [giscoR::gisco_get_nuts()]. +#' Data source: Eurostat via giscoR package function `gisco_get_nuts()`. #' #' © EuroGeographics for the administrative boundaries #' @@ -107,22 +93,22 @@ #' If you intend to use the data commercially, please contact EuroGeographics #' for information regarding their licence agreements." #' -#' @seealso [giscoR::gisco_get_nuts()] and +#' @seealso \CRANpkg{giscoR} function `gisco_get_nuts()` and #' [Eurostat. (2019). Methodological manual on territorial typologies -- 2018 edition. Manuals and guidelines.](https://ec.europa.eu/eurostat/web/products-manuals-and-guidelines/-/ks-gq-18-008) #' #' #' @examples #' -#' eurostat_geodata_60_2016 <- eurostat::eurostat_geodata_60_2016 +#' eurostat_geodata_60_2024 <- eurostat::eurostat_geodata_60_2024 #' #' # Manipulate and plot #' if (require(sf)) { #' library(sf) #' # Filter NUTS3 from select countries like in a regular data frame -#' example_nuts <- subset(eurostat_geodata_60_2016, LEVL_CODE == 3 & +#' example_nuts <- subset(eurostat_geodata_60_2024, LEVL_CODE == 3 & #' CNTR_CODE %in% c("DK", "DE", "PL")) #' #' plot(example_nuts["CNTR_CODE"]) #' } #' -NULL +"eurostat_geodata_60_2024" diff --git a/R/eurostat-package.R b/R/eurostat-package.R index 7f231408..06267ac4 100755 --- a/R/eurostat-package.R +++ b/R/eurostat-package.R @@ -12,7 +12,7 @@ #' | **Package** | eurostat | #' | **Type** | Package | #' | **Version** | `r packageVersion("eurostat")` | -#' | **Date** | 2014-2023 | +#' | **Date** | 2014-2025 | #' | **License** | `r as.character(utils::packageDescription("eurostat")["License"])`| #' | **LazyLoad** | yes | #' @@ -50,7 +50,7 @@ #' Data is downloaded from Eurostat SDMX 2.1 API endpoint #' as compressed TSV files that are transformed into tabular format. #' See Eurostat documentation for more information: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+query} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Dataquery} #' #' The new dissemination API replaces the old bulk download facility that was #' used by Eurostat before October 2023 and by the eurostat R package versions @@ -58,7 +58,7 @@ #' See Eurostat documentation about the transition from Bulk Download to API #' for more information about the differences between the old bulk download #' facility and the data provided by the new API connection: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload} #' #' See especially the document Migrating_to_API_TSV.pdf that describes the #' changes in TSV file format in new applications. @@ -75,20 +75,20 @@ #' Eurostat. We may support this feature in the future. In the meantime, if you #' are interested in filtering Dissemination API data queries manually, please #' consult the following Eurostat documentation: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+filtering} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Filteringonseries-keys} #' #' # Data source: Eurostat API Statistics (JSON API) #' #' Data is downloaded from Eurostat API Statistics. See Eurostat documentation #' for more information about data queries in API Statistics -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api} #' #' This replaces the old JSON Web Services that was used by Eurostat before #' February 2023 and by the eurostat R package versions before 3.7.13. #' See Eurostat documentation about the migration from JSON web service to API #' Statistics for more information about the differences between the old and #' the new service: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/json} #' #' For easily viewing which filtering options are available - in addition to #' the default ones, time and language - Eurostat Web services Query builder @@ -182,7 +182,7 @@ #' #' For more information about data filtering see Eurostat documentation #' on API Statistics: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} #' #' # Data source: Eurostat Table of Contents #' @@ -193,7 +193,7 @@ #' \url{https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang=de} #' #' See Eurostat documentation on TOC items: -#' \url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+-+Detailed+guidelines+-+Catalogue+API+-+TOC} +#' \url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/catalogue-api/toc} #' #' # Data source: GISCO - General Copyright #' @@ -339,4 +339,4 @@ #' regions package are imported . #' @keywords package #' @seealso `help("regions")`, -NULL +"_PACKAGE" diff --git a/R/get_bibentry.R b/R/get_bibentry.R index 0a2fc95e..8a2e8058 100755 --- a/R/get_bibentry.R +++ b/R/get_bibentry.R @@ -14,7 +14,7 @@ #' #' @return a bibentry, Bibtex or Biblatex object. #' -#' @seealso [utils::bibentry] [RefManageR::toBiblatex] +#' @seealso [utils::bibentry] #' #' @examplesIf check_access_to_data() #' \dontrun{ @@ -31,13 +31,12 @@ #' #' @importFrom lubridate dmy year month day #' @importFrom utils toBibtex person -#' @importFrom RefManageR BibEntry toBiblatex #' @importFrom stringr str_glue #' #' @export get_bibentry <- function(code, keywords = NULL, - format = "Biblatex", + format = "bibtex", lang = "en") { if (!any(class(code) %in% c("character", "factor"))) { stop("The code(s) must be added as character vector") @@ -49,10 +48,10 @@ get_bibentry <- function(code, code <- as.character(code) format <- tolower(as.character(format)) - if (!format %in% c("bibentry", "bibtex", "biblatex")) { - warning("The ", format, " is not recognized, will return Biblatex as + if (!format %in% c("bibentry", "bibtex")) { + warning("The ", format, " is not recognized, will return bibtex as default.") - format <- "biblatex" + format <- "bibtex" } toc <- get_eurostat_toc(lang = lang) @@ -66,9 +65,9 @@ get_bibentry <- function(code, lang <- check_lang(lang) lang_long <- switch(lang, - en = "english", - fr = "french", - de = "german") + en = "English", + fr = "French", + de = "German") if (nrow(toc) == 0) { warning(paste( @@ -119,19 +118,28 @@ get_bibentry <- function(code, } else { keyword_entry <- NULL } - - entry <- RefManageR::BibEntry( + + sdmx_dataflow <- get_sdmx_dataflow(id = toc$code[i], agency = "Eurostat") + + unexpanded_doi <- "" + if (!is.na(sdmx_dataflow$doi_url)) { + unexpanded_doi <- gsub("https://doi.org/", "", sdmx_dataflow$doi_url) + unexpanded_doi <- gsub("_", "\\\\_", unexpanded_doi) + } + + entry <- utils::bibentry( bibtype = "misc", key = dataset_key, title = paste0(toc$title[i], " (", dataset_id, ")"), + author = c( + utils::person(given = "Eurostat") + ), url = paste0("https://ec.europa.eu/eurostat/web/products-datasets/product?code=", toc$code[i]), language = lang_long, # date = last_update_date, year = last_update_year, - author = c( - utils::person(given = "Eurostat") - ), + doi = unexpanded_doi, keywords = keyword_entry, urldate = urldate, type = "Dataset", @@ -150,8 +158,7 @@ get_bibentry <- function(code, if (format == "bibtex") { entries <- utils::toBibtex(entries) - } else if (format == "biblatex") { - entries <- RefManageR::toBiblatex(entries) } + # if entry is bibentry then that will be returned instead of bibtex entries } diff --git a/R/get_eurostat.R b/R/get_eurostat.R index 013fea38..c38c00e0 100755 --- a/R/get_eurostat.R +++ b/R/get_eurostat.R @@ -10,19 +10,19 @@ #' @param filters #' A named list of filters. Names of list objects are Eurostat #' variable codes and values are vectors of observation codes. If `NULL` -#' (default) the whole dataset is returned. See details for more information +#' (default) the whole dataset is returned. See details for more information #' on filters and limitations per query. #' @param time_format #' a string giving a type of the conversion of the time column from the #' eurostat format. The default argument "`date`" converts to a [Date()] class -#' with the date being the first day of the period. A "`date_last`" argument -#' converts the dataset date to a [Date()] class object with the difference +#' with the date being the first day of the period. A "`date_last`" argument +#' converts the dataset date to a [Date()] class object with the difference #' that the exact date is the last date of the period. Period can be year, -#' semester (half year), quarter, month, or week (See [eurotime2date()] for +#' semester (half year), quarter, month, or week (See [eurotime2date()] for #' more information). #' Argument "`num`" converts the date into a numeric (integer) meaning that #' the first day of the year 2000 is close to 2000.01 and the last day of the -#' year is close to 2000.99 (see [eurotime2num()] for more information). +#' year is close to 2000.99 (see [eurotime2num()] for more information). #' Using the argument "`raw`" preserves the dates as they were in the original #' Eurostat data. #' @param type @@ -37,7 +37,7 @@ #' "Y" (or "A") = annual, "S" = semi-annual / semester, "Q" = quarterly, #' "M" = monthly, "W" = weekly. For all frequencies in same data #' frame `time_format = "raw"` should be used. -#' @param lang 2-letter language code, default is "`en`" (English), other +#' @param lang 2-letter language code, default is "`en`" (English), other #' options are "`fr`" (French) and "`de`" (German). Used for labeling datasets. #' @param cache #' a logical whether to do caching. Default is `TRUE`. @@ -53,7 +53,7 @@ #' @param compress_file #' a logical whether to compress the RDS-file in caching. Default is `TRUE`. #' @param stringsAsFactors -#' if `TRUE` (the default) variables are converted to factors in the original +#' if `TRUE` (the default) variables are converted to factors in the original #' Eurostat order. If `FALSE` they are returned as strings. #' @param keepFlags #' a logical whether the flags (e.g. "confidential", @@ -63,10 +63,14 @@ #' Also possible non-real zero "0n" is indicated in flags column. #' Flags are not available for eurostat API, so `keepFlags` #' can not be used with a `filters`. -#' @param use.data.table Use faster data.table functions? Default is FALSE. +#' @param use.data.table Use faster data.table functions? Default is FALSE. #' On Windows requires that RTools is installed. +#' @param legacy.data.output Use legacy column names and data object structure. +#' Default is FALSE. If TRUE, the object will try to emulate the naming +#' conventions of eurostat package version 3.7.x and earlier. +#' @param verbose Output messages when downloading data. Default is `TRUE`. #' @inheritDotParams get_eurostat_json -#' +#' #' @inherit eurostat-package references #' #' @inheritSection eurostat-package Eurostat: Copyright notice and free re-use of data @@ -77,12 +81,12 @@ #' #' @author #' Przemyslaw Biecek, Leo Lahti, Janne Huovari, Markus Kainu and Pyry Kantanen -#' +#' #' @details #' Datasets are downloaded from -#' [the Eurostat SDMX 2.1 API](https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API) +#' [the Eurostat SDMX 2.1 API](https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload) #' in TSV format or from The Eurostat -#' [API Statistics JSON API](https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query). +#' [API Statistics JSON API](https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api). #' If only the table `id` is given, the whole table is downloaded from the #' SDMX API. If any `filters` are given JSON API is used instead. #' @@ -107,7 +111,7 @@ #' . The Eurostat #' database gives codes in the Data Navigation Tree after every dataset #' in parenthesis. -#' +#' #' @return #' a tibble. #' @@ -118,10 +122,10 @@ #' dimensions are missing on particular time. In JSON API missing values are #' dropped only if all dimensions are missing on all times. The data from #' bulk download facility can be completed for example with [tidyr::complete()]. -#' +#' #' @seealso #' [search_eurostat()], [label_eurostat()] -#' +#' #' @examplesIf check_access_to_data() #' \dontrun{ #' k <- get_eurostat("nama_10_lp_ulc") @@ -138,7 +142,7 @@ #' set_eurostat_cache_dir(file.path(tempdir(), "r_cache2")) #' k <- get_eurostat("nama_10_lp_ulc") #' k <- get_eurostat("nama_10_lp_ulc", cache = FALSE) -#' k <- get_eurostat("avia_gonc", select_time = "Y", cache = FALSE) +#' k <- get_eurostat("avia_gooc", select_time = "Y", cache = FALSE) #' #' dd <- get_eurostat("nama_10_gdp", #' filters = list( @@ -186,33 +190,35 @@ get_eurostat <- function(id, stringsAsFactors = FALSE, keepFlags = FALSE, use.data.table = FALSE, + legacy.data.output = FALSE, + verbose = TRUE, ...) { - # Check if you have access to ec.europe.eu. - # If dataset is cached, access to ec.europe.eu is not needed + # Check if you have access to ec.europa.eu. + # If dataset is cached, access to ec.europa.eu is not needed # Therefore this is a warning, not a stop if (!check_access_to_data()) { # nocov start - warning("You have no access to ec.europe.eu. + warning("You have no access to ec.europa.eu. Please check your connection and/or review your proxy settings") # nocov end } - + # For better code clarity, use only NULL in code if (is.character(filters) && identical(tolower(filters), "none")) { filters <- NULL } else if (is.character(filters) && !identical(tolower(filters), "null")) { - message("Non-standard filters argument. Using argument 'filters = NULL'") + if (verbose) message("Non-standard filters argument. Using argument 'filters = NULL'") filters <- NULL } # Inform user with message if keepFlags == TRUE cannot be delivered if (keepFlags && !is.null(filters)) { - message("The keepFlags argument of the get_eurostat function + if (verbose) message("The keepFlags argument of the get_eurostat function can be used only without filters. No Flags returned.") keepFlags <- FALSE } - + # Sanity check type <- tolower(type) time_format <- tolower(time_format) @@ -308,7 +314,7 @@ get_eurostat <- function(id, if (any(grepl(query_hash, cache_list_current))) { # Check if the same query has already been made - message("Dataset query already saved in cache_list.json...") + if (verbose) message("Dataset query already saved in cache_list.json...") } else if (!any(grepl(query_hash, cache_list_current))) { # If query was not made, append cache_list with new json query if (length(cache_list_current) == 0) { @@ -443,7 +449,7 @@ get_eurostat <- function(id, # which lists files downloaded and saved to cache but maybe in some # situations the cached file could go missing? Not very likely though - message(paste("Reading cache file", cache_file_bulk, "and filtering it")) + if (verbose) message(paste("Reading cache file", cache_file_bulk, "and filtering it")) y <- readRDS(cache_file_bulk) for (i in seq_along(filters)) { y <- dplyr::filter(y, @@ -452,16 +458,20 @@ get_eurostat <- function(id, # y <- y_raw } else if (file.exists(cache_file)) { cf <- path.expand(cache_file) - message(paste("Reading cache file", cf)) + if (verbose) message(paste("Reading cache file", cf)) y <- readRDS(cache_file) - message(paste("Table ", id, " read from cache file: ", cf)) + if (verbose) message(paste("Table ", id, " read from cache file: ", cf)) } # if update_cache = TRUE or cache file does not yet exist # -> save cache file to cache directory if (cache && (update_cache || !file.exists(cache_file))) { saveRDS(y, file = cache_file, compress = compress_file) - message("Table ", id, " cached at ", path.expand(cache_file)) + if (verbose) message("Table ", id, " cached at ", path.expand(cache_file)) + } + + if (legacy.data.output){ + y <- legacy_data_format(y) } y @@ -476,36 +486,36 @@ get_eurostat <- function(id, #' as object names. The datasets are downloaded from SDMX API as TSV files, #' meaning that they are returned without filtering. No filters can be #' provided using this function. -#' +#' #' Please do not attempt to download too many datasets or the whole database -#' at once. The number of datasets that can be downloaded at once is hardcoded -#' to 20. The function also asks the user for confirmation if the number of +#' at once. The number of datasets that can be downloaded at once is hardcoded +#' to 20. The function also asks the user for confirmation if the number of #' datasets in a folder is more than 10. This is by design to discourage #' straining Eurostat API. #' @param code Folder code from Eurostat Table of Contents. #' @param env Name of the environment where downloaded datasets are assigned. #' Default is .EurostatEnv. If NULL, datasets are returned as a list object. -#' +#' #' @inheritSection eurostat-package Data source: Eurostat Table of Contents #' @inheritSection eurostat-package Data source: Eurostat SDMX 2.1 Dissemination API -#' +#' #' @author Pyry Kantanen -#' +#' #' @inherit set_eurostat_toc seealso -#' +#' #' @importFrom stringr str_glue #' @importFrom utils menu -#' -#' +#' +#' #' @export get_eurostat_folder <- function(code, env = .EurostatEnv) { - + # Limit after which the function prompts the user whether they really want # to proceed soft_limit <- 10 # Limit that cannot be crossed with this function hard_limit <- 20 - + toc <- get_eurostat_toc() if (toc[["type"]][which(toc[["code"]] == code)] != "folder") { warning("The code you provided is not a folder.") @@ -514,7 +524,7 @@ get_eurostat_folder <- function(code, env = .EurostatEnv) { children <- toc_list_children(code) # Filter out potential subfolders children <- children[which(children$type %in% c("dataset", "table")), ] - + if (nrow(children) == 0) { warning("The folder code you provided did not have any items.") return(invisible()) @@ -537,7 +547,7 @@ get_eurostat_folder <- function(code, env = .EurostatEnv) { message("Proceeding to download datasets in folder..."), return(invisible())) } - + if (!is.null(env)) { for (i in seq_len(nrow(children))) { dataset <- get_eurostat(children$code[i], cache = TRUE) @@ -566,28 +576,28 @@ get_eurostat_folder <- function(code, env = .EurostatEnv) { #' @title Get Eurostat data interactive #' @description -#' A simple interactive helper function to go through the steps of downloading -#' and/or finding suitable eurostat datasets. -#' +#' A simple interactive helper function to go through the steps of downloading +#' and/or finding suitable eurostat datasets. +#' #' @details #' This function is intended to enable easy exploration of different eurostat #' package functionalities and functions. In order to not drown the end user -#' in endless menus this function does not allow for setting +#' in endless menus this function does not allow for setting #' all possible [get_eurostat()] function arguments. It is possible to set #' `time_format`, `type`, `lang`, `stringsAsFactors`, `keepFlags`, and -#' `use.data.table` in the interactive menus. -#' -#' In some datasets setting these parameters may result in a -#' "Error in label_eurostat" error, for example: -#' "labels for XXXXXX includes duplicated labels in the Eurostat dictionary". +#' `use.data.table` in the interactive menus. +#' +#' In some datasets setting these parameters may result in a +#' "Error in label_eurostat" error, for example: +#' "labels for XXXXXX includes duplicated labels in the Eurostat dictionary". #' In these cases, and with other more complex queries, please #' use [get_eurostat()] function directly. -#' -#' @param code +#' +#' @param code #' A unique identifier / code for the dataset of interest. If code is not #' known [search_eurostat()] function can be used to search Eurostat table #' of contents. -#' +#' #' @seealso [get_eurostat()] #' @importFrom stringr str_glue #' @importFrom utils capture.output @@ -602,9 +612,9 @@ get_eurostat_interactive <- function(code = NULL) { "fr", "de" ) - + if (is.null(code)) { - + search_term <- readline(prompt = "Enter search term for data: ") results <- search_eurostat(pattern = search_term, lang = lang_selection) code_and_title <- paste0("[", results$code, "] ", results$title) @@ -619,9 +629,9 @@ get_eurostat_interactive <- function(code = NULL) { } code <- results$code[choice] } - + download_selection <- switch( - menu(choices = c("Yes", "No"), + menu(choices = c("Yes", "No"), title = "Download the dataset?") + 1, return(invisible()), TRUE, @@ -630,16 +640,16 @@ get_eurostat_interactive <- function(code = NULL) { # Set manual_selection to FALSE here to make it possible to print code # for downloading dataset later manual_selection <- FALSE - + if (download_selection) { manual_selection <- switch( - menu(choices = c("Default", "Manually selected"), + menu(choices = c("Default", "Manually selected"), title = "Would you like to use default download arguments or set them manually?") + 1, return(invisible()), FALSE, TRUE ) - + if (manual_selection) { time_format_selection <- switch( menu(choices = c("Convert to date, first day of the period (2000-04-01) (default)", @@ -652,7 +662,7 @@ get_eurostat_interactive <- function(code = NULL) { "num", "raw" ) - + type_selection <- switch( menu(choices = c("Return categorical variables as short codes (default)", "Return categorical variables in labeled (long) format"), @@ -661,7 +671,7 @@ get_eurostat_interactive <- function(code = NULL) { "code", "label" ) - + stringsAsFactors_selection <- switch( menu(choices = c("Return categorical data as characters (default)", "Convert categorical data into factors"), @@ -670,7 +680,7 @@ get_eurostat_interactive <- function(code = NULL) { FALSE, TRUE ) - + keepFlags_selection <- switch( menu(choices = c("Do not return flags, just remove them (default)", "Return flags in separate column")) + 1, @@ -678,9 +688,9 @@ get_eurostat_interactive <- function(code = NULL) { FALSE, TRUE ) - + use.data.table_selection <- switch( - menu(choices = c("Do not use data.table functions (default", + menu(choices = c("Do not use data.table functions (default)", "Use data.table functions"), title = "Using data.table functions may help reduce time used in data processing and reduce RAM usage. It is advisable especially when dealing with large datasets.") + 1, FALSE, @@ -718,33 +728,33 @@ get_eurostat_interactive <- function(code = NULL) { eurostat_data <- get_eurostat(id = code) } } - + tempfile_for_sinking <- tempfile() - + # eurostat_data <- get_eurostat(id = code) print_citation <- switch( - menu(choices = c("Yes", "No"), + menu(choices = c("Yes", "No"), title = "Print dataset citation?") + 1, - return(invisible()), - TRUE, + return(invisible()), + TRUE, FALSE ) - + if (print_citation) { citation <- get_bibentry(code, lang = lang_selection) capture.output(cat("##### DATASET CITATION:\n\n"), file = tempfile_for_sinking, append = TRUE) capture.output(print(citation), file = tempfile_for_sinking, append = TRUE) capture.output(cat("\n"), file = tempfile_for_sinking, append = TRUE) } - + print_code <- switch( - menu(choices = c("Yes", "No"), + menu(choices = c("Yes", "No"), title = "Print code for downloading dataset?") + 1, - return(invisible()), + return(invisible()), TRUE, FALSE ) - + if (print_code == TRUE && manual_selection == TRUE) { capture.output(cat("##### DOWNLOAD PARAMETERS:\n\n")) capture.output(print(stringr::str_glue(paste0("get_eurostat(id = '{code}', time_format = '{time_format_selection}', ", @@ -758,9 +768,9 @@ get_eurostat_interactive <- function(code = NULL) { capture.output(print(stringr::str_glue("get_eurostat(id = '{code}')")), file = tempfile_for_sinking, append = TRUE) capture.output(cat("\n"), file = tempfile_for_sinking, append = TRUE) } - + if (exists("eurostat_data")) { - print_code <- switch( + print_fixity <- switch( menu(choices = c("Yes", "No"), title = "Print dataset fixity checksum?") + 1, return(invisible()), @@ -768,18 +778,22 @@ get_eurostat_interactive <- function(code = NULL) { FALSE ) - if (print_code) { + if (print_fixity) { capture.output(cat("##### FIXITY CHECKSUM:\n\n"), file = tempfile_for_sinking, append = TRUE) capture.output(print(stringr::str_glue("Fixity checksum (md5) for dataset {code}: {eurostat:::fixity_checksum(eurostat_data, algorithm = 'md5')}")), file = tempfile_for_sinking, append = TRUE) capture.output(cat("\n"), file = tempfile_for_sinking, append = TRUE) } } - + if (exists("eurostat_data")) { - cat(readLines(tempfile_for_sinking), sep = "\n") + if (any(c(print_citation, print_code, print_fixity))) { + cat(readLines(tempfile_for_sinking), sep = "\n") + } return(eurostat_data) } else { - cat(readLines(tempfile_for_sinking), sep = "\n") + if (any(c(print_citation, print_code))) { + cat(readLines(tempfile_for_sinking), sep = "\n") + } return(invisible()) } # nocov end diff --git a/R/get_eurostat_geospatial.R b/R/get_eurostat_geospatial.R index 6f7525c6..267258d5 100755 --- a/R/get_eurostat_geospatial.R +++ b/R/get_eurostat_geospatial.R @@ -1,14 +1,15 @@ #' @title Download Geospatial Data from GISCO #' #' @description Downloads either a simple features (sf) or a data_frame -#' of NUTS regions. This function is a wrapper of [giscoR::gisco_get_nuts()]. -#' This function requires to have installed the packages \CRANpkg{sf} and -#' \CRANpkg{giscoR}. +#' of NUTS regions. This function is a wrapper of \CRANpkg{giscoR} function +#' `gisco_get_nuts()`. +#' This function requires to have packages \CRANpkg{sf} and +#' \CRANpkg{giscoR} installed. #' -#' @seealso [giscoR::gisco_get_nuts()] +#' @seealso giscoR package and its functions #' @param output_class Class of object returned, #' either `sf` `simple features` or `df` (`data_frame`). `spdf` output has -#' been soft-deprecated, the function would switch to `sf`. +#' been soft-deprecated, the function will switch to `sf`. #' @param resolution Resolution of the geospatial data. One of #' * "60" (1:60million), #' * "20" (1:20million) @@ -18,22 +19,16 @@ #' @param nuts_level Level of NUTS classification of the geospatial data. One of #' "0", "1", "2", "3" or "all" (mimics the original behaviour) #' @param year NUTS release year. One of -#' "2003", "2006", "2010", "2013", "2016" or "2021" -#' @param cache a logical whether to do caching. Default is `TRUE`. -#' @param update_cache a logical whether to update cache. Can be set also with -#' `options(eurostat_update = TRUE)` -#' @param cache_dir a path to a cache directory. See -#' [set_eurostat_cache_dir()]. If `NULL` and the cache dir has not been set -#' globally the file would be stored in the [tempdir()]. +#' "2003", "2006", "2010", "2013", "2016", "2021" or "2024" #' @param crs projection of the map: 4-digit #' [EPSG code](https://spatialreference.org/ref/epsg/). One of: #' * "4326" - WGS84 #' * "3035" - ETRS89 / ETRS-LAEA #' * "3857" - Pseudo-Mercator +#' @param ... additional arguments to be passed onto \CRANpkg{giscoR} function +#' `gisco_get_nuts()` #' -#' @param make_valid Deprecated -#' -#' @inheritDotParams giscoR::gisco_get_nuts -epsg +#' @inheritParams get_eurostat #' #' @details #' The objects downloaded from GISCO should contain all or some of the @@ -103,6 +98,7 @@ #' resolution = "60", #' nuts_level = "all" #' ) +#' \dontrun{ #' # Downloads dataset from server #' sf2 <- get_eurostat_geospatial( #' output_class = "sf", @@ -114,130 +110,113 @@ #' nuts_level = "0" #' ) #' } +#' } +#' +#' @importFrom utils modifyList #' #' @export get_eurostat_geospatial <- function(output_class = "sf", resolution = "60", - nuts_level = "all", year = "2016", + nuts_level = "all", year = "2024", cache = TRUE, update_cache = FALSE, cache_dir = NULL, crs = "4326", - make_valid = "DEPRECATED", ...) { - # nocov start - if (!requireNamespace("sf")) { - message("'sf' package is required for geospatial functionalities") + verbose = TRUE, ...) { + + # sf is always required + if (!requireNamespace("sf", quietly = TRUE)) { + if (verbose) message("'sf' package is required for geospatial functionalities") return(invisible()) } - # nocov end - # Simplified and leaving most of the heavy-lifting to giscoR - # Deprecation messages + has_gisco <- requireNamespace("giscoR", quietly = TRUE) + + # Deprecation handling stopifnot(length(output_class) == 1L) if (output_class == "spdf") { - message("'spdf' output deprecated. Switching to sf output") + if (verbose) message("'spdf' output deprecated. Switching to sf output") output_class <- "sf" } - if (!identical(make_valid, "DEPRECATED")) { - message("'make_valid' argument has been deprecated") - } - - # Leaving only specific validations - rest of call would be handled by giscoR output_class <- match.arg(as.character(output_class), c("sf", "df")) - # Sanity check for nuts_level - stopifnot(length(nuts_level) == 1L) - nuts_level <- regmatches(nuts_level, regexpr("^(all|[0-9]+)", nuts_level)) - nuts_level <- match.arg(nuts_level, c("all", 0:3)) - - # Performance - If df requested resolution and crs are meaningless. Switching - # to 60 and 4326 for speed (except for 2003, no available) - if (output_class == "df") { - resolution <- "60" - crs <- "4326" - - if (as.integer(year) == 2003) resolution <- "20" - } - - - # If cache file requested get the info from the internal dataset - capture_dots <- list(...) - - # Check if the pre-set call to the function has been modified on - # relevant parameters + # ---- Determine whether we should use local data ---- use_local <- all( as.character(resolution) == "60", - as.character(year) == "2016", + as.character(year) == "2024", isFALSE(update_cache), as.character(crs) == "4326", - # Check dots are empty - length(capture_dots) == 0 + length(list(...)) == 0 ) + # FORCE local if giscoR not available + if (!has_gisco) { + use_local <- TRUE + if (verbose) { + message("Package 'giscoR' not installed: using local dataset only") + } + } + + # ---- LOCAL BRANCH ---- if (use_local) { - # Not modified - using dataset included with eurostat package - message("Extracting data from eurostat::eurostat_geodata_60_2016") - shp <- eurostat::eurostat_geodata_60_2016 + if (verbose) { + message("Extracting data from eurostat::eurostat_geodata_60_2024") + } + + shp <- eurostat::eurostat_geodata_60_2024 + if (nuts_level != "all") { shp <- shp[shp$LEVL_CODE == nuts_level, ] } + } else { - # Check if package "giscoR" is installed - # nocov start - if (!requireNamespace("giscoR")) { - message("'giscoR' package is required for geospatial functionalities") - return(invisible()) + + # ---- REMOTE BRANCH (safe: giscoR exists here) ---- + + # defaults only accessed AFTER we know giscoR exists + default_args <- as.list(formals(giscoR::gisco_get_nuts)) + args <- utils::modifyList(default_args, list(...)) + + args$resolution <- resolution + args$nuts_level <- nuts_level + args$year <- year + args$epsg <- crs + args$cache <- cache + args$update_cache <- update_cache + args$cache_dir <- cache_dir + + # Sanity check + stopifnot(length(args$nuts_level) == 1L) + args$nuts_level <- regmatches(args$nuts_level, regexpr("^(all|[0-9]+)", args$nuts_level)) + args$nuts_level <- match.arg(args$nuts_level, c("all", 0:3)) + + # Performance tweak + if (output_class == "df") { + args$resolution <- "60" + args$epsg <- "4326" + if (as.integer(args$year) == 2003) args$resolution <- "20" } - # nocov end - - message(paste0( - "Extracting data using giscoR package, please report issues", - " on https://github.com/rOpenGov/giscoR/issues" - )) - - # Manage cache: Priority is eurostat cache (if set) - # If not make use of giscoR default options - detect_eurostat_cache <- eur_helper_detect_cache_dir() - if (!is.null(cache_dir)) { - # Already set by the user, no need message - cache_dir <- eur_helper_cachedir(cache_dir) - } else if (identical( - detect_eurostat_cache, - file.path(tempdir(), "eurostat") - )) { - # eurostat not set, using default giscoR cache management - message("Cache management as per giscoR. see 'giscoR::gisco_get_nuts()'") - } else { - cache_dir <- eur_helper_cachedir(cache_dir) + + if (verbose) { + message( + "Extracting data using giscoR package, please report issues ", + "on https://github.com/rOpenGov/giscoR/issues" + ) } - # giscoR call with parameters - # on input errors giscoR would show warnings, etc - shp <- giscoR::gisco_get_nuts( - resolution = resolution, - nuts_level = nuts_level, year = year, - cache = cache, update_cache = update_cache, - cache_dir = cache_dir, epsg = crs, - ... - ) + shp <- do.call(giscoR::gisco_get_nuts, args) } - # Just to capture potential NULL outputs from giscoR - this can happen - # on some errors - if (is.null(shp)) { - return(NULL) - } + # Handle NULL safely + if (is.null(shp)) return(NULL) - # Post-data treatments - # Manage col names + # Post-processing shp <- geo_names(shp) - # to df if (output_class == "df") { - # Remove geometry shp <- sf::st_drop_geometry(shp) } - return(shp) + shp } diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 3549e327..c6aff5df 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -1,11 +1,15 @@ -#' @title Get Data from Eurostat API in JSON -#' @description Retrieve data from Eurostat API in JSON format. +#' @title Get Data from Eurostat API Statistics +#' @description Retrieve data from Eurostat API Statistics in JSON-stat 2.0 format. #' @details #' Data to retrieve from -#' [The Eurostat Web Services](https://ec.europa.eu/eurostat/web/main/data/web-services) +#' [The Eurostat API Statistics](https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query) #' can be specified with filters. Normally, it is #' better to use JSON query through [get_eurostat()], than to use -#' [get_eurostat_json()] directly. +#' [get_eurostat_json()] directly. The main reason for this is that +#' [get_eurostat_json()] returns a relatively raw dataset that does not go +#' through helper functions in [get_eurostat()], such as [eurotime2date()] or +#' [eurotime2num()] functions or reading data from cache and saving data +#' to cache. #' #' Queries are limited to 50 sub-indicators at a time. A time can be #' filtered with fixed "time" filter or with "sinceTimePeriod" and @@ -26,6 +30,18 @@ #' the possible interpretation or cause of each error. These messages are #' returned if the API returns a status indicating a HTTP error #' (400 or greater). +#' +#' Additionally, there is limit on the size of the returned extractions +#' (error code 413). +#' At the time of publishing this package version, the max authorised size for +#' the extraction seems to be 5000000 (5 million) rows. The server seems to +#' estimate the size of the returned data object with a method that is +#' unknown to us so the number of rows might not be the same as the number +#' of returned rows in the JSON-stat object. You can limit the number of rows +#' in your extraction by providing more filters to the query. If the +#' extraction size continues to go over limit you can try to download the +#' whole dataset with [get_eurostat()] and filter it locally on your computer, +#' for example with base R or dplyr data wrangling functions. #' #' The Eurostat implementation seems to be based on SDMX 2.1, which is the #' reason we've used SDMX Standards guidelines as a supplementary source @@ -104,20 +120,16 @@ get_eurostat_json <- function(id, return(NULL) } - # Check if you have access to ec.europe.eu. + # Check if you have access to ec.europa.eu. if (!check_access_to_data()) { # nocov start - message("You have no access to ec.europe.eu. + message("You have no access to ec.europa.eu. Please check your connection and/or review your proxy settings") # nocov end } # construct url url <- eurostat_json_url(id = id, filters = filters, lang = lang) - # set user agent with version - # ua <- httr::user_agent(paste0("eurostat_", - # packageDescription("eurostat", fields = "Version"))) - # ua <- httr::user_agent("https://github.com/rOpenGov/eurostat") if (proxy == TRUE) { # Check if "..." has arguments needed for proxy @@ -150,13 +162,6 @@ get_eurostat_json <- function(id, httr2::req_error(is_error = function(resp) FALSE) %>% httr2::req_perform() - # RETRY GET 3 times - # resp <- httr::RETRY(verb = "GET", - # url = url, - # times = 3, - # terminate_on = c(404), - # ua) - # Source: httr vignette "Best practices for API packages" [httr_vignette] if (httr2::resp_content_type(resp) != "application/json") { stop("API did not return json", call. = FALSE) @@ -167,7 +172,6 @@ get_eurostat_json <- function(id, result <- httr2::resp_body_json( resp = resp, simplifyVector = TRUE) - if (httr2::resp_is_error(resp)) { # These objects are only needed if there is an error @@ -187,10 +191,10 @@ get_eurostat_json <- function(id, " Error id: {id} ({faultstring})\n", " Error label from API: {label}") ) - } else { + } else{ stop(stringr::str_glue("\n", "HTTP status: {status} ({status_code_label})\n", - " Error id: {id} ({faultstring})\n", + " Error id: {id} \n", " Error label from API: {label}") ) } @@ -212,8 +216,6 @@ get_eurostat_json <- function(id, ) } - #status <- httr::status_code(resp) - # check status and get json jdat <- result # no need to download same data twice diff --git a/R/get_eurostat_link.R b/R/get_eurostat_link.R new file mode 100644 index 00000000..28682731 --- /dev/null +++ b/R/get_eurostat_link.R @@ -0,0 +1,190 @@ +#' @title Download Eurostat Data from API Link (robust, no list-columns, async-aware) +#' +#' @description +#' `r lifecycle::badge('experimental')` +#' +#' Supports TSV, SDMX-CSV, SDMX-ML, JSON-stat, and Spreadsheet (.xlsx) formats. +#' Always returns a tibble where possible. +#' +#' @param link Eurostat "Copy API link" URL or Data Browser download link +#' @param destfile Optional file path for saving raw files (only for Excel) +#' @inheritParams get_eurostat +#' @return Tibble for all supported formats +#' @importFrom purrr map_dfr +#' @importFrom utils read.delim read.csv +#' @importFrom stats setNames +#' @importFrom httr2 request req_user_agent req_perform resp_check_status +#' @importFrom httr2 resp_body_raw resp_content_type url_parse +#' @export +get_eurostat_link <- function(link, destfile = NULL, verbose = TRUE) { + if (!is.character(link) || length(link) != 1) stop("Provide a valid Eurostat link.") + if (!grepl("ec\\.europa\\.eu/eurostat/", link)) stop("Only Eurostat links allowed.") + + as_tibble_safe <- function(x) { + if (is.data.frame(x)) tibble::as_tibble(x) + else tibble::tibble(data = list(x)) + } + + download_content <- function(link) { + resp <- httr2::request(link) %>% + httr2::req_user_agent("https://github.com/rOpenGov/eurostat") %>% + httr2::req_perform() + + httr2::resp_check_status(resp) + + raw <- httr2::resp_body_raw(resp) + + if (httr2::resp_content_type(resp) == "application/gzip" || grepl("compress=true", link, ignore.case = TRUE)) { + if (verbose) message("Decompressing content...") + raw <- memDecompress(raw, type = "gzip", asChar = TRUE) + } else { + raw <- rawToChar(raw) + } + + raw + } + + url_parsed <- httr2::url_parse(link) + + # Comext DS-prefixed datasets must use correct base + if (grepl("/comext/dissemination", link)) { + dataset_match <- grepl("/ds-[0-9]+", url_parsed$path) + if (!dataset_match) { + stop("Comext API requires DS-prefixed dataset codes (e.g., ds-056120).") + } + } + + # TSV format + if ("format" %in% attributes(url_parsed$query) && url_parsed$query$format == "TSV") { + if (verbose) message("Downloading TSV...") + text_content <- download_content(link) + + df <- utils::read.delim( + text = text_content, + sep = "\t", + check.names = FALSE, + na.strings = c(":", ": ") + ) + + # Unpack first column + first_col_name <- names(df)[1] + dim_names <- strsplit(first_col_name, ",", fixed = TRUE)[[1]] + dim_values <- strsplit(df[[1]], ",", fixed = TRUE) + dim_matrix <- do.call(rbind, dim_values) + dim_df <- as.data.frame(dim_matrix, stringsAsFactors = FALSE) + colnames(dim_df) <- dim_names + + # Optional: fix backslash column name + names(dim_df)[grepl("\\\\", names(dim_df))] <- sub("\\\\.*", "", names(dim_df)[grepl("\\\\", names(dim_df))]) + + final_df <- dplyr::bind_cols(dim_df, df[-1]) + return(tibble::as_tibble(final_df)) + } + + # SDMX-CSV + if ("format" %in% attributes(url_parsed$query)$names && url_parsed$query$format == "csvdata") { + if (verbose) message(paste0("Downloading SDMX-CSV ", url_parsed$query$formatVersion, "...")) + text_content <- download_content(link) + lines <- strsplit(text_content, "\n", fixed = TRUE)[[1]] + data_start <- which(sapply(lines, function(l) { + fields <- strsplit(l, ",", fixed = TRUE)[[1]] + length(fields) > 1 && all(nzchar(fields) | fields == "") + }))[1] + if (is.na(data_start)) stop("Could not detect valid SDMX-CSV header for Comext dataset.") + clean_text <- paste(lines[data_start:length(lines)], collapse = "\n") + return(as_tibble_safe(utils::read.csv(text = clean_text, check.names = FALSE, row.names = NULL))) + } + + # SDMX-ML 2.1 StructureSpecific + if ("format" %in% attributes(url_parsed$query)$names && url_parsed$query$format == "structurespecificdata") { + if (!requireNamespace("xml2", quietly = TRUE)) stop("Please install 'xml2' for XML parsing.") + if (verbose) message(paste("Downloading SDMX-ML", url_parsed$query$formatVersion, url_parsed$query$format, "...")) + xml_text <- download_content(link) + doc <- xml2::read_xml(xml_text) + ns <- xml2::xml_ns(doc) + series_nodes <- xml2::xml_find_all(doc, ".//Series", ns) + result <- purrr::map_dfr(series_nodes, function(series) { + series_atts <- xml2::xml_attrs(series) + obs_nodes <- xml2::xml_find_all(series, ".//Obs", ns) + purrr::map_dfr(obs_nodes, function(o) { + data.frame(as.list(series_atts), time = xml2::xml_attr(o, "TIME_PERIOD"), value = as.numeric(xml2::xml_attr(o, "OBS_VALUE")), stringsAsFactors = FALSE) + }) + }) + return(tibble::as_tibble(result)) + } + + # SDMX-ML 2.1 GenericData + if ("format" %in% attributes(url_parsed$query)$names && url_parsed$query$format == "genericdata") { + if (!requireNamespace("xml2", quietly = TRUE)) stop("Please install 'xml2' for XML parsing.") + if (verbose) message(paste("Downloading SDMX-ML", url_parsed$query$formatVersion, url_parsed$query$format, "...")) + xml_text <- download_content(link) + doc <- xml2::read_xml(xml_text) + ns <- xml2::xml_ns(doc) + series_nodes <- xml2::xml_find_all(doc, ".//g:Series", ns) + result <- purrr::map_dfr(series_nodes, function(series) { + key_nodes <- xml2::xml_find_all(series, ".//g:Value", ns) + key_vals <- stats::setNames(xml2::xml_attr(key_nodes, "value"), xml2::xml_attr(key_nodes, "id")) + obs_nodes <- xml2::xml_find_all(series, ".//g:Obs", ns) + purrr::map_dfr(obs_nodes, function(obs) { + data.frame(as.list(key_vals), time = xml2::xml_attr(xml2::xml_find_first(obs, ".//g:ObsDimension", ns), "value"), value = as.numeric(xml2::xml_attr(xml2::xml_find_first(obs, ".//g:ObsValue", ns), "value")), stringsAsFactors = FALSE) + }) + }) + return(tibble::as_tibble(result)) + } + + # SDMX-ML 3.0 StructureSpecific + if (grepl("/sdmx/3.0/data", link) && (grepl("\\.xml", link, ignore.case = TRUE) || !grepl("format=", link, ignore.case = TRUE))) { + if (!requireNamespace("xml2", quietly = TRUE)) stop("Please install 'xml2' for XML parsing.") + if (verbose) message(paste("Downloading SDMX-ML", url_parsed$query$formatVersion, url_parsed$query$format, "...")) + xml_text <- download_content(link) + doc <- xml2::read_xml(xml_text) + ns <- xml2::xml_ns(doc) + series_nodes <- xml2::xml_find_all(doc, ".//m:Series", ns) + if (length(series_nodes) == 0) { + series_nodes <- xml2::xml_find_all(doc, ".//Series", ns) + } + result <- purrr::map_dfr(series_nodes, function(series) { + series_atts <- xml2::xml_attrs(series) + obs <- xml2::xml_find_all(series, ".//Obs", ns) + if (length(obs) == 0) { + obs <- xml2::xml_find_all(series, ".//Obs", ns) + } + purrr::map_dfr(obs, function(o) { + row <- c(series_atts, list( + time = xml2::xml_attr(o, "TIME_PERIOD"), + value = as.numeric(xml2::xml_attr(o, "OBS_VALUE")) + )) + data.frame(row, stringsAsFactors = FALSE) + }) + }) + return(tibble::as_tibble(result)) + } + + # JSON-stat handling + if ("format" %in% attributes(url_parsed$query)$names && url_parsed$query$format == "JSON") { + if (verbose) message("Downloading JSON-stat 2.0...") + json_data <- jsonlite::fromJSON(link) + value_data <- json_data$value + dims <- json_data$dimension + if (is.null(value_data) && !is.null(json_data$dataset$value)) { + value_data <- json_data$dataset$value + dims <- json_data$dataset$dimension + } + if (!is.null(value_data) && !is.null(dims)) { + dim_ids <- names(dims) + dim_lists <- lapply(dim_ids, function(id) names(dims[[id]]$category$index)) + names(dim_lists) <- dim_ids + grid <- expand.grid(dim_lists, stringsAsFactors = FALSE) + values <- rep(NA_real_, nrow(grid)) + idx <- as.integer(names(value_data)) + 1 + values[idx] <- as.numeric(unlist(value_data)) + grid$value <- values + if (verbose) message("Returning flattened tibble.") + return(tibble::as_tibble(grid)) + } + flat <- jsonlite::flatten(json_data) + return(as_tibble_safe(flat)) + } + + stop("Unsupported or unknown format.") +} diff --git a/R/get_eurostat_local.R b/R/get_eurostat_local.R new file mode 100644 index 00000000..9c3c84fc --- /dev/null +++ b/R/get_eurostat_local.R @@ -0,0 +1,122 @@ +#' @title Read Local SDMX-CSV files +#' @description +#' `r lifecycle::badge('experimental')` +#' Read compressed or uncompressed SDMX-CSV files +#' +#' @details +#' This implementation is experimental. It uses only data.table methods to read +#' and wrangle data files. +#' +#' Download datasets in sdmx-csv format from \url{https://ec.europa.eu/eurostat/databrowser/bulk?lang=en} +#' +#' @inheritParams get_eurostat +#' @param file file path as string +#' @importFrom data.table fread +#' @export + +get_eurostat_local <- function(file, + time_format = "date", + type = "code", + lang = "en", + select_time = NULL, + stringsAsFactors = FALSE, + keepFlags = FALSE, + legacy.data.output = FALSE) { + file_attrs <- parse_filename(file) + + if (file_attrs$filetype == "csv.gz"){ + # decompress and read + # dat <- readr::read_csv(gzfile(file), + # na = "", + # progress = TRUE, + # col_types = readr::cols(.default = readr::col_character()) + # ) + dat <- data.table::fread(file = file, + na.strings = "", + header = TRUE, + colClasses = "character", + stringsAsFactors = stringsAsFactors) + } else if (file_attrs$filetype == "csv") { + # read + dat <- data.table::fread(file = file, + na.strings = "", + header = TRUE, + colClasses = "character", + stringsAsFactors = stringsAsFactors) + } else { + stop("Invalid file") + } + + # For better code clarity, use only NULL in code + if (is.character(filters) && identical(tolower(filters), "none")) { + filters <- NULL + } else if (is.character(filters) && !identical(tolower(filters), "null")) { + message("Non-standard filters argument. Using argument 'filters = NULL'") + filters <- NULL + } + + # Inform user with message if keepFlags == TRUE cannot be delivered + if (keepFlags && !is.null(filters)) { + message("The keepFlags argument of the get_eurostat function + can be used only without filters. No Flags returned.") + keepFlags <- FALSE + } + + # Sanity check + type <- tolower(type) + time_format <- tolower(time_format) + lang <- check_lang(lang) + + y <- tidy_eurostat_sdmx_csv( + dat, + time_format, + select_time, + keepFlags = keepFlags + ) + + if (identical(type, "code")) { + # do nothing + # y <- y + } else if (identical(type, "label")) { + agency <- build_agencyID(file_attrs$agency) + y <- label_eurostat_sdmx(y, lang, id = file_attrs$code, agency = agency) + } else if (identical(type, "both")) { + stop(paste("type = \"both\" can be only used with JSON API.", + "Set filters argument")) + } else { + stop("Invalid type.") + } + + if (legacy.data.output) { + y <- legacy_data_format(y) + } + + y <- tibble::as_tibble(y) + return(y) +} + +parse_filename <- function(file){ + string = file + # If file path contains folders, or slashes + if (grepl("/", string)){ + # Find last instance of "/" + last_slash_index <- max(unlist(gregexpr("/", string))) + string <- substr(string, start = last_slash_index + 1, stop = nchar(string)) + } + # Initialize list object to return + x <- list() + split_string <- unlist(strsplit(string, "_")) + # Get agency name + x$agency <- split_string[1] + + # Get ID + x$code <- paste(split_string[2:(length(split_string)-1)], collapse = "_") + + # Get file type and language + filetype_and_language <- unlist(strsplit(split_string[length(split_string)], "\\.")) + x$lang <- filetype_and_language[1] + x$filetype <- paste(filetype_and_language[2:length(filetype_and_language)], collapse = ".") + + return(x) + +} \ No newline at end of file diff --git a/R/get_eurostat_raw.R b/R/get_eurostat_raw.R index edb01459..c4e9a897 100755 --- a/R/get_eurostat_raw.R +++ b/R/get_eurostat_raw.R @@ -7,9 +7,9 @@ #' years and column names are years with preceding X. Data is in #' character format as it contains values together with eurostat #' flags for data. -#' @seealso +#' @seealso #' [get_eurostat()] -#' +#' #' @inheritSection eurostat-package Data source: Eurostat SDMX 2.1 Dissemination API #' @inheritSection eurostat-package Eurostat: Copyright notice and free re-use of data #' @inheritSection eurostat-package Citing Eurostat data @@ -56,7 +56,7 @@ get_eurostat_raw <- function(id, use.data.table = FALSE) { on.exit(options(op), add = TRUE) utils::download.file(url, tfile) } - + if (!use.data.table) { # OLD CODE dat <- readr::read_tsv(gzfile(tfile), @@ -70,10 +70,10 @@ get_eurostat_raw <- function(id, use.data.table = FALSE) { na.strings = ":", header = TRUE, colClasses = "character") - + # OLD CODE - # data.table object does not need to be converted into a tibble at this - # point as it will handled by data.table functions in tidy_eurostat. + # data.table object does not need to be converted into a tibble at this + # point as it will handled by data.table functions in tidy_eurostat. # dat <- tibble::as_tibble(dat) } diff --git a/R/get_eurostat_sdmx.R b/R/get_eurostat_sdmx.R new file mode 100644 index 00000000..fe0ea880 --- /dev/null +++ b/R/get_eurostat_sdmx.R @@ -0,0 +1,654 @@ +#' @title Get Eurostat Data from SDMX 2.1 API +#' +#' @description +#' `r lifecycle::badge('experimental')` +#' +#' Download data sets from Eurostat using the same logic as `get_eurostat()` +#' function. +#' +#' @details +#' This function is experimental because while it works as intended and is +#' useful in the same way as other get_ functions in the package, we would +#' like to test it for a while and listen to user feedback before deciding on +#' what is the best way to interact with SDMX APIs. +#' +#' @inheritParams get_eurostat +#' @param agency Either "Eurostat" (default), "Eurostat_comext" +#' (for Comext and Prodcom datasets), "COMP", "EMPL" or "GROW" +#' @param use.data.table Use data.table to process files? Default is FALSE. +#' If data.table is used, data will be downloaded as a TSV file and +#' processed using [tidy_eurostat()] +#' @param wait Integer. Seconds between status checks. Default is 1 second. +#' @param max_wait Integer. Max time to wait in seconds. Default is 60 seconds. +#' @param compressed Logical. Download data in compressed format? Default is TRUE. +#' +#' @importFrom curl curl_download +#' @importFrom utils download.file +#' @importFrom readr read_csv read_tsv cols col_character +#' @importFrom data.table fread +#' @importFrom httr2 request req_url_path_append req_url_query req_perform +#' +#' @export +get_eurostat_sdmx <- function( + id, + time_format = "date", + filters = NULL, + type = "code", + lang = "en", + use.data.table = FALSE, + agency = "Eurostat", + compressed = TRUE, + keepFlags = FALSE, + legacy.data.output = FALSE, + wait = 10, + max_wait = 600, + verbose = TRUE + ) { + + # Check if you have access to ec.europa.eu. + # If dataset is cached, access to ec.europa.eu is not needed + # Therefore this is a warning, not a stop + if (!check_access_to_data()) { + # nocov start + stop("You have no access to ec.europa.eu. + Please check your connection and/or review your proxy settings") + # nocov end + } + + lang <- check_lang(lang) + + agency <- tolower(agency) + + api_base_uri <- build_api_base_uri(agency) + agencyID <- build_agencyID(agency) + + if (is.null(api_base_uri)) stop("Use valid agency") + + if (is.null(filters) && agency == "eurostat_comext") { + stop("Use filters when querying data from Eurostat COMEXT or PRODCOM") + } + + # Following resource is supported: data + resource <- "data" + # The identifier of the dataflow reference + flowRef <- id + key <- data_filtering_on_dimension(agency, id, filters) + compressed_string <- if (compressed) "&compressed=true" else "&compressed=false" + + if (use.data.table) { + tfile <- tempfile() + on.exit(unlink(tfile)) + + httr2::request(api_base_uri) %>% + httr2::req_url_path_append("sdmx", "2.1", resource, id, key) %>% + httr2::req_url_query(format = "TSV", compressed = compressed, detail = "dataonly") %>% + httr2::req_perform(path = tfile) + + dat <- readr::read_tsv(tfile, progress = verbose, show_col_types = verbose) + dat2 <- data.table::fread(tfile, na.strings = c(":"), verbose = verbose) + # Columns containing NA's don't play well with data.table::melt in + # tidy_eurostat + # This turns logical columns (containing NA's) into integers (NA_integer_) + offending_cols <- names(dat2[, .SD, .SDcols = anyNA]) + dat2 <- dat2[ , (offending_cols) := lapply(.SD, as.integer), .SDcols = offending_cols] + + dat <- tidy_eurostat(dat2, use.data.table = use.data.table) + + return(dat) + } else { + res <- httr2::request(api_base_uri) %>% + httr2::req_url_path_append("sdmx", "2.1", resource, id, key) %>% + httr2::req_url_query(format = "SDMX-CSV", compressed = compressed, detail = "dataonly") + } + + tfile <- tempfile() + on.exit(unlink(tfile)) + + httr2::req_perform(res, path = tfile) + + dat <- readr::read_csv(tfile, progress = verbose, show_col_types = verbose) + if (!keepFlags) { + col_names <- names(dat) + dat <- dat[setdiff(col_names, "OBS_FLAG")] + } + + if (identical(type, "label")){ + dat <- label_eurostat_sdmx(x = dat, + agency = agency, + id = id, + lang = lang) + } + + dat$TIME_PERIOD <- convert_time_col(x = dat$TIME_PERIOD, + time_format = time_format) + + dat$OBS_VALUE <- as.numeric(dat$OBS_VALUE) + + if (legacy.data.output) { + dat <- legacy_data_format(dat) + } + + dat + +} + +get_sdmx_codelist <- function(codelist_id, agency = "Eurostat", type = "list", lang = NULL) { + lang <- check_lang(lang) + + api_base_uri <- build_api_base_uri(agency) + + xml_url <- paste0( + api_base_uri, + "/sdmx/2.1/codelist/estat/", + codelist_id) + + xml_object <- xml2::read_xml(xml_url) + + if (identical(type, "raw")) { + return(xml_object) + } + + namespaces <- xml2::xml_ns(xml_object) + + codelists <- xml2::xml_find_first(xml_object, ".//s:Codelists", namespaces) + + code_nodes <- xml2::xml_find_all(codelists, ".//s:Code", namespaces) + + id <- c() + name <- c() + + for (node in code_nodes) { + id <- c(id, xml2::xml_attr(node, "id")) + name <- c(name, xml2::xml_text(xml2::xml_find_first(node, sprintf(".//c:Name[@xml:lang='%s']", lang)))) + } + + df <- data.frame( + id = id, + name = name + ) + + return(df) +} + +get_sdmx_conceptscheme <- function(id, agency = "Eurostat", type = "list", lang = NULL) { + + lang <- check_lang(lang) + + api_base_uri <- build_api_base_uri(agency) + + xml_url <- paste0( + api_base_uri, + "/sdmx/2.1/conceptscheme/estat/", + id) + + xml_object <- xml2::read_xml(xml_url) + + if (identical(type, "raw")) { + return(xml_object) + } + + namespaces <- xml2::xml_ns(xml_object) + + # Continue with dimensions and concept extraction + data_structure_components <- xml2::xml_find_first(xml_object, ".//s:Concepts", namespaces) + + concept_nodes <- xml2::xml_find_all(data_structure_components, ".//s:Concept", namespaces) + + # metadata <- xml2::as_list(concept_nodes) + + id <- c() + urn <- c() + name <- c() + core_representation <- c() + version <- c() + + for (node in concept_nodes) { + id <- c(id, xml2::xml_attr(node, "id")) + urn <- c(urn, xml2::xml_attr(node, "urn")) + name <- c(name, xml2::xml_text(xml2::xml_find_first(node, sprintf(".//c:Name[@xml:lang='%s']", lang)))) + if (is.na(xml2::xml_attr(xml2::xml_find_first(node, ".//Ref"), "class"))) { + core_representation <- c(core_representation, xml2::xml_attr(xml2::xml_find_first(node, ".//s:TextFormat"), "textType")) + } else { + core_representation <- c(core_representation, xml2::xml_attr(xml2::xml_find_first(node, ".//Ref"), "class")) + } + version <- c(version, xml2::xml_attr(xml2::xml_find_first(node, ".//Ref"), "version")) + } + + metadata_returnable <- data.frame( + id = id, + urn = urn, + name = name, + core_representation = core_representation, + version = version + ) + + return(metadata_returnable) +} + +#' @importFrom xml2 xml_ns xml_find_first xml_text xml_attr xml_find_all +get_sdmx_dsd <- function(id, agency = "Eurostat", type = "list", lang = NULL) { + api_base_uri <- build_api_base_uri(agency) + + dsd_url <- paste0( + api_base_uri, + "/sdmx/2.1/datastructure/estat/", + id) + + xml_object <- xml2::read_xml(dsd_url) + + if (identical(type, "raw")) { + return(xml_object) + } + + # Define namespaces + namespaces <- xml2::xml_ns(xml_object) + + # Extract Header information + header <- xml2::xml_find_first(xml_object, ".//m:Header", namespaces) + header_id <- xml2::xml_text(xml2::xml_find_first(header, ".//m:ID", namespaces)) + prepared <- substr(xml2::xml_text(xml2::xml_find_first(header, ".//m:Prepared", namespaces)),1,10) + sender_id <- xml2::xml_attr(xml2::xml_find_first(header, ".//m:Sender", namespaces), "id") + + + # Continue with dimensions and concept extraction + data_structure_components <- xml2::xml_find_first(xml_object, ".//s:DataStructureComponents", namespaces) + + dimension_nodes <- xml2::xml_find_all(data_structure_components, ".//s:Dimension | .//s:TimeDimension | .//s:AttributeList/s:Attribute | .//s:MeasureList/s:PrimaryMeasure", namespaces) + + metadata_returnable <- list() + + metadata <- xml2::as_list(dimension_nodes) + + id <- c() + urn <- c() + concept_identity_class <- c() + local_representation <- c() + local_representation_version <- c() + + for (node in dimension_nodes) { + id <- c(id, xml2::xml_attr(node, "id")) + urn <- c(urn, xml2::xml_attr(node, "urn")) + concept_identity_class <- c(concept_identity_class, xml2::xml_attr(xml_find_first(node, ".//Ref"), "class")) + if (is.na(xml2::xml_attr(xml2::xml_find_first(node, ".//Ref"), "class"))) { + local_representation <- c(local_representation, xml2::xml_attr(xml2::xml_find_first(node, ".//s:TextFormat"), "textType")) + } else { + local_representation <- c(local_representation, xml2::xml_attr(xml2::xml_find_first(node, ".//Ref"), "class")) + } + local_representation_version <- c(local_representation_version, xml2::xml_attr(xml2::xml_find_first(node, ".//s:Enumeration/Ref"), "version")) + } + + metadata_returnable <- data.frame( + id = id, + urn = urn, + concept_identity_class = concept_identity_class, + local_representation = local_representation, + local_representation_version = local_representation_version + ) + + return(metadata_returnable) +} + +get_sdmx_dataflow <- function(id, agency = "Eurostat", type = "list", lang = NULL) { + + api_base_uri <- build_api_base_uri(agency) + + dataflow_url <- paste0( + api_base_uri, + "/sdmx/2.1/dataflow/estat/", + id) + + # With DSD + # dataflow_url <- paste0( + # api_base_uri, + # "/sdmx/2.1/dataflow/estat/", + # id, + # "/", + # "?references=children") + + xml_object <- xml2::read_xml(dataflow_url) + + if (identical(type, "raw")) { + return(xml_object) + } + + # Define namespaces + namespaces <- xml2::xml_ns(xml_object) + + # Extract Header information + header <- xml2::xml_find_first(xml_object, ".//m:Header", namespaces) + header_id <- xml2::xml_text(xml2::xml_find_first(header, ".//m:ID", namespaces)) + prepared <- substr(xml2::xml_text(xml2::xml_find_first(header, ".//m:Prepared", namespaces)),1,10) + sender_id <- xml2::xml_attr(xml2::xml_find_first(header, ".//m:Sender", namespaces), "id") + + # Continue with dataflow and annotations extraction + dataflow <- xml2::xml_find_first(xml_object, ".//s:Dataflow", namespaces) + dataflow_id <- xml2::xml_attr(dataflow, "id") + urn <- xml2::xml_attr(dataflow, "urn") + agencyID <- xml2::xml_attr(dataflow, "agencyID") + version <- xml2::xml_attr(dataflow, "version") + isFinal <- xml2::xml_attr(dataflow, "isFinal") + # Extract names in different languages + # Extract names in different languages independently + name_de <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='de']", namespaces)) + name_en <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='en']", namespaces)) + name_fr <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='fr']", namespaces)) + if (!is.null(lang)) { + name <- xml2::xml_text(xml2::xml_find_first(dataflow, sprintf(".//c:Name[@xml:lang='%s']", lang), namespaces)) + } else { + name <- name_en + } + + source_institutions <- list() + doi_details <- NULL + + annotations_nodes <- xml2::xml_find_all(dataflow, ".//c:Annotation", namespaces) + for (node in annotations_nodes) { + title <- xml2::xml_text(xml2::xml_find_first(node, ".//c:AnnotationTitle", namespaces)) + type <- xml2::xml_text(xml2::xml_find_first(node, ".//c:AnnotationType", namespaces)) + texts_nodes <- xml2::xml_find_all(node, ".//c:AnnotationText", namespaces) + + + # Assign specific annotations based on type + if (type == "OBS_PERIOD_OVERALL_LATEST") { + latest_period_timestamp <- title # Directly store the latest period timestamp + } else if (type == "OBS_PERIOD_OVERALL_OLDEST") { + oldest_period_timestamp <- title + } else if (type == "UPDATE_DATA") { + update_data_timestamp <- title + } else if(type == "SOURCE_INSTITUTIONS"){ + if (!is.null(lang)) { + source_institutions <- xml2::xml_text( + xml2::xml_find_all(node, sprintf(".//c:AnnotationText[@xml:lang='%s']", lang)) + ) + } else { + source_institutions <- xml2::xml_text( + xml2::xml_find_all(node, ".//c:AnnotationText[@xml:lang='en']") + ) + } + } + + if (grepl("adms:Identifier", title)) { + title_xml <- xml2::read_xml(title) + doi_url <- xml2::xml_attr(xml2::xml_find_first(title_xml, ".//adms:Identifier"), "rdf:about", xml2::xml_ns(title_xml)) + } + } + + # # Extract DOI URL if the annotation contains adms:Identifier + # if (grepl("adms:Identifier", title)) { + # title_xml <- xml2::read_xml(title) + # doi_url <- xml2::xml_attr(xml2::xml_find_first(title_xml, ".//adms:Identifier"), "rdf:about", xml2::xml_ns(title_xml)) + # } + + + metadata <- list( + name = name, + name_en = name_en, + name_de = name_de, + name_fr = name_fr, + doi_url = ifelse(exists("doi_url"), eval(doi_url), NA_character_), + dataflow_id = dataflow_id, + agency_id = agencyID, + id = header_id, + prepared = prepared, + sender_id = sender_id, + oldest_period_timestamp = oldest_period_timestamp, + latest_period_timestamp = latest_period_timestamp, + update_data_timestamp = update_data_timestamp, + source_institutions = source_institutions, + urn = urn, + version = version, + is_final = isFinal + ) + + return(metadata) + +} + +legacy_data_format <- function(x, cols_to_drop = c("DATAFLOW", "LAST UPDATE", "freq")) { + if (inherits(x, "data.table")) { + # Drop columns that were not used in old API + for (i in seq_along(cols_to_drop)) { + if (cols_to_drop[i] %in% names(x)){ + x[, (cols_to_drop[i]):=NULL] + } + } + + # Rename columns + non_legacy_col_name = c("TIME_PERIOD", "OBS_VALUE", "OBS_FLAG") + legacy_col_name = c("time", "values", "flags") + for (i in seq_along(non_legacy_col_name)) { + if (non_legacy_col_name[i] %in% names(x)){ + data.table::setnames(x, (non_legacy_col_name[i]), (legacy_col_name[i])) + } + } + + } else { + x <- x[setdiff(names(x), cols_to_drop)] + cols_to_rename <- data.frame(non_legacy_col_name = c("TIME_PERIOD", "OBS_VALUE", "OBS_FLAG"), + legacy_col_name = c("time", "values", "flags")) + for (i in seq_len(nrow(cols_to_rename))) { + non_legacy_col_name <- cols_to_rename[i,1] + if (non_legacy_col_name %in% names(x)) { + col_num <- which(names(x) == non_legacy_col_name) + colnames(x)[col_num] <- cols_to_rename[i,2] + } + } + } + x +} + +build_api_base_uri <- function(agency) { + agency <- tolower(agency) + api_base_uri <- switch( + agency, + eurostat = "https://ec.europa.eu/eurostat/api/dissemination", + estat = "https://ec.europa.eu/eurostat/api/dissemination", + eurostat_comext = "https://ec.europa.eu/eurostat/api/comext/dissemination", + comext = "https://ec.europa.eu/eurostat/api/comext/dissemination", + eurostat_prodcom = "https://ec.europa.eu/eurostat/api/comext/dissemination", + prodcom = "https://ec.europa.eu/eurostat/api/comext/dissemination", + comp = "https://webgate.ec.europa.eu/comp/redisstat/api/dissemination", + empl = "https://webgate.ec.europa.eu/empl/redisstat/api/dissemination", + grow = "https://webgate.ec.europa.eu/grow/redisstat/api/dissemination") + + api_base_uri +} + +build_agencyID <- function(agency) { + agency <- tolower(agency) + agencyID <- switch( + agency, + eurostat = "ESTAT", + estat = "ESTAT", + eurostat_comext = "ESTAT", + comext = "ESTAT", + eurostat_prodcom = "ESTAT", + prodcom = "ESTAT", + comp = "COMP", + empl = "EMPL", + grow = "GROW" + ) + + agencyID +} + +data_filtering_on_dimension <- function(agency, id, filters) { + # data_structure_definition_url <- paste0( + # api_base_url, + # "/sdmx/2.1/datastructure/estat/", + # id) + + filter_names <- toupper(names(filters)) + + dimension_df <- get_codelist_id(agency = agency, + id = id) + dimension_id_upper <- dimension_df$dimension_id_upper + + if (!rlang::is_empty(setdiff(filter_names, dimension_id_upper))) { + stop(paste0("Use valid filter dimensions in the correct order: ", paste(dimension_df$dimension_id, collapse = "."))) + } + + # Assumes that dimensions are listed in the order of their positions + # If there is an example to the contrary somewhere, this should be changed + filter_string <- "" + for (i in seq_along(dimension_df$dimension_id_upper)){ + if (dimension_df$dimension_id_upper[i] %in% filter_names) { + x <- paste(filters[[dimension_df$dimension_id_upper[i]]], collapse = "+") + filter_string <- paste0(filter_string, x, ".") + } else { + filter_string <- paste0(filter_string, ".") + } + if (i == length(dimension_df$dimension_id_upper)) { + # Remove final dot + filter_string <- substr(filter_string, 1, nchar(filter_string)-1) + } + } + return(filter_string) +} + +get_codelist_id <- function(agency, id) { + + api_base_uri <- build_api_base_uri(agency) + + data_structure_definition_url <- paste0( + api_base_uri, + "/sdmx/2.1/datastructure/ESTAT/", + id) + + dsd <- xml2::read_xml(data_structure_definition_url) + + # dimension_id <- xml2::xml_text(xml2::xml_find_all(xml2::xml_find_all(dsd, ".//s:Dimension"), ".//Ref[@class='Codelist']/@id")) + dimension_id <- xml2::xml_text(xml2::xml_find_all(dsd, ".//s:Dimension/@id")) + dimension_position <- xml2::xml_text(xml2::xml_find_all(dsd, ".//s:Dimension/@position")) + codelist_id <- xml2::xml_text(xml2::xml_find_all(dsd, ".//s:Dimension/s:LocalRepresentation/s:Enumeration/Ref/@id")) + dimension_id_upper <- toupper(dimension_id) + dimension_df <- data.frame(dimension_position, dimension_id, dimension_id_upper, codelist_id) + + dimension_df +} + +# references_resolution <- function(api_base_uri, resource, agencyID, resourceID) { +# # resource <- "datastructure" +# +# url <- paste0( +# api_base_uri, +# "/sdmx/2.1/", +# resource, +# "/", +# agencyID, +# "/", +# resourceID +# ) +# +# parsed <- xml2::read_xml(url) +# +# } + +# get_codelist <- function(api_base_uri, id, dimension_id) { +# +# } + +label_eurostat_sdmx <- function(x, agency, id, lang = "en", verbose = TRUE) { + # how many columns there are that can be labeled with a codelist + dimension_df <- get_codelist_id(agency = agency, id = id) + resource <- "codelist" + lang <- check_lang(lang) + + # non-destructive editing + y <- x + api_base_uri <- build_api_base_uri(agency) + agencyID <- build_agencyID(agency) + + agencyID <- agencyID + # data.table objects need different kind of handling + if (inherits(y, "data.table")) { + for (i in seq_len(nrow(dimension_df))) { + resourceID <- dimension_df$codelist_id[[i]] + if (verbose) message(paste("Building codelist URL for resourceID:", resourceID)) + codelist_url <- paste0( + api_base_uri, + "/sdmx/2.1/", + resource, + "/", + agencyID, + "/", + resourceID, + "?format=TSV", + "&lang=", + lang + ) + tryCatch({ + codelist <- as.data.frame(readr::read_tsv(file = codelist_url, col_types = "cc", col_names = FALSE)) + column_to_handle <- dimension_df$dimension_id[[i]] + codes_to_label <- unique(y[[column_to_handle]]) + codelist_subset <- codelist[which(codelist[,1] %in% codes_to_label),] + if (verbose) message(paste("Labeling dimension (column):", column_to_handle)) + for (j in seq_len(nrow(codelist_subset))) { + data.table::set(y, i=which(y[[column_to_handle]] == codelist_subset[j,1]), j = column_to_handle, value = codelist_subset[j,2]) + } + }, + error = function(e) message(paste("Couldn't label", resourceID)) + ) + } + } else { + for (i in seq_len(nrow(dimension_df))) { + resourceID <- dimension_df$codelist_id[[i]] + message(paste("Building codelist URL for resourceID:", resourceID)) + codelist_url <- paste0( + api_base_uri, + "/sdmx/2.1/", + resource, + "/", + agencyID, + "/", + resourceID, + "?format=TSV", + "&lang=", + lang + ) + tryCatch({ + codelist <- as.data.frame(readr::read_tsv(file = codelist_url, col_types = "cc", col_names = FALSE)) + column_to_handle <- dimension_df$dimension_id[[i]] + col <- x[[column_to_handle]] + codes_to_label <- codelist[(codelist[,1] %in% col),] + message(paste("Labeling dimension (column):", column_to_handle)) + for (j in seq_len(nrow(codes_to_label))) { + col[which(col == as.character(codes_to_label[j,][1]))] <- as.character(codes_to_label[j,2]) + } + x[column_to_handle] <- col + }, + error = function(e) message(paste("Couldn't label", resourceID)) + ) + } + if ("OBS_FLAG" %in% names(x)){ + tryCatch({ + resourceID <- "OBS_FLAG" + message(paste("Building codelist URL for resourceID:", resourceID)) + codelist_url <- paste0( + api_base_uri, + "/sdmx/2.1/", + resource, + "/", + agencyID, + "/", + resourceID, + "?format=TSV", + "&lang=", + lang + ) + codelist <- as.data.frame(readr::read_tsv(file = codelist_url, col_types = "cc")) + column_to_handle <- "OBS_FLAG" + col <- x[[column_to_handle]] + codes_to_label <- codelist[(codelist[,1] %in% col),] + message(paste("Labeling dimension (column):", column_to_handle)) + for (j in seq_len(nrow(codes_to_label))) { + col[which(col == as.character(codes_to_label[j,][1]))] <- as.character(codes_to_label[j,2]) + } + x[column_to_handle] <- col + }, + error = function(e) message(paste("Couldn't label", resourceID)) + ) + } + } + x +} diff --git a/R/label_eurostat.R b/R/label_eurostat.R index bf2ad8bf..e33502df 100755 --- a/R/label_eurostat.R +++ b/R/label_eurostat.R @@ -93,9 +93,9 @@ label_eurostat <- custom_dic = NULL, fix_duplicated = FALSE) { - # Check if you have access to ec.europe.eu. + # Check if you have access to ec.europa.eu. if (!check_access_to_data()) { - message(paste("You have no access to ec.europe.eu. Please check your", + message(paste("You have no access to ec.europa.eu. Please check your", "connection and/or review your proxy settings")) } else { diff --git a/R/search_eurostat.R b/R/search_eurostat.R index ae439dbc..6d56c038 100755 --- a/R/search_eurostat.R +++ b/R/search_eurostat.R @@ -66,9 +66,9 @@ search_eurostat <- function(pattern, column <- "title" } - # Check if you have access to ec.europe.eu. + # Check if you have access to ec.europa.eu. if (!check_access_to_data()) { - message("You have no access to ec.europe.eu. + message("You have no access to ec.europa.eu. Please check your connection and/or review your proxy settings") return(invisible()) } diff --git a/R/set_eurostat_cache.R b/R/set_eurostat_cache.R index eaf66c6e..eee4ad81 100644 --- a/R/set_eurostat_cache.R +++ b/R/set_eurostat_cache.R @@ -210,9 +210,9 @@ eur_helper_cachedir <- function(cache_dir = NULL) { #' @description #' Parses cache_list.json file and returns a data.frame #' @return -#' A data.frame object with 3 columns: dataset code, download date and +#' A data.frame object with 3 columns: dataset code, download date and #' query md5 hash -#' +#' #' @inheritParams get_eurostat #' @importFrom jsonlite fromJSON #' @export @@ -220,18 +220,18 @@ list_eurostat_cache_items <- function(cache_dir = NULL) { if (is.null(cache_dir)) { cache_dir <- eur_helper_detect_cache_dir() } - + path <- paste0(cache_dir, "/cache_list.json") - + if (!file.exists(path)) { warning("No cache_list.json file found!") return(invisible()) } - + json_file <- jsonlite::fromJSON(path) # return(json_file) columns <- c("code", "download_date", "query_md5_hash") - df <- data.frame(matrix(nrow = 0, ncol = length(columns))) + df <- data.frame(matrix(nrow = 0, ncol = length(columns))) for (i in seq_len(length(json_file))) { df_i <- data.frame( code = json_file[[i]][["id"]], @@ -241,4 +241,4 @@ list_eurostat_cache_items <- function(cache_dir = NULL) { df <- rbind(df, df_i) } df -} \ No newline at end of file +} diff --git a/R/tidy_eurostat.R b/R/tidy_eurostat.R index ee3e9aa9..2807b836 100755 --- a/R/tidy_eurostat.R +++ b/R/tidy_eurostat.R @@ -1,5 +1,5 @@ -#' @title Transform Data into Row-Column-Value Format -#' @description Transform raw Eurostat data table downloaded from the +#' @title Transform TSV files into Row-Column-Value Format +#' @description Transform raw Eurostat TSV files downloaded from the #' API into a tidy row-column-value format (RCV). #' @param dat #' a data_frame from [get_eurostat_raw()]. @@ -78,7 +78,7 @@ tidy_eurostat <- function(dat, dat$values <- as.numeric(gsub("[^0-9.-]+", "", as.character(dat$values))) # variable columns - var_cols <- names(dat)[!(names(dat) %in% c("TIME_PERIOD", "values"))] + var_cols <- setdiff(names(dat), c("TIME_PERIOD", "values")) # reorder to standard order # OLD CODE @@ -107,8 +107,8 @@ tidy_eurostat <- function(dat, dat <- data.table::melt(data = dat, measure.vars = setdiff(names(dat), cnames1), variable.name = cnames2, - value.name = "values") %>% - na.omit(cols = "values") + value.name = "values", na.rm = TRUE)# %>% + # stats::na.omit(cols = "values") # NEW CODE: data.table # should be using S3 method for data.table here... @@ -250,3 +250,124 @@ convert_time_col <- function(x, time_format) { } y } + +#' @title Transform CSV files into Row-Column-Value Format data.table object +#' @description Transform raw Eurostat TSV files downloaded from the +#' API into a tidy row-column-value format (RCV). +#' @details +#' Can read only SDMX-CSV files, in compressed (.csv.gz) and uncompressed (.csv) +#' format. As opposed to regular tidy_eurostat function, stringsAsFactors is +#' determined outside this function, in `get_eurostat_local()`. +#' +#' @param dat +#' a data_frame from [get_eurostat_raw()]. +#' @inheritParams get_eurostat +#' @return data.table object in the melted format with the last column 'values' or 'OBS_VALUE'. +#' @seealso [get_eurostat_local()] +#' @inherit eurostat-package references +#' @author Pyry Kantanen +#' +#' @importFrom stringi stri_extract_first_regex stri_replace_all_regex +#' @importFrom stringi stri_replace_all_fixed +#' @importFrom tidyr separate pivot_longer +#' @importFrom dplyr filter +#' @importFrom data.table setDT melt .SD := setcolorder +#' @importFrom stats na.omit +#' @importFrom rlang .data +#' +#' @keywords internal utilities database +tidy_eurostat_sdmx_csv <- function(dat, + time_format = "date", + select_time = NULL, + keepFlags = FALSE) { + + # Fix for NSE NOTEs using data.table + OBS_FLAG = OBS_VALUE = NULL + + dat <- dat %>% + dplyr::filter(!is.na(.data$OBS_VALUE)) + + if (keepFlags == FALSE) { + # Remove OBS_FLAG column + dat[,OBS_FLAG:=NULL] + } + + # clean time and values + dat$TIME_PERIOD <- gsub("X", "", dat$TIME_PERIOD, fixed = TRUE) + dat$OBS_VALUE <- as.numeric(gsub("[^0-9.-]+", "", as.character(dat$OBS_VALUE))) + + # variable columns + var_cols <- setdiff(names(dat), c("TIME_PERIOD", "OBS_VALUE")) + + # reorder to standard order + data.table::setcolorder(dat, c(var_cols, "TIME_PERIOD", "OBS_VALUE")) + + # columns from var_cols are converted into factors + # avoid convert = FALSE since it converts T into TRUE instead of TOTAL + # not needed in data.table + # if (stringsAsFactors) { + # dat[, var_cols] <- lapply( + # dat[, var_cols, drop = FALSE], + # function(x) factor(x, levels = unique(x)) + # ) + # } + + # For multiple time frequency + freqs <- unique(dat$freq) + + if (!is.null(select_time)) { + if (length(select_time) > 1) { + message( + "Selected multiple time frequencies with select_time parameter: ", + shQuote(select_time) + ) + } + + # Filter dataset according to select_time filter + # This only works when a single frequency is chosen + if (identical(select_time, "Y")) { + # Annual with old style notation, "Y" for annual + dat <- subset(dat, dat$freq == "A") + } else if (identical(select_time, "A")) { + # Annual with new notation, "A" for annual + dat <- subset(dat, dat$freq == "A") + } else { + # Others, subset the data with whatever choices + dat <- subset(dat, dat$freq %in% select_time) + } + # Test if filtered dataset actually contains any data + if (nrow(dat) == 0) { + stop( + "No data selected with select_time:", dQuote(select_time), "\n", + "Available frequencies: ", shQuote(freqs) + ) + } + } else { + + if (length(freqs) > 1 && time_format != "raw") { + message( + "Data includes several time frequencies. Select a single frequency \n", + "with select_time or use time_format = \"raw\" to return all data \n", + "without any filtering. Available frequencies: ", shQuote(freqs), "\n", + "Returning the dataset with multiple frequencies." + ) + } + } + + if (length(select_time) > 1) { + dat_copy <- data.frame() + for (i in seq_along(select_time)) { + dat_subset <- subset(dat, dat$freq == select_time[i]) + dat_subset$TIME_PERIOD <- convert_time_col(x = dat_subset$TIME_PERIOD, + time_format = time_format) + dat_copy <- rbind(dat_copy, dat_subset) + } + dat <- dat_copy + } else { + # convert time column to Date + dat$TIME_PERIOD <- convert_time_col(x = dat$TIME_PERIOD, + time_format = time_format) + } + + dat +} diff --git a/README.Rmd b/README.Rmd index 33ff09ff..4365f9f6 100755 --- a/README.Rmd +++ b/README.Rmd @@ -106,13 +106,11 @@ Contributions are very welcome: ### Acknowledgements -**Kindly cite this package** by citing the following R Journal article: +**Kindly cite this work** as follows: -[Lahti L.](https://github.com/antagomir), Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. [The R Journal 9(1), pp. 385-392](https://journal.r-project.org/archive/2017/RJ-2017-019/index.html). doi: [10.32614/RJ-2017-019](https://doi.org/10.32614/RJ-2017-019). - -In addition, please provide a citation to the specific software version used: - -Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data [Computer software]. R package version 4.0.0.9003. https://github.com/rOpenGov/eurostat +```{r, comment = "", highlight=FALSE} +print(citation("eurostat"), bibtex = TRUE) +``` We are grateful to all [contributors](https://github.com/ropengov/eurostat/graphs/contributors), including Daniel Antal, Joona Lehtomäki, Francois Briatte, and Oliver Reiter, and for the [Eurostat](https://ec.europa.eu/eurostat/) open data portal! This project is part of [rOpenGov](https://ropengov.org). diff --git a/README.md b/README.md index db0aa2e0..dfc45436 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ + [![rOG-badge](https://ropengov.github.io/rogtemplate/reference/figures/ropengov-badge.svg)](https://ropengov.org/) @@ -23,15 +24,25 @@ GitHub](https://img.shields.io/github/stars/ropengov/eurostat.svg?style=social)] [![Follow](https://img.shields.io/twitter/follow/ropengov.svg?style=social)](https://twitter.com/ropengov) + + + + + + + + + + # eurostat R package @@ -81,14 +92,14 @@ passengers <- search_eurostat("passenger transport") knitr::kable(head(passengers)) ``` -| title | code | type | last.update.of.data | last.table.structure.change | data.start | data.end | values | hierarchy | -|:---------------------------------------------------------------------------------------------|:-----------------|:--------|:--------------------|:----------------------------|:-----------|:---------|--------:|----------:| -| Air passenger transport | enps_avia_pa | dataset | 13.03.2023 | 13.03.2023 | 2005 | 2021 | 406 | 6 | -| Modal split of air, sea and inland passenger transport | tran_hv_ms_psmod | dataset | 29.06.2023 | 29.06.2023 | 2008 | 2021 | 2100 | 4 | -| Modal split of inland passenger transport | tran_hv_psmod | dataset | 29.06.2023 | 29.06.2023 | 1990 | 2021 | 4219 | 4 | -| Volume of passenger transport relative to GDP | tran_hv_pstra | dataset | 11.08.2023 | 29.06.2023 | 1990 | 2021 | 969 | 4 | -| Maritime passenger transport performed in the Exclusive Economic Zone (EEZ) of the countries | mar_tp_pa | dataset | 21.02.2023 | 21.02.2023 | 2005 | 2021 | 1752 | 4 | -| Air passenger transport by reporting country | avia_paoc | dataset | 04.12.2023 | 28.11.2023 | 1993 | 2023-Q3 | 2482969 | 5 | +| title | code | type | last.update.of.data | last.table.structure.change | data.start | data.end | values | hierarchy | +|:---|:---|:---|:---|:---|:---|:---|---:|---:| +| Air passenger transport - ENP-South countries | enps_avia_pa | dataset | 12.05.2025 | 12.05.2025 | 2005 | 2023 | 480 | 6 | +| Air passenger transport by type of schedule, transport coverage and country | avia_paoc | dataset | 15.09.2025 | 15.09.2025 | 1993 | 2025-Q2 | 2524910 | 5 | +| Air passenger transport by type of schedule, transport coverage and main airports | avia_paoa | dataset | 15.09.2025 | 15.09.2025 | 1993 | 2025-Q2 | 20609595 | 5 | +| Air passenger transport between reporting and partner countries by type of schedule | avia_paocc | dataset | 15.09.2025 | 15.09.2025 | 1993 | 2025-Q2 | 10518367 | 5 | +| Air passenger transport between main airports and partner reporting countries | avia_paoac | dataset | 15.09.2025 | 15.09.2025 | 1993 | 2025-Q2 | 20189218 | 5 | +| Air passenger transport by aircraft model, distance bands and transport coverage | avia_paodis | dataset | 16.06.2025 | 06.12.2024 | 2008 | 2023 | 852432 | 5 | See the [Tutorial](https://ropengov.github.io/eurostat/articles/articles/eurostat_tutorial.html) @@ -116,21 +127,51 @@ Contributions are very welcome: ### Acknowledgements -**Kindly cite this package** by citing the following R Journal article: +**Kindly cite this work** as follows: + +``` text +print(citation("eurostat"), bibtex = TRUE) +Kindly cite this package by citing the following R Journal article: -[Lahti L.](https://github.com/antagomir), Huovari J., Kainu M., and -Biecek P. (2017). Retrieval and analysis of Eurostat open data with the -eurostat package. [The R Journal 9(1), -pp. 385-392](https://journal.r-project.org/archive/2017/RJ-2017-019/index.html). -doi: [10.32614/RJ-2017-019](https://doi.org/10.32614/RJ-2017-019). + Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and + analysis of Eurostat open data with the eurostat package. The R + Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 + +A BibTeX entry for LaTeX users is + + @Article{10.32614/RJ-2017-019, + title = {Retrieval and Analysis of Eurostat Open Data with the eurostat Package}, + author = {Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek}, + journal = {The R Journal}, + volume = {9}, + number = {1}, + pages = {385--392}, + year = {2017}, + doi = {10.32614/RJ-2017-019}, + url = {https://doi.org/10.32614/RJ-2017-019}, + } In addition, please provide a citation to the specific software version used: -Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., -and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data -\[Computer software\]. R package version 4.0.0.9003. - + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2025). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9003, . + +A BibTeX entry for LaTeX users is + + @Misc{R-eurostat, + title = {eurostat: Tools for Eurostat Open Data}, + doi = {10.32614/CRAN.package.eurostat}, + author = {Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen}, + url = {https://github.com/rOpenGov/eurostat}, + type = {Computer software}, + year = {2025}, + note = {R package version 4.1.0.9003}, + } +``` We are grateful to all [contributors](https://github.com/ropengov/eurostat/graphs/contributors), @@ -138,7 +179,10 @@ including Daniel Antal, Joona Lehtomäki, Francois Briatte, and Oliver Reiter, and for the [Eurostat](https://ec.europa.eu/eurostat/) open data portal! This project is part of [rOpenGov](https://ropengov.org). -This project has received funding from the European Union under grant No 101095295 (OpenMUSE), the FIN-CLARIAH research infrastructure and the Strategic Research Council's YOUNG program by the Research Council of Finland (decisions 345630, 358720, 367756, 352604). +This project has received funding from the European Union under grant No +101095295 (OpenMUSE), the FIN-CLARIAH research infrastructure and the +Strategic Research Council’s YOUNG program by the Research Council of +Finland (decisions 345630, 358720, 367756, 352604). ### Disclaimer diff --git a/data-raw/eurostat_geodata_60_2024.R b/data-raw/eurostat_geodata_60_2024.R new file mode 100644 index 00000000..25f8745f --- /dev/null +++ b/data-raw/eurostat_geodata_60_2024.R @@ -0,0 +1,35 @@ +## code to prepare `eurostat_geodata_60_2016` dataset goes here +library(giscoR) +library(tidyverse) + +# Current names internal) +from_gisco <- gisco_get_nuts( + year = 2024, resolution = 60, + epsg = 4326, update_cache = TRUE, + verbose = TRUE +) + +from_gisco$geo <- from_gisco$NUTS_ID +from_gisco$id <- from_gisco$NUTS_ID + +# End + +eurostat_geodata_60_2024 <- from_gisco +unique(sf::st_is_valid(from_gisco)) + +# Sort by level and alphabetically +eurostat_geodata_60_2024 <- eurostat_geodata_60_2024 %>% + arrange(LEVL_CODE, NUTS_ID) + +# Arrange names in proper order +sfcol <- attr(eurostat_geodata_60_2024, "sf_column") +rest <- c( + "id", "LEVL_CODE", "NUTS_ID", "CNTR_CODE", "NAME_LATN", + "NUTS_NAME", "MOUNT_TYPE", "URBN_TYPE", "COAST_TYPE", + "FID", "geo" +) + +reorder <- intersect(unique(c(rest, sfcol)), names(eurostat_geodata_60_2024)) +eurostat_geodata_60_2024 <- eurostat_geodata_60_2024[, reorder] + +usethis::use_data(eurostat_geodata_60_2024, overwrite = TRUE, compress = "xz") diff --git a/data/ea_countries.rda b/data/ea_countries.rda index 76ecea1d..8dbfd741 100644 Binary files a/data/ea_countries.rda and b/data/ea_countries.rda differ diff --git a/data/efta_countries.rda b/data/efta_countries.rda index f4c0124c..8c61df74 100644 Binary files a/data/efta_countries.rda and b/data/efta_countries.rda differ diff --git a/data/eu_candidate_countries.rda b/data/eu_candidate_countries.rda index 4c790ec4..37bcd084 100644 Binary files a/data/eu_candidate_countries.rda and b/data/eu_candidate_countries.rda differ diff --git a/data/eu_countries.rda b/data/eu_countries.rda index b9305db0..761a93e0 100644 Binary files a/data/eu_countries.rda and b/data/eu_countries.rda differ diff --git a/data/eurostat_geodata_60_2016.rda b/data/eurostat_geodata_60_2016.rda deleted file mode 100644 index 0e453269..00000000 Binary files a/data/eurostat_geodata_60_2016.rda and /dev/null differ diff --git a/data/eurostat_geodata_60_2024.rda b/data/eurostat_geodata_60_2024.rda new file mode 100644 index 00000000..a3f4cac5 Binary files /dev/null and b/data/eurostat_geodata_60_2024.rda differ diff --git a/inst/CITATION b/inst/CITATION index ca7e261b..36b23c4d 100755 --- a/inst/CITATION +++ b/inst/CITATION @@ -1,9 +1,11 @@ -citHeader("Kindly cite the eurostat R package as follows:") - year <- sub("-.*", "", meta$Date) -vers <- paste("R package version", meta$Version) +version_note <- paste("R package version", meta$Version) +pkg <- meta$Package +title <- gsub("'", "", meta$Title) +doi <- paste0("10.32614/CRAN.package.", pkg) bibentry("Article", + header = "Kindly cite this package by citing the following R Journal article:", key = "10.32614/RJ-2017-019", title = "Retrieval and Analysis of Eurostat Open Data with the eurostat Package", author = c( @@ -28,8 +30,10 @@ bibentry("Article", ) bibentry(bibtype = "Misc", - key = "eurostat", + header = "In addition, please provide a citation to the specific software version used:", + key = paste0("R-", pkg), title = "eurostat: Tools for Eurostat Open Data", + doi = doi, author = c( person("Leo", "Lahti"), person("Janne", "Huovari"), @@ -42,13 +46,5 @@ bibentry(bibtype = "Misc", url = "https://github.com/rOpenGov/eurostat", type = "Computer software", year = year, - note = vers, - textVersion = - paste0( - "Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., and Kantanen P. (", - sub("-.*", "", meta$Date), - "). eurostat: Tools for Eurostat Open Data [Computer software]. ", - vers, - ". https://github.com/rOpenGov/eurostat" - ) -) \ No newline at end of file + note = version_note +) diff --git a/man/check_access_to_data.Rd b/man/check_access_to_data.Rd index 836794f6..a588bbe6 100755 --- a/man/check_access_to_data.Rd +++ b/man/check_access_to_data.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/check_access_to_data.R \name{check_access_to_data} \alias{check_access_to_data} -\title{Check access to ec.europe.eu} +\title{Check access to ec.europa.eu} \usage{ check_access_to_data() } diff --git a/man/cut_to_classes.Rd b/man/cut_to_classes.Rd index eb34cc09..db441b76 100755 --- a/man/cut_to_classes.Rd +++ b/man/cut_to_classes.Rd @@ -38,7 +38,7 @@ Categorises a numeric vector into automatic or manually defined categories and polishes the labels ready for used in mapping with \code{ggplot2}. } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ # lp <- get_eurostat("nama_aux_lp") lp <- get_eurostat("nama_10_lp_ulc") diff --git a/man/eu_countries.Rd b/man/eu_countries.Rd index e7112547..86bef990 100755 --- a/man/eu_countries.Rd +++ b/man/eu_countries.Rd @@ -10,16 +10,19 @@ \format{ A data_frame: \itemize{ -\item \strong{code}: Country code in the Eurostat database. +\item \strong{code}: Country code in the Eurostat database (two-letter ISO code (ISO 3166 alpha-2) except in the case of Greece where EL is used). \item \strong{name}: Country name in English. -\item \strong{label}: Country name in the Eurostat database. +\item \strong{label}: Country name in the Eurostat database +\item \strong{name_fr}: Country name in French +\item \strong{name_de}: Country name in German +\item \strong{country_language}: Country name in national language(s). } -An object of class \code{tbl_df} (inherits from \code{tbl}, \code{data.frame}) with 19 rows and 3 columns. +An object of class \code{data.frame} with 20 rows and 6 columns. -An object of class \code{tbl_df} (inherits from \code{tbl}, \code{data.frame}) with 4 rows and 3 columns. +An object of class \code{data.frame} with 4 rows and 6 columns. -An object of class \code{tbl_df} (inherits from \code{tbl}, \code{data.frame}) with 7 rows and 3 columns. +An object of class \code{data.frame} with 9 rows and 6 columns. } \source{ \url{https://ec.europa.eu/eurostat/statistics-explained/index.php/Tutorial:Country_codes_and_protocol_order}, @@ -38,9 +41,13 @@ eu_candidate_countries Countries and country codes in EU, Euro area, EFTA and EU candidate countries. } +\details{ +Country codes are two-letter ISO codes (ISO 3166 alpha-2) except in the case +of Greece where EL is used instead of the standard ISO code. +} \seealso{ Other datasets: -\code{\link{eurostat_geodata_60_2016}}, +\code{\link{eurostat_geodata_60_2024}}, \code{\link{tgs00026}} } \concept{datasets} diff --git a/man/eurostat-package.Rd b/man/eurostat-package.Rd index 19643fc5..becb6a07 100755 --- a/man/eurostat-package.Rd +++ b/man/eurostat-package.Rd @@ -15,8 +15,8 @@ utilities. \tab \cr \strong{Package} \tab eurostat \cr \strong{Type} \tab Package \cr - \strong{Version} \tab 4.0.0 \cr - \strong{Date} \tab 2014-2023 \cr + \strong{Version} \tab 4.1.0.9007 \cr + \strong{Date} \tab 2014-2025 \cr \strong{License} \tab BSD_2_clause + file LICENSE \cr \strong{LazyLoad} \tab yes \cr } @@ -39,7 +39,7 @@ the data." Data is downloaded from Eurostat SDMX 2.1 API endpoint as compressed TSV files that are transformed into tabular format. See Eurostat documentation for more information: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+query} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Dataquery} The new dissemination API replaces the old bulk download facility that was used by Eurostat before October 2023 and by the eurostat R package versions @@ -47,7 +47,7 @@ before 4.0.0. See Eurostat documentation about the transition from Bulk Download to API for more information about the differences between the old bulk download facility and the data provided by the new API connection: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload} See especially the document Migrating_to_API_TSV.pdf that describes the changes in TSV file format in new applications. @@ -64,20 +64,20 @@ datasets downloaded through the SDMX Dissemination API is also supported by Eurostat. We may support this feature in the future. In the meantime, if you are interested in filtering Dissemination API data queries manually, please consult the following Eurostat documentation: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+filtering} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Filteringonseries-keys} } \section{Data source: Eurostat API Statistics (JSON API)}{ Data is downloaded from Eurostat API Statistics. See Eurostat documentation for more information about data queries in API Statistics -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api} This replaces the old JSON Web Services that was used by Eurostat before February 2023 and by the eurostat R package versions before 3.7.13. See Eurostat documentation about the migration from JSON web service to API Statistics for more information about the differences between the old and the new service: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/json} For easily viewing which filtering options are available - in addition to the default ones, time and language - Eurostat Web services Query builder @@ -179,7 +179,7 @@ Example: For more information about data filtering see Eurostat documentation on API Statistics: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} } } @@ -191,7 +191,7 @@ The Eurostat Table of Contents (TOC) is downloaded from \url{https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang=de} See Eurostat documentation on TOC items: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+-+Detailed+guidelines+-+Catalogue+API+-+TOC} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/catalogue-api/toc} } \section{Data source: GISCO - General Copyright}{ @@ -348,41 +348,24 @@ library(eurostat) \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is - - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} - - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +In addition, please provide a citation to the specific software version +used: + + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . + +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/eurostat_geodata_60_2016.Rd b/man/eurostat_geodata_60_2024.Rd old mode 100755 new mode 100644 similarity index 54% rename from man/eurostat_geodata_60_2016.Rd rename to man/eurostat_geodata_60_2024.Rd index fcce0288..ab01a402 --- a/man/eurostat_geodata_60_2016.Rd +++ b/man/eurostat_geodata_60_2024.Rd @@ -1,88 +1,69 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/data_spatial.R \docType{data} -\name{eurostat_geodata_60_2016} -\alias{eurostat_geodata_60_2016} +\name{eurostat_geodata_60_2024} +\alias{eurostat_geodata_60_2024} \title{Geospatial data of Europe from GISCO in 1:60 million scale from -year 2016} +year 2024} \format{ sf object + +The dataset contains 2016 observations (rows) and 12 variables (columns): +\describe{ +\item{id}{JSON id code, the same as NUTS_ID. See NUTS_ID below for further clarification.} +\item{LEVL_CODE}{NUTS level code: 0 (national level), 1 (major socio-economic regions), 2 (basic regions for the application of regional policies) or 3 (small regions).} +\item{NUTS_ID}{NUTS ID code, consisting of country code and numbers (1 for NUTS 1, 2 for NUTS 2 and 3 for NUTS 3)} +\item{CNTR_CODE}{Country code: two-letter ISO code (ISO 3166 alpha-2), except in the case of Greece (EL).} +\item{NAME_LATN}{NUTS name in local language, transliterated to Latin script} +\item{NUTS_NAME}{NUTS name in local language, in local script.} +\item{MOUNT_TYPE}{Mountain typology for NUTS 3 regions. +1: "where more than 50 \% of the surface is covered by topographic mountain areas" +2: "in which more than 50 \% of the regional population lives in topographic mountain areas" +3: "where more than 50 \% of the surface is covered by topographic mountain areas and where more than 50 \% of the regional population lives in these mountain areas" +4: non-mountain region / other region +0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 and non-EU countries)} +\item{URBN_TYPE}{Urban-rural typology for NUTS 3 regions. +1: predominantly urban region +2: intermediate region +3: predominantly rural region +0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 regions)} +\item{COAST_TYPE}{Coastal typology for NUTS 3 regions. +1: coastal (on coast) +2: coastal (>= 50\% of population living within 50km of the coastline) +3: non-coastal region +0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 regions)} +\item{FID}{Same as NUTS_ID} +\item{geo}{Same as NUTS_ID} +\item{geometry}{geospatial information} +} + +Dataset updated: 2023-06-29. For a possibly more recent version, please use +\CRANpkg{giscoR} function \code{gisco_get_nuts()} function. } \source{ -Data source: Eurostat via \code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts()}}. +Data source: Eurostat via giscoR package function \code{gisco_get_nuts()}. © EuroGeographics for the administrative boundaries Data downloaded from: \url{https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units} } +\usage{ +eurostat_geodata_60_2024 +} \description{ Geospatial data of Europe from GISCO in 1:60 million scale -from year 2016 -} -\details{ -The dataset contains 2016 observations (rows) and 12 variables (columns). - -The object contains the following columns: -\itemize{ -\item \strong{id}: JSON id code, the same as \strong{NUTS_ID}. See \strong{NUTS_ID} below for -further clarification. -\item \strong{LEVL_CODE}: NUTS level code: 0 (national level), 1 (major -socio-economic regions), 2 (basic regions for the application of regional -policies) or 3 (small regions). -\item \strong{NUTS_ID}: NUTS ID code, consisting of country code and numbers (1 for -NUTS 1, 2 for NUTS 2 and 3 for NUTS 3) -\item \strong{CNTR_CODE}: Country code: two-letter ISO code (ISO 3166 alpha-2), except -in the case of Greece (EL). -\item \strong{NAME_LATN}: NUTS name in local language, transliterated to Latin script -\item \strong{NUTS_NAME}: NUTS name in local language, in local script. -\item \strong{MOUNT_TYPE}: Mountain typology for NUTS 3 regions. -\itemize{ -\item 1: "where more than 50 \% of the surface is covered by topographic -mountain areas" -\item 2: "in which more than 50 \% of the regional population lives in -topographic mountain areas" -\item 3: "where more than 50 \% of the surface is covered by topographic -mountain areas and where more than 50 \% of the regional population lives -in these mountain areas" -\item 4: non-mountain region / other region -\item 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 and -non-EU countries) -} -\item \strong{URBN_TYPE}: Urban-rural typology for NUTS 3 regions. -\itemize{ -\item 1: predominantly urban region -\item 2: intermediate region -\item 3: predominantly rural region -\item 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 -regions) -} -\item \strong{COAST_TYPE}: Coastal typology for NUTS 3 regions. -\itemize{ -\item 1: coastal (on coast) -\item 2: coastal (>= 50\% of population living within 50km of the coastline) -\item 3: non-coastal region -\item 0: no classification provided (e.g. in the case of NUTS 1 and NUTS 2 -regions) -} -\item \strong{FID}: Same as NUTS_ID. -\item \strong{geo}: Same as NUTS_ID, added for for easier joins with dplyr. However, -it is recommended to use other identical fields for this purpose. -\item \strong{geometry}: geospatial information. -} - -Dataset updated: 2023-06-29. For a more recent version, please use -\code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts()}} function. +from year 2024 } \examples{ -eurostat_geodata_60_2016 <- eurostat::eurostat_geodata_60_2016 +eurostat_geodata_60_2024 <- eurostat::eurostat_geodata_60_2024 # Manipulate and plot if (require(sf)) { library(sf) # Filter NUTS3 from select countries like in a regular data frame - example_nuts <- subset(eurostat_geodata_60_2016, LEVL_CODE == 3 & + example_nuts <- subset(eurostat_geodata_60_2024, LEVL_CODE == 3 & CNTR_CODE \%in\% c("DK", "DE", "PL")) plot(example_nuts["CNTR_CODE"]) @@ -135,7 +116,7 @@ for information regarding their licence agreements." } } \seealso{ -\code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts()}} and +\CRANpkg{giscoR} function \code{gisco_get_nuts()} and \href{https://ec.europa.eu/eurostat/web/products-manuals-and-guidelines/-/ks-gq-18-008}{Eurostat. (2019). Methodological manual on territorial typologies -- 2018 edition. Manuals and guidelines.} Other datasets: @@ -147,3 +128,4 @@ Other geospatial: } \concept{datasets} \concept{geospatial} +\keyword{datasets} diff --git a/man/eurotime2date.Rd b/man/eurotime2date.Rd index 06a925c7..fddedf19 100755 --- a/man/eurotime2date.Rd +++ b/man/eurotime2date.Rd @@ -14,11 +14,11 @@ the first date of the period (month, quarter or year). If \code{TRUE} the date is the last date of the period.} } \value{ -an object of class \code{\link[=Date]{Date()}}. +an object of class \code{\link[lubridate:date_utils]{lubridate::Date()}}. } \description{ Date conversion from Eurostat time format. A function to -convert Eurostat time values to objects of class \code{\link[=Date]{Date()}} +convert Eurostat time values to objects of class \code{\link[lubridate:date_utils]{lubridate::Date()}} representing calendar dates. } \details{ @@ -26,7 +26,7 @@ Available patterns are YYYY (year), YYYY-SN (semester), YYYY-QN (quarter), YYYY-MM (month), YYYY-WNN (week) and YYYY-MM-DD (day). } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ na_q <- get_eurostat("namq_10_pc", time_format = "raw") na_q$TIME_PERIOD <- eurotime2date(x = na_q$TIME_PERIOD) @@ -46,41 +46,24 @@ get_eurostat( \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{# Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{# Kindly cite this package by citing the following R Journal article: # # Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and # analysis of Eurostat open data with the eurostat package. The R # Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 # -# A BibTeX entry for LaTeX users is +# In addition, please provide a citation to the specific software version +# used: # -# @Article\{10.32614/RJ-2017-019, -# title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, -# journal = \{The R Journal\}, -# volume = \{9\}, -# number = \{1\}, -# pages = \{385--392\}, -# year = \{2017\}, -# doi = \{10.32614/RJ-2017-019\}, -# url = \{https://doi.org/10.32614/RJ-2017-019\}, -# \} +# Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, +# Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." +# doi:10.32614/CRAN.package.eurostat +# , R package version +# 4.1.0.9007, . # -# Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., -# and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data -# [Computer software]. R package version 4.0.0. -# https://github.com/rOpenGov/eurostat -# -# A BibTeX entry for LaTeX users is -# -# @Misc\{eurostat, -# title = \{eurostat: Tools for Eurostat Open Data\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, -# url = \{https://github.com/rOpenGov/eurostat\}, -# type = \{Computer software\}, -# year = \{2023\}, -# note = \{R package version 4.0.0\}, -# \} +# To see these entries in BibTeX format, use 'print(, +# bibtex=TRUE)', 'toBibtex(.)', or set +# 'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} } \seealso{ diff --git a/man/eurotime2num.Rd b/man/eurotime2num.Rd index 5e408d4d..ebc9137a 100755 --- a/man/eurotime2num.Rd +++ b/man/eurotime2num.Rd @@ -21,7 +21,7 @@ a fraction of the year in beginning of the period. Conversion of daily data is not supported. } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ na_q <- get_eurostat("namq_10_pc", time_format = "raw") na_q$TIME_PERIOD <- eurotime2num(x = na_q$TIME_PERIOD) diff --git a/man/figures/lifecycle-deprecated.svg b/man/figures/lifecycle-deprecated.svg new file mode 100644 index 00000000..b61c57c3 --- /dev/null +++ b/man/figures/lifecycle-deprecated.svg @@ -0,0 +1,21 @@ + + lifecycle: deprecated + + + + + + + + + + + + + + + lifecycle + + deprecated + + diff --git a/man/figures/lifecycle-experimental.svg b/man/figures/lifecycle-experimental.svg new file mode 100644 index 00000000..5d88fc2c --- /dev/null +++ b/man/figures/lifecycle-experimental.svg @@ -0,0 +1,21 @@ + + lifecycle: experimental + + + + + + + + + + + + + + + lifecycle + + experimental + + diff --git a/man/figures/lifecycle-stable.svg b/man/figures/lifecycle-stable.svg new file mode 100644 index 00000000..9bf21e76 --- /dev/null +++ b/man/figures/lifecycle-stable.svg @@ -0,0 +1,29 @@ + + lifecycle: stable + + + + + + + + + + + + + + + + lifecycle + + + + stable + + + diff --git a/man/figures/lifecycle-superseded.svg b/man/figures/lifecycle-superseded.svg new file mode 100644 index 00000000..db8d757f --- /dev/null +++ b/man/figures/lifecycle-superseded.svg @@ -0,0 +1,21 @@ + + lifecycle: superseded + + + + + + + + + + + + + + + lifecycle + + superseded + + diff --git a/man/get_bibentry.Rd b/man/get_bibentry.Rd index 76a800be..30600b0b 100755 --- a/man/get_bibentry.Rd +++ b/man/get_bibentry.Rd @@ -4,7 +4,7 @@ \alias{get_bibentry} \title{Create A Data Bibliography} \usage{ -get_bibentry(code, keywords = NULL, format = "Biblatex", lang = "en") +get_bibentry(code, keywords = NULL, format = "bibtex", lang = "en") } \arguments{ \item{code}{A Eurostat data code or a vector of Eurostat data codes as @@ -55,7 +55,7 @@ in \code{\link[=get_eurostat]{get_eurostat()}} documentation. } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \dontrun{ my_bibliography <- get_bibentry( code = c("tran_hv_frtra", "tec00001"), @@ -70,7 +70,7 @@ in \code{\link[=get_eurostat]{get_eurostat()}} documentation. \dontshow{\}) # examplesIf} } \seealso{ -\link[utils:bibentry]{utils::bibentry} \link[RefManageR:toBiblatex]{RefManageR::toBiblatex} +\link[utils:bibentry]{utils::bibentry} } \author{ Daniel Antal, Przemyslaw Biecek diff --git a/man/get_eurostat.Rd b/man/get_eurostat.Rd index 1e0ee2a8..b6887077 100755 --- a/man/get_eurostat.Rd +++ b/man/get_eurostat.Rd @@ -18,6 +18,8 @@ get_eurostat( stringsAsFactors = FALSE, keepFlags = FALSE, use.data.table = FALSE, + legacy.data.output = FALSE, + verbose = TRUE, ... ) } @@ -27,9 +29,9 @@ known \code{\link[=search_eurostat]{search_eurostat()}} function can be used to of contents.} \item{time_format}{a string giving a type of the conversion of the time column from the -eurostat format. The default argument "\code{date}" converts to a \code{\link[=Date]{Date()}} class +eurostat format. The default argument "\code{date}" converts to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class with the date being the first day of the period. A "\code{date_last}" argument -converts the dataset date to a \code{\link[=Date]{Date()}} class object with the difference +converts the dataset date to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class object with the difference that the exact date is the last date of the period. Period can be year, semester (half year), quarter, month, or week (See \code{\link[=eurotime2date]{eurotime2date()}} for more information). @@ -86,6 +88,12 @@ can not be used with a \code{filters}.} \item{use.data.table}{Use faster data.table functions? Default is FALSE. On Windows requires that RTools is installed.} +\item{legacy.data.output}{Use legacy column names and data object structure. +Default is FALSE. If TRUE, the object will try to emulate the naming +conventions of eurostat package version 3.7.x and earlier.} + +\item{verbose}{Output messages when downloading data. Default is \code{TRUE}.} + \item{...}{ Arguments passed on to \code{\link[=get_eurostat_json]{get_eurostat_json}} \describe{ @@ -108,9 +116,9 @@ Download data sets from Eurostat \url{https://ec.europa.eu/eurostat} } \details{ Datasets are downloaded from -\href{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API}{the Eurostat SDMX 2.1 API} +\href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat SDMX 2.1 API} in TSV format or from The Eurostat -\href{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query}{API Statistics JSON API}. +\href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api}{API Statistics JSON API}. If only the table \code{id} is given, the whole table is downloaded from the SDMX API. If any \code{filters} are given JSON API is used instead. @@ -252,7 +260,7 @@ Example: For more information about data filtering see Eurostat documentation on API Statistics: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} } } @@ -290,7 +298,7 @@ datasets downloaded through the SDMX Dissemination API is also supported by Eurostat. We may support this feature in the future. In the meantime, if you are interested in filtering Dissemination API data queries manually, please consult the following Eurostat documentation: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+filtering} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Filteringonseries-keys} } \section{Strategies for handling large datasets more efficiently}{ @@ -319,7 +327,7 @@ what different arguments do } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \dontrun{ k <- get_eurostat("nama_10_lp_ulc") k <- get_eurostat("nama_10_lp_ulc", time_format = "num") @@ -335,7 +343,7 @@ options(eurostat_update = FALSE) set_eurostat_cache_dir(file.path(tempdir(), "r_cache2")) k <- get_eurostat("nama_10_lp_ulc") k <- get_eurostat("nama_10_lp_ulc", cache = FALSE) -k <- get_eurostat("avia_gonc", select_time = "Y", cache = FALSE) +k <- get_eurostat("avia_gooc", select_time = "Y", cache = FALSE) dd <- get_eurostat("nama_10_gdp", filters = list( @@ -369,41 +377,24 @@ dd3_filter <- get_eurostat("AVIA_GOR_ME", \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is - - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} - - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +In addition, please provide a citation to the specific software version +used: + + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . + +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/get_eurostat_dic.Rd b/man/get_eurostat_dic.Rd index c236e422..bc69f693 100755 --- a/man/get_eurostat_dic.Rd +++ b/man/get_eurostat_dic.Rd @@ -24,7 +24,7 @@ human-readable labels. To translate codes to labels, use \code{\link[=label_eurostat]{label_eurostat()}}. } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ get_eurostat_dic("crop_pro") @@ -36,41 +36,24 @@ get_eurostat_dic("crop_pro", lang = "fr") \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{# Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{# Kindly cite this package by citing the following R Journal article: # # Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and # analysis of Eurostat open data with the eurostat package. The R # Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 # -# A BibTeX entry for LaTeX users is +# In addition, please provide a citation to the specific software version +# used: # -# @Article\{10.32614/RJ-2017-019, -# title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, -# journal = \{The R Journal\}, -# volume = \{9\}, -# number = \{1\}, -# pages = \{385--392\}, -# year = \{2017\}, -# doi = \{10.32614/RJ-2017-019\}, -# url = \{https://doi.org/10.32614/RJ-2017-019\}, -# \} +# Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, +# Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." +# doi:10.32614/CRAN.package.eurostat +# , R package version +# 4.1.0.9007, . # -# Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., -# and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data -# [Computer software]. R package version 4.0.0. -# https://github.com/rOpenGov/eurostat -# -# A BibTeX entry for LaTeX users is -# -# @Misc\{eurostat, -# title = \{eurostat: Tools for Eurostat Open Data\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, -# url = \{https://github.com/rOpenGov/eurostat\}, -# type = \{Computer software\}, -# year = \{2023\}, -# note = \{R package version 4.0.0\}, -# \} +# To see these entries in BibTeX format, use 'print(, +# bibtex=TRUE)', 'toBibtex(.)', or set +# 'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} } \seealso{ diff --git a/man/get_eurostat_folder.Rd b/man/get_eurostat_folder.Rd index 4551e0db..ac5054a3 100644 --- a/man/get_eurostat_folder.Rd +++ b/man/get_eurostat_folder.Rd @@ -36,14 +36,14 @@ The Eurostat Table of Contents (TOC) is downloaded from \url{https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang=de} See Eurostat documentation on TOC items: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+-+Detailed+guidelines+-+Catalogue+API+-+TOC} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/catalogue-api/toc} } \section{Data source: Eurostat SDMX 2.1 Dissemination API}{ Data is downloaded from Eurostat SDMX 2.1 API endpoint as compressed TSV files that are transformed into tabular format. See Eurostat documentation for more information: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+query} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Dataquery} The new dissemination API replaces the old bulk download facility that was used by Eurostat before October 2023 and by the eurostat R package versions @@ -51,7 +51,7 @@ before 4.0.0. See Eurostat documentation about the transition from Bulk Download to API for more information about the differences between the old bulk download facility and the data provided by the new API connection: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload} See especially the document Migrating_to_API_TSV.pdf that describes the changes in TSV file format in new applications. diff --git a/man/get_eurostat_geospatial.Rd b/man/get_eurostat_geospatial.Rd index 8634fa4c..a387405e 100755 --- a/man/get_eurostat_geospatial.Rd +++ b/man/get_eurostat_geospatial.Rd @@ -15,19 +15,19 @@ get_eurostat_geospatial( output_class = "sf", resolution = "60", nuts_level = "all", - year = "2016", + year = "2024", cache = TRUE, update_cache = FALSE, cache_dir = NULL, crs = "4326", - make_valid = "DEPRECATED", + verbose = TRUE, ... ) } \arguments{ \item{output_class}{Class of object returned, either \code{sf} \verb{simple features} or \code{df} (\code{data_frame}). \code{spdf} output has -been soft-deprecated, the function would switch to \code{sf}.} +been soft-deprecated, the function will switch to \code{sf}.} \item{resolution}{Resolution of the geospatial data. One of \itemize{ @@ -42,16 +42,18 @@ been soft-deprecated, the function would switch to \code{sf}.} "0", "1", "2", "3" or "all" (mimics the original behaviour)} \item{year}{NUTS release year. One of -"2003", "2006", "2010", "2013", "2016" or "2021"} +"2003", "2006", "2010", "2013", "2016", "2021" or "2024"} \item{cache}{a logical whether to do caching. Default is \code{TRUE}.} \item{update_cache}{a logical whether to update cache. Can be set also with \code{options(eurostat_update = TRUE)}} -\item{cache_dir}{a path to a cache directory. See -\code{\link[=set_eurostat_cache_dir]{set_eurostat_cache_dir()}}. If \code{NULL} and the cache dir has not been set -globally the file would be stored in the \code{\link[=tempdir]{tempdir()}}.} +\item{cache_dir}{a path to a cache directory. \code{NULL} (default) uses and creates +'eurostat' directory in the temporary directory defined by base R +\code{\link[=tempdir]{tempdir()}} function. The user can set the cache directory to an existing +directory by using this argument. The cache directory can also be set with +\code{\link[=set_eurostat_cache_dir]{set_eurostat_cache_dir()}} function.} \item{crs}{projection of the map: 4-digit \href{https://spatialreference.org/ref/epsg/}{EPSG code}. One of: @@ -61,34 +63,20 @@ globally the file would be stored in the \code{\link[=tempdir]{tempdir()}}.} \item "3857" - Pseudo-Mercator }} -\item{make_valid}{Deprecated} +\item{verbose}{Output messages when downloading data. Default is \code{TRUE}.} -\item{...}{ - Arguments passed on to \code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts}} - \describe{ - \item{\code{verbose}}{Logical, displays information. Useful for debugging, -default is \code{FALSE}.} - \item{\code{spatialtype}}{Type of geometry to be returned: -\itemize{ -\item \strong{"BN"}: Boundaries - \code{LINESTRING} object. -\item \strong{"LB"}: Labels - \code{POINT} object. -\item \strong{"RG"}: Regions - \code{MULTIPOLYGON/POLYGON} object. -}} - \item{\code{country}}{Optional. A character vector of country codes. It could be -either a vector of country names, a vector of ISO3 country codes or a -vector of Eurostat country codes. Mixed types (as \code{c("Turkey","US","FRA")}) -would not work. See also \code{\link[countrycode:countrycode]{countrycode::countrycode()}}.} - \item{\code{nuts_id}}{Optional. A character vector of NUTS IDs.} - }} +\item{...}{additional arguments to be passed onto \CRANpkg{giscoR} function +\code{gisco_get_nuts()}} } \value{ a sf or data_frame } \description{ Downloads either a simple features (sf) or a data_frame -of NUTS regions. This function is a wrapper of \code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts()}}. -This function requires to have installed the packages \CRANpkg{sf} and -\CRANpkg{giscoR}. +of NUTS regions. This function is a wrapper of \CRANpkg{giscoR} function +\code{gisco_get_nuts()}. +This function requires to have packages \CRANpkg{sf} and +\CRANpkg{giscoR} installed. } \details{ The objects downloaded from GISCO should contain all or some of the @@ -239,6 +227,7 @@ sf <- get_eurostat_geospatial( resolution = "60", nuts_level = "all" ) +\dontrun{ # Downloads dataset from server sf2 <- get_eurostat_geospatial( output_class = "sf", @@ -250,13 +239,14 @@ df <- get_eurostat_geospatial( nuts_level = "0" ) } +} } \seealso{ -\code{\link[giscoR:gisco_get_nuts]{giscoR::gisco_get_nuts()}} +giscoR package and its functions Other geospatial: -\code{\link{eurostat_geodata_60_2016}} +\code{\link{eurostat_geodata_60_2024}} } \author{ Markus Kainu \href{mailto:markuskainu@gmail.com}{markuskainu@gmail.com}, Diego Hernangomez diff --git a/man/get_eurostat_json.Rd b/man/get_eurostat_json.Rd index 83583600..eab9ab93 100755 --- a/man/get_eurostat_json.Rd +++ b/man/get_eurostat_json.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/get_eurostat_json.R \name{get_eurostat_json} \alias{get_eurostat_json} -\title{Get Data from Eurostat API in JSON} +\title{Get Data from Eurostat API Statistics} \usage{ get_eurostat_json( id, @@ -39,25 +39,29 @@ Eurostat order. If \code{FALSE} they are returned as strings.} \item{...}{ Arguments passed on to \code{\link[httr2:req_proxy]{httr2::req_proxy}} \describe{ - \item{\code{req}}{A \link[httr2]{request}.} + \item{\code{req}}{A httr2 \link[httr2]{request} object.} \item{\code{url,port}}{Location of proxy.} \item{\code{username,password}}{Login details for proxy, if needed.} \item{\code{auth}}{Type of HTTP authentication to use. Should be one of the -following: \code{basic}, digest, digest_ie, gssnegotiate, ntlm, any.} +following: \code{basic}, \code{digest}, \code{digest_ie}, \code{gssnegotiate}, \code{ntlm}, \code{any}.} }} } \value{ A dataset as an object of \code{data.frame} class. } \description{ -Retrieve data from Eurostat API in JSON format. +Retrieve data from Eurostat API Statistics in JSON-stat 2.0 format. } \details{ Data to retrieve from -\href{https://ec.europa.eu/eurostat/web/main/data/web-services}{The Eurostat Web Services} +\href{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query}{The Eurostat API Statistics} can be specified with filters. Normally, it is better to use JSON query through \code{\link[=get_eurostat]{get_eurostat()}}, than to use -\code{\link[=get_eurostat_json]{get_eurostat_json()}} directly. +\code{\link[=get_eurostat_json]{get_eurostat_json()}} directly. The main reason for this is that +\code{\link[=get_eurostat_json]{get_eurostat_json()}} returns a relatively raw dataset that does not go +through helper functions in \code{\link[=get_eurostat]{get_eurostat()}}, such as \code{\link[=eurotime2date]{eurotime2date()}} or +\code{\link[=eurotime2num]{eurotime2num()}} functions or reading data from cache and saving data +to cache. Queries are limited to 50 sub-indicators at a time. A time can be filtered with fixed "time" filter or with "sinceTimePeriod" and @@ -79,6 +83,18 @@ the possible interpretation or cause of each error. These messages are returned if the API returns a status indicating a HTTP error (400 or greater). +Additionally, there is limit on the size of the returned extractions +(error code 413). +At the time of publishing this package version, the max authorised size for +the extraction seems to be 5000000 (5 million) rows. The server seems to +estimate the size of the returned data object with a method that is +unknown to us so the number of rows might not be the same as the number +of returned rows in the JSON-stat object. You can limit the number of rows +in your extraction by providing more filters to the query. If the +extraction size continues to go over limit you can try to download the +whole dataset with \code{\link[=get_eurostat]{get_eurostat()}} and filter it locally on your computer, +for example with base R or dplyr data wrangling functions. + The Eurostat implementation seems to be based on SDMX 2.1, which is the reason we've used SDMX Standards guidelines as a supplementary source that we have included in the dataset. What this means in practice is that @@ -88,14 +104,14 @@ in the Eurostat website. We hope you never encounter them. \section{Data source: Eurostat API Statistics (JSON API)}{ Data is downloaded from Eurostat API Statistics. See Eurostat documentation for more information about data queries in API Statistics -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api} This replaces the old JSON Web Services that was used by Eurostat before February 2023 and by the eurostat R package versions before 3.7.13. See Eurostat documentation about the migration from JSON web service to API Statistics for more information about the differences between the old and the new service: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/json} For easily viewing which filtering options are available - in addition to the default ones, time and language - Eurostat Web services Query builder @@ -197,7 +213,7 @@ Example: For more information about data filtering see Eurostat documentation on API Statistics: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/api#APIStatisticsdataquery-TheparametersdefinedintheRESTrequest} } } @@ -257,7 +273,7 @@ datasets downloaded through the SDMX Dissemination API is also supported by Eurostat. We may support this feature in the future. In the meantime, if you are interested in filtering Dissemination API data queries manually, please consult the following Eurostat documentation: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+filtering} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Filteringonseries-keys} } \examples{ @@ -293,41 +309,24 @@ dd <- get_eurostat("nama_10_gdp", \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is - - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} - - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +In addition, please provide a citation to the specific software version +used: + + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . + +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/get_eurostat_link.Rd b/man/get_eurostat_link.Rd new file mode 100644 index 00000000..6e39959b --- /dev/null +++ b/man/get_eurostat_link.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_eurostat_link.R +\name{get_eurostat_link} +\alias{get_eurostat_link} +\title{Download Eurostat Data from API Link (robust, no list-columns, async-aware)} +\usage{ +get_eurostat_link(link, destfile = NULL, verbose = TRUE) +} +\arguments{ +\item{link}{Eurostat "Copy API link" URL or Data Browser download link} + +\item{destfile}{Optional file path for saving raw files (only for Excel)} + +\item{verbose}{Output messages when downloading data. Default is \code{TRUE}.} +} +\value{ +Tibble for all supported formats +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} + +Supports TSV, SDMX-CSV, SDMX-ML, JSON-stat, and Spreadsheet (.xlsx) formats. +Always returns a tibble where possible. +} diff --git a/man/get_eurostat_local.Rd b/man/get_eurostat_local.Rd new file mode 100644 index 00000000..ea3c0be9 --- /dev/null +++ b/man/get_eurostat_local.Rd @@ -0,0 +1,73 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_eurostat_local.R +\name{get_eurostat_local} +\alias{get_eurostat_local} +\title{Read Local SDMX-CSV files} +\usage{ +get_eurostat_local( + file, + time_format = "date", + type = "code", + lang = "en", + select_time = NULL, + stringsAsFactors = FALSE, + keepFlags = FALSE, + legacy.data.output = FALSE +) +} +\arguments{ +\item{file}{file path as string} + +\item{time_format}{a string giving a type of the conversion of the time column from the +eurostat format. The default argument "\code{date}" converts to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class +with the date being the first day of the period. A "\code{date_last}" argument +converts the dataset date to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class object with the difference +that the exact date is the last date of the period. Period can be year, +semester (half year), quarter, month, or week (See \code{\link[=eurotime2date]{eurotime2date()}} for +more information). +Argument "\code{num}" converts the date into a numeric (integer) meaning that +the first day of the year 2000 is close to 2000.01 and the last day of the +year is close to 2000.99 (see \code{\link[=eurotime2num]{eurotime2num()}} for more information). +Using the argument "\code{raw}" preserves the dates as they were in the original +Eurostat data.} + +\item{type}{A type of variables, "\code{code}" (default), "\code{label}" or "\code{both}". +The parameter "\code{both}" will return a data_frame with named vectors, +labels as values and codes as names.} + +\item{lang}{2-letter language code, default is "\code{en}" (English), other +options are "\code{fr}" (French) and "\code{de}" (German). Used for labeling datasets.} + +\item{select_time}{a character symbol for a time frequency or \code{NULL}, +which is used by default as most datasets have just one time +frequency. For datasets with multiple time +frequencies, select one or more of the desired frequencies with: +"Y" (or "A") = annual, "S" = semi-annual / semester, "Q" = quarterly, +"M" = monthly, "W" = weekly. For all frequencies in same data +frame \code{time_format = "raw"} should be used.} + +\item{stringsAsFactors}{if \code{TRUE} (the default) variables are converted to factors in the original +Eurostat order. If \code{FALSE} they are returned as strings.} + +\item{keepFlags}{a logical whether the flags (e.g. "confidential", +"provisional") should be kept in a separate column or if they +can be removed. Default is \code{FALSE}. For flag values see: +\url{https://ec.europa.eu/eurostat/data/database/information}. +Also possible non-real zero "0n" is indicated in flags column. +Flags are not available for eurostat API, so \code{keepFlags} +can not be used with a \code{filters}.} + +\item{legacy.data.output}{Use legacy column names and data object structure. +Default is FALSE. If TRUE, the object will try to emulate the naming +conventions of eurostat package version 3.7.x and earlier.} +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +Read compressed or uncompressed SDMX-CSV files +} +\details{ +This implementation is experimental. It uses only data.table methods to read +and wrangle data files. + +Download datasets in sdmx-csv format from \url{https://ec.europa.eu/eurostat/databrowser/bulk?lang=en} +} diff --git a/man/get_eurostat_raw.Rd b/man/get_eurostat_raw.Rd index d3aca865..6f50b79a 100755 --- a/man/get_eurostat_raw.Rd +++ b/man/get_eurostat_raw.Rd @@ -29,7 +29,7 @@ dissemination API. Data is downloaded from Eurostat SDMX 2.1 API endpoint as compressed TSV files that are transformed into tabular format. See Eurostat documentation for more information: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+query} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Dataquery} The new dissemination API replaces the old bulk download facility that was used by Eurostat before October 2023 and by the eurostat R package versions @@ -37,7 +37,7 @@ before 4.0.0. See Eurostat documentation about the transition from Bulk Download to API for more information about the differences between the old bulk download facility and the data provided by the new API connection: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload} See especially the document Migrating_to_API_TSV.pdf that describes the changes in TSV file format in new applications. @@ -103,11 +103,11 @@ datasets downloaded through the SDMX Dissemination API is also supported by Eurostat. We may support this feature in the future. In the meantime, if you are interested in filtering Dissemination API data queries manually, please consult the following Eurostat documentation: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+data+filtering} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Filteringonseries-keys} } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ eurostat:::get_eurostat_raw("educ_iste") } @@ -116,41 +116,24 @@ eurostat:::get_eurostat_raw("educ_iste") \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{# Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{# Kindly cite this package by citing the following R Journal article: # # Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and # analysis of Eurostat open data with the eurostat package. The R # Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 # -# A BibTeX entry for LaTeX users is +# In addition, please provide a citation to the specific software version +# used: # -# @Article\{10.32614/RJ-2017-019, -# title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, -# journal = \{The R Journal\}, -# volume = \{9\}, -# number = \{1\}, -# pages = \{385--392\}, -# year = \{2017\}, -# doi = \{10.32614/RJ-2017-019\}, -# url = \{https://doi.org/10.32614/RJ-2017-019\}, -# \} +# Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, +# Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." +# doi:10.32614/CRAN.package.eurostat +# , R package version +# 4.1.0.9007, . # -# Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., -# and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data -# [Computer software]. R package version 4.0.0. -# https://github.com/rOpenGov/eurostat -# -# A BibTeX entry for LaTeX users is -# -# @Misc\{eurostat, -# title = \{eurostat: Tools for Eurostat Open Data\}, -# author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, -# url = \{https://github.com/rOpenGov/eurostat\}, -# type = \{Computer software\}, -# year = \{2023\}, -# note = \{R package version 4.0.0\}, -# \} +# To see these entries in BibTeX format, use 'print(, +# bibtex=TRUE)', 'toBibtex(.)', or set +# 'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} } \seealso{ diff --git a/man/get_eurostat_sdmx.Rd b/man/get_eurostat_sdmx.Rd new file mode 100644 index 00000000..012a1e1d --- /dev/null +++ b/man/get_eurostat_sdmx.Rd @@ -0,0 +1,91 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_eurostat_sdmx.R +\name{get_eurostat_sdmx} +\alias{get_eurostat_sdmx} +\title{Get Eurostat Data from SDMX 2.1 API} +\usage{ +get_eurostat_sdmx( + id, + time_format = "date", + filters = NULL, + type = "code", + lang = "en", + use.data.table = FALSE, + agency = "Eurostat", + compressed = TRUE, + keepFlags = FALSE, + legacy.data.output = FALSE, + wait = 10, + max_wait = 600, + verbose = TRUE +) +} +\arguments{ +\item{id}{A unique identifier / code for the dataset of interest. If code is not +known \code{\link[=search_eurostat]{search_eurostat()}} function can be used to search Eurostat table +of contents.} + +\item{time_format}{a string giving a type of the conversion of the time column from the +eurostat format. The default argument "\code{date}" converts to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class +with the date being the first day of the period. A "\code{date_last}" argument +converts the dataset date to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class object with the difference +that the exact date is the last date of the period. Period can be year, +semester (half year), quarter, month, or week (See \code{\link[=eurotime2date]{eurotime2date()}} for +more information). +Argument "\code{num}" converts the date into a numeric (integer) meaning that +the first day of the year 2000 is close to 2000.01 and the last day of the +year is close to 2000.99 (see \code{\link[=eurotime2num]{eurotime2num()}} for more information). +Using the argument "\code{raw}" preserves the dates as they were in the original +Eurostat data.} + +\item{filters}{A named list of filters. Names of list objects are Eurostat +variable codes and values are vectors of observation codes. If \code{NULL} +(default) the whole dataset is returned. See details for more information +on filters and limitations per query.} + +\item{type}{A type of variables, "\code{code}" (default), "\code{label}" or "\code{both}". +The parameter "\code{both}" will return a data_frame with named vectors, +labels as values and codes as names.} + +\item{lang}{2-letter language code, default is "\code{en}" (English), other +options are "\code{fr}" (French) and "\code{de}" (German). Used for labeling datasets.} + +\item{use.data.table}{Use data.table to process files? Default is FALSE. +If data.table is used, data will be downloaded as a TSV file and +processed using \code{\link[=tidy_eurostat]{tidy_eurostat()}}} + +\item{agency}{Either "Eurostat" (default), "Eurostat_comext" +(for Comext and Prodcom datasets), "COMP", "EMPL" or "GROW"} + +\item{compressed}{Logical. Download data in compressed format? Default is TRUE.} + +\item{keepFlags}{a logical whether the flags (e.g. "confidential", +"provisional") should be kept in a separate column or if they +can be removed. Default is \code{FALSE}. For flag values see: +\url{https://ec.europa.eu/eurostat/data/database/information}. +Also possible non-real zero "0n" is indicated in flags column. +Flags are not available for eurostat API, so \code{keepFlags} +can not be used with a \code{filters}.} + +\item{legacy.data.output}{Use legacy column names and data object structure. +Default is FALSE. If TRUE, the object will try to emulate the naming +conventions of eurostat package version 3.7.x and earlier.} + +\item{wait}{Integer. Seconds between status checks. Default is 1 second.} + +\item{max_wait}{Integer. Max time to wait in seconds. Default is 60 seconds.} + +\item{verbose}{Output messages when downloading data. Default is \code{TRUE}.} +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} + +Download data sets from Eurostat using the same logic as \code{get_eurostat()} +function. +} +\details{ +This function is experimental because while it works as intended and is +useful in the same way as other get_ functions in the package, we would +like to test it for a while and listen to user feedback before deciding on +what is the best way to interact with SDMX APIs. +} diff --git a/man/get_eurostat_toc.Rd b/man/get_eurostat_toc.Rd index 1efe3545..d052534d 100755 --- a/man/get_eurostat_toc.Rd +++ b/man/get_eurostat_toc.Rd @@ -49,11 +49,11 @@ The Eurostat Table of Contents (TOC) is downloaded from \url{https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang=de} See Eurostat documentation on TOC items: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+-+Detailed+guidelines+-+Catalogue+API+-+TOC} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/catalogue-api/toc} } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ tmp <- get_eurostat_toc() head(tmp) @@ -83,41 +83,24 @@ tmp$data.end <- lubridate::as_date( \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is +In addition, please provide a citation to the specific software version +used: - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/harmonize_country_code.Rd b/man/harmonize_country_code.Rd index 27e58eb5..bea8fee8 100755 --- a/man/harmonize_country_code.Rd +++ b/man/harmonize_country_code.Rd @@ -20,7 +20,7 @@ United Kingdom. This function turns country codes into to ISO 3166-1 alpha-2. } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ lp <- get_eurostat("nama_10_lp_ulc") lp$geo <- harmonize_country_code(lp$geo) diff --git a/man/search_eurostat.Rd b/man/search_eurostat.Rd index 2f96125b..e4270cdb 100755 --- a/man/search_eurostat.Rd +++ b/man/search_eurostat.Rd @@ -74,11 +74,11 @@ The Eurostat Table of Contents (TOC) is downloaded from \url{https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang=de} See Eurostat documentation on TOC items: -\url{https://wikis.ec.europa.eu/display/EUROSTATHELP/API+-+Detailed+guidelines+-+Catalogue+API+-+TOC} +\url{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/catalogue-api/toc} } \examples{ -\dontshow{if (check_access_to_data()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontshow{if (check_access_to_data()) withAutoprint(\{ # examplesIf} \donttest{ tmp <- search_eurostat("education") head(tmp) @@ -91,41 +91,24 @@ tmp <- search_eurostat("Live births (total) by NUTS 3 region", fixed = TRUE) \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is +In addition, please provide a citation to the specific software version +used: - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/tgs00026.Rd b/man/tgs00026.Rd index 548e1dfc..8dfe183b 100755 --- a/man/tgs00026.Rd +++ b/man/tgs00026.Rd @@ -21,7 +21,7 @@ Data retrieval date: 2022-06-27 \seealso{ Other datasets: \code{\link{eu_countries}}, -\code{\link{eurostat_geodata_60_2016}} +\code{\link{eurostat_geodata_60_2024}} } \concept{datasets} \keyword{datasets} diff --git a/man/tidy_eurostat.Rd b/man/tidy_eurostat.Rd index 55156ae6..308c904b 100755 --- a/man/tidy_eurostat.Rd +++ b/man/tidy_eurostat.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/tidy_eurostat.R \name{tidy_eurostat} \alias{tidy_eurostat} -\title{Transform Data into Row-Column-Value Format} +\title{Transform TSV files into Row-Column-Value Format} \usage{ tidy_eurostat( dat, @@ -17,9 +17,9 @@ tidy_eurostat( \item{dat}{a data_frame from \code{\link[=get_eurostat_raw]{get_eurostat_raw()}}.} \item{time_format}{a string giving a type of the conversion of the time column from the -eurostat format. The default argument "\code{date}" converts to a \code{\link[=Date]{Date()}} class +eurostat format. The default argument "\code{date}" converts to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class with the date being the first day of the period. A "\code{date_last}" argument -converts the dataset date to a \code{\link[=Date]{Date()}} class object with the difference +converts the dataset date to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class object with the difference that the exact date is the last date of the period. Period can be year, semester (half year), quarter, month, or week (See \code{\link[=eurotime2date]{eurotime2date()}} for more information). @@ -55,7 +55,7 @@ On Windows requires that RTools is installed.} tibble in the melted format with the last column 'values'. } \description{ -Transform raw Eurostat data table downloaded from the +Transform raw Eurostat TSV files downloaded from the API into a tidy row-column-value format (RCV). } \examples{ @@ -71,41 +71,24 @@ get_eurostat("AVIA_GOR_ME", \references{ See \code{citation("eurostat")}: -\if{html}{\out{
}}\preformatted{Kindly cite the eurostat R package as follows: +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and analysis of Eurostat open data with the eurostat package. The R Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 -A BibTeX entry for LaTeX users is +In addition, please provide a citation to the specific software version +used: - @Article\{10.32614/RJ-2017-019, - title = \{Retrieval and Analysis of Eurostat Open Data with the eurostat Package\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek\}, - journal = \{The R Journal\}, - volume = \{9\}, - number = \{1\}, - pages = \{385--392\}, - year = \{2017\}, - doi = \{10.32614/RJ-2017-019\}, - url = \{https://doi.org/10.32614/RJ-2017-019\}, - \} + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . - Lahti, L., Huovari J., Kainu M., Biecek P., Hernangomez D., Antal D., - and Kantanen P. (2023). eurostat: Tools for Eurostat Open Data - [Computer software]. R package version 4.0.0. - https://github.com/rOpenGov/eurostat - -A BibTeX entry for LaTeX users is - - @Misc\{eurostat, - title = \{eurostat: Tools for Eurostat Open Data\}, - author = \{Leo Lahti and Janne Huovari and Markus Kainu and Przemyslaw Biecek and Diego Hernangomez and Daniel Antal and Pyry Kantanen\}, - url = \{https://github.com/rOpenGov/eurostat\}, - type = \{Computer software\}, - year = \{2023\}, - note = \{R package version 4.0.0\}, - \} +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. }\if{html}{\out{
}} When citing data downloaded from Eurostat, see section "Citing Eurostat data" diff --git a/man/tidy_eurostat_sdmx_csv.Rd b/man/tidy_eurostat_sdmx_csv.Rd new file mode 100644 index 00000000..3a99abc1 --- /dev/null +++ b/man/tidy_eurostat_sdmx_csv.Rd @@ -0,0 +1,92 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tidy_eurostat.R +\name{tidy_eurostat_sdmx_csv} +\alias{tidy_eurostat_sdmx_csv} +\title{Transform CSV files into Row-Column-Value Format data.table object} +\usage{ +tidy_eurostat_sdmx_csv( + dat, + time_format = "date", + select_time = NULL, + keepFlags = FALSE +) +} +\arguments{ +\item{dat}{a data_frame from \code{\link[=get_eurostat_raw]{get_eurostat_raw()}}.} + +\item{time_format}{a string giving a type of the conversion of the time column from the +eurostat format. The default argument "\code{date}" converts to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class +with the date being the first day of the period. A "\code{date_last}" argument +converts the dataset date to a \code{\link[lubridate:date_utils]{lubridate::Date()}} class object with the difference +that the exact date is the last date of the period. Period can be year, +semester (half year), quarter, month, or week (See \code{\link[=eurotime2date]{eurotime2date()}} for +more information). +Argument "\code{num}" converts the date into a numeric (integer) meaning that +the first day of the year 2000 is close to 2000.01 and the last day of the +year is close to 2000.99 (see \code{\link[=eurotime2num]{eurotime2num()}} for more information). +Using the argument "\code{raw}" preserves the dates as they were in the original +Eurostat data.} + +\item{select_time}{a character symbol for a time frequency or \code{NULL}, +which is used by default as most datasets have just one time +frequency. For datasets with multiple time +frequencies, select one or more of the desired frequencies with: +"Y" (or "A") = annual, "S" = semi-annual / semester, "Q" = quarterly, +"M" = monthly, "W" = weekly. For all frequencies in same data +frame \code{time_format = "raw"} should be used.} + +\item{keepFlags}{a logical whether the flags (e.g. "confidential", +"provisional") should be kept in a separate column or if they +can be removed. Default is \code{FALSE}. For flag values see: +\url{https://ec.europa.eu/eurostat/data/database/information}. +Also possible non-real zero "0n" is indicated in flags column. +Flags are not available for eurostat API, so \code{keepFlags} +can not be used with a \code{filters}.} +} +\value{ +data.table object in the melted format with the last column 'values' or 'OBS_VALUE'. +} +\description{ +Transform raw Eurostat TSV files downloaded from the +API into a tidy row-column-value format (RCV). +} +\details{ +Can read only SDMX-CSV files, in compressed (.csv.gz) and uncompressed (.csv) +format. As opposed to regular tidy_eurostat function, stringsAsFactors is +determined outside this function, in \code{get_eurostat_local()}. +} +\references{ +See \code{citation("eurostat")}: + +\if{html}{\out{
}}\preformatted{Kindly cite this package by citing the following R Journal article: + + Lahti L., Huovari J., Kainu M., and Biecek P. (2017). Retrieval and + analysis of Eurostat open data with the eurostat package. The R + Journal 9(1), pp. 385-392. doi: 10.32614/RJ-2017-019 + +In addition, please provide a citation to the specific software version +used: + + Lahti L, Huovari J, Kainu M, Biecek P, Hernangomez D, Antal D, + Kantanen P (2026). "eurostat: Tools for Eurostat Open Data." + doi:10.32614/CRAN.package.eurostat + , R package version + 4.1.0.9007, . + +To see these entries in BibTeX format, use 'print(, +bibtex=TRUE)', 'toBibtex(.)', or set +'options(citation.bibtex.max=999)'. +}\if{html}{\out{
}} + +When citing data downloaded from Eurostat, see section "Citing Eurostat data" +in \code{\link[=get_eurostat]{get_eurostat()}} documentation. +} +\seealso{ +\code{\link[=get_eurostat_local]{get_eurostat_local()}} +} +\author{ +Pyry Kantanen +} +\keyword{database} +\keyword{internal} +\keyword{utilities} diff --git a/tests/testthat/_snaps/03_get_eurostat_geospatial.md b/tests/testthat/_snaps/03_get_eurostat_geospatial.md index 93616591..bcaf231a 100644 --- a/tests/testthat/_snaps/03_get_eurostat_geospatial.md +++ b/tests/testthat/_snaps/03_get_eurostat_geospatial.md @@ -18,5 +18,4 @@ a <- get_eurostat_geospatial(nuts_id = "LU", nuts_level = 0) Message Extracting data using giscoR package, please report issues on https://github.com/rOpenGov/giscoR/issues - Cache management as per giscoR. see 'giscoR::gisco_get_nuts()' diff --git a/tests/testthat/test_01_bibliography.R b/tests/testthat/test_01_bibliography.R index 32374ba3..1327c320 100755 --- a/tests/testthat/test_01_bibliography.R +++ b/tests/testthat/test_01_bibliography.R @@ -1,18 +1,7 @@ test_that("Bibentry gives correct results", { skip_on_cran() skip_if_offline() - expect_equal( - class(get_bibentry( - code = c("sts_inpr_a", "nama_10_gdp"), - keywords = list( - c("production", "industry"), - c("GDP") - ), - format = "Biblatex" - )), - "Bibtex" - ) - + expect_equal( class( suppressWarnings( @@ -23,11 +12,11 @@ test_that("Bibentry gives correct results", { ))), "Bibtex" ) - + expect_error( get_bibentry(code = 123456) ) - + expect_error( get_bibentry( code = c("sts_inpr_a"), @@ -35,21 +24,26 @@ test_that("Bibentry gives correct results", { keywords = "production" ) ) - + + # First cache to get rid of possible vroom problem + cache <- suppressWarnings(get_bibentry( + code = "sts_inpr_b" + )) + # None of the codes not found expect_warning( get_bibentry( code = "sts_inpr_b" - ) + ), "None of the codes were found in the Eurostat table of contents" ) - + # Some of the codes not found expect_warning( get_bibentry( code = c("sts_inpr_b", "sts_inpr_a") ) ) - + expect_warning( get_bibentry( code = c("sts_inpr_a", "nama_10_gdp"), @@ -60,6 +54,6 @@ test_that("Bibentry gives correct results", { format = "character" ) ) - - + + }) diff --git a/tests/testthat/test_03_get_eurostat_geospatial.R b/tests/testthat/test_03_get_eurostat_geospatial.R index 7ea873d3..9a92370b 100755 --- a/tests/testthat/test_03_get_eurostat_geospatial.R +++ b/tests/testthat/test_03_get_eurostat_geospatial.R @@ -6,32 +6,26 @@ test_that("get_eurostat_geospatial errors", { skip_if(!giscoR::gisco_check_access(), "No access to GISCO") # Testing argument 'output_class' - expect_error(get_eurostat_geospatial(output_class = 0)) - expect_error(get_eurostat_geospatial(output_class = "foo")) - expect_error(get_eurostat_geospatial(output_class = "sf", "df")) + expect_error(get_eurostat_geospatial(output_class = 0, verbose = FALSE)) + expect_error(get_eurostat_geospatial(output_class = "foo", verbose = FALSE)) # Testing argument 'resolution' - expect_error(get_eurostat_geospatial(resolution = 12345)) - expect_error(get_eurostat_geospatial(resolution = 1:2)) + expect_error(get_eurostat_geospatial(resolution = 12345, verbose = FALSE)) + expect_error(get_eurostat_geospatial(resolution = 1:2, verbose = FALSE)) # Testing argument nuts_level - expect_error(get_eurostat_geospatial(nuts_level = 12345)) - expect_error(get_eurostat_geospatial(nuts_level = 1:2)) + expect_error(get_eurostat_geospatial(nuts_level = 1:2, verbose = FALSE)) # Testing argument year - expect_error(get_eurostat_geospatial(year = 1900)) - expect_error(get_eurostat_geospatial(year = c(2003, 2006))) - - # Testing argment cache - expect_error(get_eurostat_geospatial(cache = as.logical(NA), year = 2021)) - expect_error(get_eurostat_geospatial(cache = c(TRUE, FALSE), year = 2021)) + expect_error(get_eurostat_geospatial(year = 1900, verbose = FALSE)) + expect_error(get_eurostat_geospatial(year = c(2003, 2006, verbose = FALSE))) # Testing argument CRS expect_error(get_eurostat_geospatial(crs = "north polar stereographic")) - expect_error(get_eurostat_geospatial(crs = c(4326, 3035))) + expect_warning(get_eurostat_geospatial(crs = c(4326, 3035), verbose = FALSE)) # Invalid combinations - expect_error(get_eurostat_geospatial(resolution = 60, year = 2003)) + expect_error(get_eurostat_geospatial(resolution = 60, year = 2003, verbose = FALSE)) }) test_that("get_eurostat_geospatial messages", { @@ -41,13 +35,6 @@ test_that("get_eurostat_geospatial messages", { skip_if_offline() skip_if(!giscoR::gisco_check_access(), "No access to GISCO") - - # Deprecations - expect_message( - get_eurostat_geospatial(make_valid = TRUE), - "'make_valid' argument has been deprecated" - ) - expect_message( spdf <- get_eurostat_geospatial(output_class = "spdf"), "'spdf' output deprecated. Switching to sf output" @@ -61,13 +48,14 @@ test_that("get_eurostat_geospatial messages", { test_that("get_eurostat_geospatial nuts levels", { + skip_if_not_installed(pkg = "giscoR") skip_if_not_installed(pkg = "sf") # From internal data with default args - expect_message(all <- get_eurostat_geospatial(nuts_level = "all"), "eurostat") - expect_message(n0 <- get_eurostat_geospatial(nuts_level = "0"), "eurostat") - expect_message(n1 <- get_eurostat_geospatial(nuts_level = "1"), "eurostat") - expect_message(n2 <- get_eurostat_geospatial(nuts_level = "2"), "eurostat") - expect_message(n3 <- get_eurostat_geospatial(nuts_level = "3"), "eurostat") + expect_message(all <- get_eurostat_geospatial(nuts_level = "all", verbose = TRUE), "eurostat") + expect_message(n0 <- get_eurostat_geospatial(nuts_level = "0", verbose = TRUE), "eurostat") + expect_message(n1 <- get_eurostat_geospatial(nuts_level = "1", verbose = TRUE), "eurostat") + expect_message(n2 <- get_eurostat_geospatial(nuts_level = "2", verbose = TRUE), "eurostat") + expect_message(n3 <- get_eurostat_geospatial(nuts_level = "3", verbose = TRUE), "eurostat") expect_gt(nrow(all), nrow(n3)) expect_gt(nrow(n3), nrow(n2)) @@ -120,6 +108,7 @@ test_that("get_eurostat_geospatial nuts levels", { }) test_that("get_eurostat_geospatial df", { + skip_if_not_installed(pkg = "giscoR") skip_if_not_installed(pkg = "sf") # From internal data with default args expect_message( @@ -167,7 +156,7 @@ test_that("get_eurostat_geospatial df", { nuts_level = "all", output_class = "df", verbose = TRUE - ), "giscoR") + ), "Extracting data from eurostat::eurostat_geodata_60_2024") expect_message(gn0 <- get_eurostat_geospatial( nuts_level = "0", crs = 3035, @@ -259,28 +248,8 @@ test_that("get_eurostat_geospatial cache_dir", { }) -test_that("giscoR returns NULL", { - skip_if_not_installed(pkg = "sf") - skip_if_not_installed(pkg = "giscoR") - skip_on_cran() - skip_if_offline() - skip_if(!giscoR::gisco_check_access(), "No access to GISCO") - skip_if(packageVersion("giscoR") < "0.3.5", "Use latest giscoR release") - - options(giscoR_test_offline = TRUE) - expect_message( - n <- get_eurostat_geospatial( - country = "AT", nuts_level = "0", - update_cache = TRUE - ), - "not reachable" - ) - expect_null(n) - options(giscoR_test_offline = FALSE) -}) - - test_that("Check column names", { + skip_if_not_installed(pkg = "giscoR") skip_if_not_installed(pkg = "sf") # See https://github.com/rOpenGov/eurostat/issues/240 @@ -291,12 +260,12 @@ test_that("Check column names", { ) - cached <- get_eurostat_geospatial() + cached <- get_eurostat_geospatial(verbose = FALSE) expect_s3_class(cached, "sf") expect_identical(names(cached), col_order) # df - cached_df <- get_eurostat_geospatial(output_class = "df") + cached_df <- get_eurostat_geospatial(output_class = "df", verbose = FALSE) expect_s3_class(cached_df, "data.frame") expect_identical(names(cached_df), col_order[-length(col_order)]) }) @@ -308,7 +277,7 @@ test_that("Check column names POLYGONS from GISCO", { skip_on_cran() skip_if_offline() skip_if(!giscoR::gisco_check_access(), "No access to GISCO") - skip_if(packageVersion("giscoR") < "0.3.5", "Use latest giscoR release") + skip_if(packageVersion("giscoR") < "1.1.0", "Use latest giscoR release") col_order <- c( "id", "LEVL_CODE", "NUTS_ID", "CNTR_CODE", "NAME_LATN", @@ -324,21 +293,24 @@ test_that("Check column names POLYGONS from GISCO", { # df poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, - resolution = 20, year = 2003 + resolution = 20, year = 2003, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") expect_identical(names(poly_df), col_order[-length(col_order)]) # Polygons 2006 - poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2006) + poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2006, + verbose = FALSE) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) # df poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, - resolution = 60, year = 2006 + resolution = 60, year = 2006, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -346,28 +318,32 @@ test_that("Check column names POLYGONS from GISCO", { # Polygons 2010 - poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2010) + poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2010, + verbose = FALSE) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) # df poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, - resolution = 60, year = 2010 + resolution = 60, year = 2010, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") expect_identical(names(poly_df), col_order[-length(col_order)]) # Polygons 2013 - poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2013) + poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2013, + verbose = FALSE) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) # df poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, - resolution = 60, year = 2013 + resolution = 60, year = 2013, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -376,7 +352,8 @@ test_that("Check column names POLYGONS from GISCO", { # Polygons 2016 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2016, - update_cache = TRUE + update_cache = TRUE, + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -385,21 +362,40 @@ test_that("Check column names POLYGONS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2016, - update_cache = TRUE + update_cache = TRUE, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") expect_identical(names(poly_df), col_order[-length(col_order)]) # Polygons 2021 - poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2021) + poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2021, + verbose = FALSE) + expect_s3_class(poly, "sf") + expect_identical(names(poly), col_order) + + # df + poly_df <- get_eurostat_geospatial( + output_class = "df", nuts_level = 0, + resolution = 60, year = 2021, + verbose = FALSE + ) + + expect_s3_class(poly_df, "data.frame") + expect_identical(names(poly_df), col_order[-length(col_order)]) + + # Polygons 2024 + poly <- get_eurostat_geospatial(nuts_level = 0, resolution = 60, year = 2024, + verbose = FALSE) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) # df poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, - resolution = 60, year = 2021 + resolution = 60, year = 2024, + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -423,7 +419,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2003 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 20, year = 2003, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -432,7 +429,8 @@ test_that("Check column names LABELS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 20, year = 2003, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -441,7 +439,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2006 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2006, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -450,7 +449,8 @@ test_that("Check column names LABELS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2006, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -460,7 +460,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2010 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2010, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -469,7 +470,8 @@ test_that("Check column names LABELS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2010, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -478,7 +480,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2013 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2013, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -487,7 +490,8 @@ test_that("Check column names LABELS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2013, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -496,7 +500,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2016 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2016, - update_cache = TRUE, spatialtype = "LB" + update_cache = TRUE, spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -506,7 +511,8 @@ test_that("Check column names LABELS from GISCO", { output_class = "df", nuts_level = 0, resolution = 60, year = 2016, update_cache = TRUE, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -515,7 +521,8 @@ test_that("Check column names LABELS from GISCO", { # Labels 2021 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2021, - spatialtype = "LB" + spatialtype = "LB", + verbose = FALSE ) expect_s3_class(poly, "sf") expect_identical(names(poly), col_order) @@ -529,6 +536,31 @@ test_that("Check column names LABELS from GISCO", { expect_s3_class(poly_df, "data.frame") expect_identical(names(poly_df), col_order[-length(col_order)]) + + names_2024 <- c("id", "LEVL_CODE", "NUTS_ID", "CNTR_CODE", "NAME_LATN", + "NUTS_NAME", "MOUNT_TYPE", "URBN_TYPE", "COAST_TYPE", + "FID", "geo", "LAT", "LON", "NAME_ENGL", "NAME_FREN", + "ISO3_CODE", "SVRG_UN", "CAPT", "EU_STAT", "EFTA_STAT", + "CC_STAT", "NAME_GERM", "geometry") + + # Labels 2024 + poly <- get_eurostat_geospatial( + nuts_level = 0, resolution = 60, year = 2024, + spatialtype = "LB", + verbose = FALSE + ) + expect_s3_class(poly, "sf") + expect_identical(names(poly), names_2024) + + # df + poly_df <- get_eurostat_geospatial( + output_class = "df", nuts_level = 0, + resolution = 60, year = 2024, + spatialtype = "LB" + ) + + expect_s3_class(poly_df, "data.frame") + expect_identical(names(poly_df), names_2024[-length(names_2024)]) }) @@ -538,12 +570,13 @@ test_that("Check column names BORDERS from GISCO", { skip_on_cran() skip_if_offline() skip_if(!giscoR::gisco_check_access(), "No access to GISCO") - skip_if(packageVersion("giscoR") < "0.3.5", "Use latest giscoR release") + skip_if(packageVersion("giscoR") < "1.1.0", "Use latest giscoR release") # BORDERS 2003 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 20, year = 2003, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -551,7 +584,8 @@ test_that("Check column names BORDERS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 20, year = 2003, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -559,7 +593,8 @@ test_that("Check column names BORDERS from GISCO", { # BORDERS 2006 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2006, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -567,7 +602,8 @@ test_that("Check column names BORDERS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2006, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -576,7 +612,8 @@ test_that("Check column names BORDERS from GISCO", { # BORDERS 2010 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2010, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -584,7 +621,8 @@ test_that("Check column names BORDERS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2010, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -592,7 +630,8 @@ test_that("Check column names BORDERS from GISCO", { # BORDERS 2013 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2013, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -600,7 +639,8 @@ test_that("Check column names BORDERS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2013, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -608,7 +648,8 @@ test_that("Check column names BORDERS from GISCO", { # BORDERS 2016 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2016, - update_cache = TRUE, spatialtype = "BN" + update_cache = TRUE, spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -617,7 +658,8 @@ test_that("Check column names BORDERS from GISCO", { output_class = "df", nuts_level = 0, resolution = 60, year = 2016, update_cache = TRUE, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") @@ -625,7 +667,8 @@ test_that("Check column names BORDERS from GISCO", { # BORDERS 2021 poly <- get_eurostat_geospatial( nuts_level = 0, resolution = 60, year = 2021, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly, "sf") @@ -633,7 +676,26 @@ test_that("Check column names BORDERS from GISCO", { poly_df <- get_eurostat_geospatial( output_class = "df", nuts_level = 0, resolution = 60, year = 2021, - spatialtype = "BN" + spatialtype = "BN", + verbose = FALSE + ) + + expect_s3_class(poly_df, "data.frame") + + # BORDERS 2024 + poly <- get_eurostat_geospatial( + nuts_level = 0, resolution = 60, year = 2024, + spatialtype = "BN", + verbose = FALSE + ) + expect_s3_class(poly, "sf") + + # df + poly_df <- get_eurostat_geospatial( + output_class = "df", nuts_level = 0, + resolution = 60, year = 2024, + spatialtype = "BN", + verbose = FALSE ) expect_s3_class(poly_df, "data.frame") diff --git a/tests/testthat/test_04_get.R b/tests/testthat/test_04_get.R index dc525b7d..679e324c 100755 --- a/tests/testthat/test_04_get.R +++ b/tests/testthat/test_04_get.R @@ -21,14 +21,14 @@ test_that("get_eurostat (dissemination API) includes TIME_PERIOD and value", { test_that("get_eurostat (dissemination API) produces a message with multiple select_time", { skip_on_cran() skip_if_offline() - expect_message(get_eurostat(id = "avia_gonc", + expect_message(get_eurostat(id = "avia_gooc", select_time = c("A", "M", "Q"))) }) test_that("get_eurostat (dissemination API) produces an error with imaginary select_time parameters", { skip_on_cran() skip_if_offline() - expect_error(get_eurostat(id = "avia_gonc", + expect_error(get_eurostat(id = "avia_gooc", select_time = c("X", "Y", "Z"))) # sleep for a while to let the API rest Sys.sleep(5) @@ -37,11 +37,11 @@ test_that("get_eurostat (dissemination API) produces an error with imaginary sel test_that("get_eurostat (dissemination API) works correctly with multi-frequency", { skip_on_cran() skip_if_offline() - expect_message(get_eurostat("avia_gonc", + expect_message(get_eurostat("avia_gooc", cache = FALSE, time_format = "date_last") ) - expect_match(as.character(unique(get_eurostat("avia_gonc", + expect_match(as.character(unique(get_eurostat("avia_gooc", select_time = NULL, cache = FALSE)$TIME_PERIOD)), "-01-01") diff --git a/tests/testthat/test_06_label.R b/tests/testthat/test_06_label.R index dead5bcd..9619f137 100755 --- a/tests/testthat/test_06_label.R +++ b/tests/testthat/test_06_label.R @@ -2,7 +2,7 @@ test_that("Variable names are labeled", { skip_on_cran() skip_if_offline() expect_equal(label_eurostat_vars(id = "nama_10_lp_ulc", x = "geo", lang = "en"), "Geopolitical entity (reporting)") - expect_equal(label_eurostat_tables("nama_10_lp_ulc"), "Labour productivity and unit labour costs") + expect_equal(label_eurostat_tables("nama_10_lp_ulc"), "Labour productivity and unit labour costs - annual data") expect_true(any(grepl( "_code", @@ -38,7 +38,7 @@ test_that("Countrycodes are labelled for factors", { )), c("Finland", "Germany", "EU28") ) - + # sleep for a while to let the API rest Sys.sleep(5) }) @@ -84,4 +84,4 @@ test_that("wrong code (for this dataset) produces an error", { get_eurostat("road_eqr_trams"), code = "coef" ) ) -}) \ No newline at end of file +}) diff --git a/tests/testthat/test_07_misc.R b/tests/testthat/test_07_misc.R index 9e638371..8eac5b01 100755 --- a/tests/testthat/test_07_misc.R +++ b/tests/testthat/test_07_misc.R @@ -28,24 +28,6 @@ test_that("Duplicated with fix_duplicated does not give an error", { ), "P5", all = TRUE) }) -# W99 values are now reported in separate Excel files that are published -# in demo_r_mweek3 Reference metadata page so this test no longer -# applies, as of 2025-02-17 -# test_that("Get eurotime2date error message related to W99 values", { -# skip_on_cran() -# skip_if_offline() -# expect_warning( -# get_eurostat( -# "demo_r_mweek3", -# filters = list( -# sex = "F", -# age = "TOTAL", -# geo = "BG" -# ) -# ) -# ) -# }) - test_that("cut_to_classes works", { skip_on_cran() skip_if_offline() diff --git a/tests/testthat/test_10_sdmx.R b/tests/testthat/test_10_sdmx.R new file mode 100644 index 00000000..0b9ca13c --- /dev/null +++ b/tests/testthat/test_10_sdmx.R @@ -0,0 +1,32 @@ +test_that("get_eurostat_sdmx works", { + skip_on_cran() + skip_if_offline() + + # Dimension order for DS-059328: + # Reporter.partner.product.flow.period.indicators + # although it seems that omitting period is no problem? + + prodcom <- get_eurostat_sdmx(id = "DS-059328", agency = "eurostat_comext", + filters = + list( + FREQ = c("A"), # Annual + REPORTER = "FR", # France + PARTNER = "US", # United States + PRODUCT = c("122"), # 122-Food and beverages / Processed / Mainly for household consumption + FLOW = "2", # 1-IMPORT, 2-EXPORT + INDICATORS = "VALUE_EUR"), verbose = FALSE) + + expect_equal(unique(prodcom$freq), "A") + + prodcom2 <- get_eurostat_sdmx(id = "ds-059328", agency = "eurostat_comext", + filters = + list( + FREQ = c("M"), + REPORTER = "FR", + PARTNER = "US", + PRODUCT = c("310"), # 310-Fuels and lubricants / Primary + FLOW = "1", + INDICATORS = "VALUE_EUR"), verbose = FALSE, use.data.table = TRUE) + + expect_equal(unique(prodcom2$freq), "M") +}) diff --git a/vignettes/articles/dimlst_vs_allconceptschemes.Rmd b/vignettes/articles/dimlst_vs_allconceptschemes.Rmd index ecfa452e..3d77efa7 100644 --- a/vignettes/articles/dimlst_vs_allconceptschemes.Rmd +++ b/vignettes/articles/dimlst_vs_allconceptschemes.Rmd @@ -32,7 +32,7 @@ AGECHILD Age of the child AGEDEF Age definition ``` -An example of the new Concept Scheme file for dataset `NAMA_10_GDP` (see instructions for downloading [here](https://wikis.ec.europa.eu/display/EUROSTATHELP/API+SDMX+2.1+-+metadata+query#APISDMX2.1metadataquery-SDMX2.1endpoint-REST-SDMX-ML2.1ConceptScheme)): +An example of the new Concept Scheme file for dataset `NAMA_10_GDP` (see instructions for downloading [here](https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-getting-started/sdmx2.1#APIGettingstartedwithSDMX2.1API-Lookingupinthemetadataofadataset)): ```{xml}