Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
78 commits
Select commit Hold shift + click to select a range
c9d8683
use withr to prevent local file being left (#98)
Jul 18, 2025
446e6c5
Fix test to fail on missing date in interval
PietrH Aug 14, 2025
ab9a5ac
Add test for longer interval
PietrH Aug 14, 2025
73aa1ef
Add test for daterange.
PietrH Aug 14, 2025
ec63bab
Add code sections
PietrH Aug 14, 2025
db5c2b0
Create interval from beginning to end
PietrH Aug 14, 2025
4704d59
Add bugfix to NEWS
PietrH Aug 14, 2025
90d7088
Add test for presence of all expected timesteps in interval.
PietrH Aug 14, 2025
efd616f
Merge branch 'main' into 101-get_vpts-not-downloading-full-interval-i…
PietrH Aug 14, 2025
a149f02
Merge branch 'main' into 101-get_vpts-not-downloading-full-interval-i…
PietrH Aug 14, 2025
80bfa8d
Merge branch '98-prevent-remaining-files' into resolve-conflicts
bart1 Aug 14, 2025
eb06833
Merge pull request #107 from aloftdata/resolve-conflicts
bart1 Aug 14, 2025
25a5f0e
add retries to `SE` downloads
Aug 14, 2025
8481256
also use temp file and do not retry 404
Aug 14, 2025
83fd64d
Update tests/testthat/test-get_vpts.R
PietrH Aug 14, 2025
561204a
switch to setequal: more strict
PietrH Aug 14, 2025
b8ce37a
Switch to setequal, more strict
PietrH Aug 14, 2025
0a1925a
small fixes
Aug 14, 2025
57291d1
Switch test to using a fixed interval to prevent intermitend errors
Aug 14, 2025
9a085f6
Generalize reading data from multiple pvols and support romania
Aug 14, 2025
b4f9f2f
Merge pull request #105 from aloftdata/101-get_vpts-not-downloading-f…
PietrH Aug 18, 2025
ffffe7e
Merge pull request #110 from aloftdata/main
bart1 Aug 18, 2025
408ba95
Update R/get_pvol_se.R
bart1 Aug 18, 2025
138b138
Skip test if radar has not recently been updated
Aug 20, 2025
c8f51a1
Generalize to function
Aug 20, 2025
1262edd
Merge pull request #113 from aloftdata/skip_test_se_404
bart1 Aug 20, 2025
31a3459
Merge pull request #109 from aloftdata/104-romanian-radar-data-is-ope…
barthoekstra Aug 25, 2025
77aa7c5
Merge branch 'main' into 98-prevent-remaining-files
bart1 Aug 25, 2025
5c2860f
Generalize radar recoding
Aug 25, 2025
3341129
Style
PietrH Aug 25, 2025
05c2374
devtools::document()
PietrH Aug 25, 2025
55e9276
Style, clarify return type.
PietrH Aug 25, 2025
896823a
Style. Add comment.
PietrH Aug 25, 2025
e5a0c86
Merge pull request #100 from aloftdata/98-prevent-remaining-files
bart1 Aug 25, 2025
d1087a3
Style
PietrH Aug 25, 2025
c87fe10
Clarify option fallbacks
PietrH Aug 25, 2025
6b6ca7a
Merge branch 'main' into 99-generalize-handling-of-non-existing-radars
bart1 Aug 25, 2025
5938852
update to prevent note on global visible variables
Aug 25, 2025
059b8b5
make url more general
Aug 25, 2025
c0d0a6a
style updates
Aug 25, 2025
67b9842
fix scoping issue
Aug 25, 2025
20db19a
reorganize some util functions
Aug 25, 2025
60a6c89
place onload in zzz.R following devtools and rlang
Aug 25, 2025
288957f
consistent utils naming
Aug 25, 2025
ce3fd62
use building pluck
Aug 25, 2025
8f94a5c
Merge pull request #114 from aloftdata/99-generalize-handling-of-non-…
bart1 Aug 25, 2025
e27a36e
style and update news
Aug 25, 2025
eb19030
remove extra space
Aug 25, 2025
4d23e7d
typo
Aug 25, 2025
46952f8
cran comments
Aug 25, 2025
6a2cbb4
Increment version number to 0.2.1
Aug 25, 2025
b0ddc92
Increment version number to 0.2.1.9000
Aug 25, 2025
8f34d6e
Merge pull request #115 from aloftdata/111-release-getrad-021
bart1 Aug 25, 2025
f7418da
retry for nexrad (#116)
Aug 25, 2025
3d1bf81
fix
Aug 25, 2025
84a0a79
update url us
Aug 28, 2025
6bc09a7
add news item
Aug 28, 2025
78da350
Merge pull request #119 from aloftdata/118-nexrad-moving-to-new-bucke…
bart1 Aug 28, 2025
b5f0714
also in test
Aug 28, 2025
f23d668
Merge pull request #121 from aloftdata/118-nexrad-moving-to-new-bucke…
bart1 Aug 28, 2025
cc4a7cb
clean cache after tests
Aug 29, 2025
986b406
Merge branch 'main' into 116-503-on-nexrad-when-getting-radars
bart1 Sep 2, 2025
89b2382
Support slovakia
Sep 2, 2025
0daac59
document german availability site
Sep 3, 2025
046ccf3
resolve error in denmark and make test run atleast locally so it woul…
Sep 15, 2025
3b4687c
increase local test coverage
Sep 15, 2025
0eaf4e7
Explicitly set backend to prevent warning
Sep 15, 2025
75db8db
skip test on cran and when host offline
Sep 19, 2025
9472816
remove duplicated test
Sep 19, 2025
6e5cf6e
Merge pull request #117 from aloftdata/116-503-on-nexrad-when-getting…
bart1 Sep 19, 2025
9068caf
Merge pull request #123 from aloftdata/122-files-left-on-disk-by-cach…
bart1 Sep 19, 2025
e08dfbf
Merge branch 'main' into 124-slovakian-radar-data-is-open
bart1 Sep 19, 2025
b8f62bc
Merge pull request #125 from aloftdata/124-slovakian-radar-data-is-open
bart1 Sep 19, 2025
6ddba4b
polish news
Sep 19, 2025
354bfd4
update cran comments
Sep 29, 2025
4e3d7bc
Increment version number to 0.2.2
Sep 29, 2025
1a09b27
Increment version number to 0.2.2.9000
Sep 29, 2025
980f05f
Merge pull request #127 from aloftdata/126-release-getrad-022
bart1 Sep 29, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 28 additions & 26 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ message: 'To cite package "getRad" in publications use:'
type: software
license: MIT
title: 'getRad: Download Radar Data for Biological Research'
version: 0.2.0
version: 0.2.0.9000
doi: 10.32614/CRAN.package.getRad
abstract: Load polar volume and vertical profile data for aeroecological research
directly into R. With 'getRad' you can access data from several sources in Europe
and the US and standardize it to facilitate further exploration in tools such as
Expand All @@ -29,6 +30,7 @@ authors:
email: peter.desmet@inbo.be
orcid: https://orcid.org/0000-0002-8442-8025
affiliation: Research Institute for Nature and Forest (INBO)
repository: https://CRAN.R-project.org/package=getRad
repository-code: https://github.com/aloftdata/getRad
url: https://aloftdata.github.io/getRad/
contact:
Expand Down Expand Up @@ -267,6 +269,31 @@ references:
orcid: https://orcid.org/0000-0002-6983-2759
year: '2025'
doi: 10.32614/CRAN.package.vroom
- type: software
title: withr
abstract: 'withr: Run Code ''With'' Temporarily Modified Global State'
notes: Imports
url: https://withr.r-lib.org
repository: https://CRAN.R-project.org/package=withr
authors:
- family-names: Hester
given-names: Jim
- family-names: Henry
given-names: Lionel
email: lionel@posit.co
- family-names: Müller
given-names: Kirill
email: krlmlr+r@mailbox.org
- family-names: Ushey
given-names: Kevin
email: kevinushey@gmail.com
- family-names: Wickham
given-names: Hadley
email: hadley@posit.co
- family-names: Chang
given-names: Winston
year: '2025'
doi: 10.32614/CRAN.package.withr
- type: software
title: xml2
abstract: 'xml2: Parse XML'
Expand Down Expand Up @@ -483,29 +510,4 @@ references:
orcid: https://orcid.org/0000-0001-6573-066X
year: '2025'
doi: 10.32614/CRAN.package.vol2birdR
- type: software
title: withr
abstract: 'withr: Run Code ''With'' Temporarily Modified Global State'
notes: Suggests
url: https://withr.r-lib.org
repository: https://CRAN.R-project.org/package=withr
authors:
- family-names: Hester
given-names: Jim
- family-names: Henry
given-names: Lionel
email: lionel@posit.co
- family-names: Müller
given-names: Kirill
email: krlmlr+r@mailbox.org
- family-names: Ushey
given-names: Kevin
email: kevinushey@gmail.com
- family-names: Wickham
given-names: Hadley
email: hadley@posit.co
- family-names: Chang
given-names: Winston
year: '2025'
doi: 10.32614/CRAN.package.withr

8 changes: 4 additions & 4 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: getRad
Title: Download Radar Data for Biological Research
Version: 0.2.0.9000
Version: 0.2.2.9000
Authors@R: c(
person("Bart", "Kranstauber", , "b.kranstauber@uva.nl", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-8303-780X", affiliation = "University of Amsterdam")),
Expand All @@ -15,7 +15,7 @@ Authors@R: c(
person("Hidde", "Leijnse", role = "ctb",
comment = c(ORCID = "0000-0001-7835-4480", affiliation = "Royal Netherlands Meteorological Institute")),
person("Bart", "Hoekstra", role = "ctb",
comment = c(ORCID = " 0000-0002-7085-3805", affiliation = "University of Amsterdam")),
comment = c(ORCID = "0000-0002-7085-3805", affiliation = "University of Amsterdam")),
person("University of Amsterdam", role = "cph",
comment = c(ROR = "04dkp9463")),
person("Biodiversa+", role = "fnd",
Expand Down Expand Up @@ -45,6 +45,7 @@ Imports:
tools,
utils,
vroom,
withr,
xml2
Suggests:
askpass,
Expand All @@ -58,8 +59,7 @@ Suggests:
sf,
testthat (>= 3.0.0),
tidyr,
vol2birdR,
withr
vol2birdR
VignetteBuilder:
knitr
Config/testthat/edition: 3
Expand Down
15 changes: 14 additions & 1 deletion NEWS.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
# getRad (development version)

* Support downloading Swedish data (#96).
# getRad 0.2.2

* Support downloading Slovakian polar volume data (#124).
* Add retry attempts to `get_weather_radars` for NEXRAD to prevent failure (#116).
* Update of NEXRAD url (#118).
* Fix CRAN warning where cache was not cleaned after tests (#122).
* Resolve `withr` error for Danish radars.

# getRad 0.2.1

* A bug (#101) in `get_vpts()` was fixed that caused the function to only return the first day of an interval, regardless of the length of the interval (#105).
* Support downloading Swedish polar volume data (#96).
* Support downloading Romanian polar volume data (#104).
* How attribute is now present in Czech data (#102).
* Use `withr` to prevent files being left in temporary directories (#98).

# getRad 0.2.0

Expand Down
5 changes: 5 additions & 0 deletions R/get_pvol.R
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ get_pvol <- function(radar = NULL, datetime = NULL, ...) {


# Helper function to find the function for a specific radar
# This function is only helpful in get_pvol and therefor not in a utils file
select_get_pvol_function <- function(radar, ..., call = rlang::caller_env()) {
if (is_nexrad(radar)) {
return("get_pvol_us")
Expand All @@ -133,6 +134,8 @@ select_get_pvol_function <- function(radar, ..., call = rlang::caller_env()) {
cntry_code == "ee" ~ "get_pvol_ee",
cntry_code == "cz" ~ "get_pvol_cz",
cntry_code == "se" ~ "get_pvol_se",
cntry_code == "ro" ~ "get_pvol_ro",
cntry_code == "sk" ~ "get_pvol_sk",
.default = NA
))
if (rlang::is_na(fun)) {
Expand All @@ -144,3 +147,5 @@ select_get_pvol_function <- function(radar, ..., call = rlang::caller_env()) {
}
return(fun)
}


51 changes: 1 addition & 50 deletions R/get_pvol_cz.R
Original file line number Diff line number Diff line change
Expand Up @@ -36,55 +36,6 @@ get_pvol_cz <- function(radar, time, ..., call = rlang::caller_env()) {
time + lubridate::minutes(5)
)
))
files_to_get <- files_to_get |> dplyr::mutate(
req = purrr::pmap(list(x = base, y = file), function(x, y) {
httr2::request(x) |>
httr2::req_url_path_append(y) |>
req_user_agent_getrad()
})
)


files_to_get$resp <- files_to_get$req |>
httr2::req_perform_parallel(
paths = replicate(
length(files_to_get$req),
tempfile(fileext = ".h5")
)
)
polar_volumes_tibble <- files_to_get |>
dplyr::mutate(
tempfile = purrr::map_chr(resp, purrr::chuck, "body"),
pvol = purrr::map(tempfile, ~ bioRad::read_pvolfile(.x)),
remove = purrr::map(tempfile, ~ file.remove(.x))
)
# Check if all parameter have same attributes
list_of_attribute_tables <- purrr::map(
purrr::chuck(polar_volumes_tibble, "pvol"),
bioRad::attribute_table
)
all_params_same_attributes <- all(unlist(lapply(
lapply(list_of_attribute_tables[-1], dplyr::select, -"param"), all.equal,
dplyr::select(list_of_attribute_tables[[1]], -"param")
)))
if (!all_params_same_attributes) {
cli::cli_abort("Not all polar volumes have the same attributes",
class = "getRad_error_differing_attributes_cz", call = call
)
}
pvol <- Reduce(
function(x, y) {
x$scans <- mapply(
function(i, j) {
i$params <- c(i$params, j$params)
i
},
x$scans, y$scans,
SIMPLIFY = FALSE
)
x
},
polar_volumes_tibble$pvol
)
pvol<-read_pvol_from_url_per_param(paste0(files_to_get$base, files_to_get$file))
pvol
}
61 changes: 31 additions & 30 deletions R/get_pvol_de.R
Original file line number Diff line number Diff line change
Expand Up @@ -59,38 +59,39 @@ get_pvol_de <- function(radar, time, ...,
}
)
)

files_to_get$resp <- files_to_get$req |>
httr2::req_perform_parallel(
paths = replicate(
length(files_to_get$req),
tempfile(fileext = ".h5")
pvol <- withr::with_tempdir({
files_to_get$resp <- files_to_get$req |>
httr2::req_perform_parallel(
paths = replicate(
length(files_to_get$req),
tempfile(fileext = ".h5", tmpdir = getwd())
)
)
files_to_get <- files_to_get |>
dplyr::mutate(
tempfile = purrr::map_chr(resp, ~ .x$body)
) |>
dplyr::mutate(
scan = purrr::map(tempfile, ~ read_scan(.x)),
remove = purrr::map(tempfile, ~ file.remove(.x))
) |>
tidyr::separate_wider_delim(sweep,
delim = "_", cols_remove = FALSE,
names = c("vol", "name", "param", "iter")
) |>
dplyr::group_by(iter) |>
dplyr::summarize(
scan = list(scan),
param = list(param), radar = unique(radar)
) |>
dplyr::mutate(
scan = purrr::map2(scan, param, ~ list_to_scan(.x, .y))
)
list_to_pvol(files_to_get$scan,
time = time, radar = radar,
source = glue::glue("NOD:{radar},CMT:constructed from opendata.dwd.de")
)
files_to_get <- files_to_get |>
dplyr::mutate(
tempfile = purrr::map_chr(resp, ~ .x$body)
) |>
dplyr::mutate(
scan = purrr::map(tempfile, ~ read_scan(.x)),
remove = purrr::map(tempfile, ~ file.remove(.x))
) |>
tidyr::separate_wider_delim(sweep,
delim = "_", cols_remove = FALSE,
names = c("vol", "name", "param", "iter")
) |>
dplyr::group_by(iter) |>
dplyr::summarize(
scan = list(scan),
param = list(param), radar = unique(radar)
) |>
dplyr::mutate(
scan = purrr::map2(scan, param, ~ list_to_scan(.x, .y))
)
pvol <- list_to_pvol(files_to_get$scan,
time = time, radar = radar,
source = glue::glue("NOD:{radar},CMT:constructed from opendata.dwd.de")
)
})
return(pvol)
}

Expand Down
33 changes: 17 additions & 16 deletions R/get_pvol_dk.R
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
get_pvol_dk <- function(radar, time, ..., call = rlang::caller_env()) {
req <- httr2::request(
getOption(
"getRad.dk_url",
"https://dmigw.govcloud.dk/v1/radardata/download"
)
) |>
req_user_agent_getrad() |>
httr2::req_url_path_append(
glue::glue(getOption(
"getRad.dk_file_format",
"{radar}_{strftime(time,'%Y%m%d%H%M', tz='UTC')}.vol.h5"
))
withr::with_file("file.h5", {
req <- httr2::request(
getOption(
"getRad.dk_url",
"https://dmigw.govcloud.dk/v1/radardata/download"
)
) |>
httr2::req_url_query(`api-key` = get_secret("dk_api_key")) |>
httr2::req_perform(path = tempfile(fileext = ".h5"), error_call = call)
pvol <- bioRad::read_pvolfile(req$body, ...)
file.remove(req$body)
req_user_agent_getrad() |>
httr2::req_url_path_append(
glue::glue(getOption(
"getRad.dk_file_format",
"{radar}_{strftime(time,'%Y%m%d%H%M', tz='UTC')}.vol.h5"
))
) |>
httr2::req_url_query(`api-key` = get_secret("dk_api_key")) |>
httr2::req_perform(path = "file.h5", error_call = call)
pvol<-bioRad::read_pvolfile(req$body, ...)
})
return(pvol)
}
23 changes: 12 additions & 11 deletions R/get_pvol_ee.R
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ get_pvol_ee <- function(radar, time, ...,
),
list(isEqual = list(
field = "Radar",
value = dplyr::case_match(
value = radar_recode(
radar,
"eesur" ~ "S\u00FCrgavere radar (SUR)",
"eehar" ~ "Harku radar (HAR)"
"eesur" = "S\u00FCrgavere radar (SUR)",
"eehar" = "Harku radar (HAR)", call = call
)
))
))))
Expand All @@ -44,13 +44,14 @@ get_pvol_ee <- function(radar, time, ...,
class = "getRad_error_get_pvol_ee_differing_n_files"
)
}
req <- httr2::request("https://avaandmed.keskkonnaportaal.ee/_vti_bin/RmApi.svc/active/items/") |>
req_user_agent_getrad() |>
httr2::req_url_path_append(files$documents[[1]]$id) |>
httr2::req_url_path_append("files/0") |>
req_retry_getrad() |>
httr2::req_perform(path = tempfile(fileext = ".h5"), error_call = call)
pvol <- bioRad::read_pvolfile(req$body, ...)
file.remove(req$body)
pvol <- withr::with_tempfile("file", fileext = ".h5", {
req <- httr2::request("https://avaandmed.keskkonnaportaal.ee/_vti_bin/RmApi.svc/active/items/") |>
req_user_agent_getrad() |>
httr2::req_url_path_append(files$documents[[1]]$id) |>
httr2::req_url_path_append("files/0") |>
req_retry_getrad() |>
httr2::req_perform(path = file, error_call = call)
bioRad::read_pvolfile(req$body, ...)
})
return(pvol)
}
22 changes: 11 additions & 11 deletions R/get_pvol_fi.R
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,17 @@ get_pvol_fi <- function(radar, time, ..., call = rlang::caller_env()) {
"{strftime(time,'%Y', tz='UTC')}/{strftime(time,'%m', tz='UTC')}/{strftime(time,'%d', tz='UTC')}/{radar}/{strftime(time,'%Y%m%d%H%M', tz='UTC')}_{radar}_PVOL.h5"
))
)

req <- req |>
httr2::req_perform(path = tempfile(fileext = ".h5"), error_call = call)
rlang::check_installed("rhdf5", "To adjust the polar volume files for Finish data.", call = call)
hdf_connection <- rhdf5::H5Fopen(req$body)
group <- rhdf5::H5Gopen(hdf_connection, "what")
rhdf5::h5writeAttribute("PVOL", group, "object")
rhdf5::H5Fclose(hdf_connection)
rhdf5::H5Gclose(group)
pvol <- bioRad::read_pvolfile(req$body, ...)
file.remove(req$body)
pvol <- withr::with_tempfile("file", fileext = ".h5", {
req <- req |>
httr2::req_perform(path = file, error_call = call)
rlang::check_installed("rhdf5", "To adjust the polar volume files for Finish data.", call = call)
hdf_connection <- rhdf5::H5Fopen(req$body)
group <- rhdf5::H5Gopen(hdf_connection, "what")
rhdf5::h5writeAttribute("PVOL", group, "object")
rhdf5::H5Fclose(hdf_connection)
rhdf5::H5Gclose(group)
bioRad::read_pvolfile(req$body, ...)
})
return(pvol)
}
# https://en.ilmatieteenlaitos.fi/radar-data-on-aws-s3
Expand Down
Loading