Skip to content

Commit 665f253

Browse files
Add parallel download support to fly_fetch()
New `workers` param enables concurrent downloads via furrr/future. Default remains sequential (workers = 1) for safety. Fixes #21 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 5777ab0 commit 665f253

5 files changed

Lines changed: 77 additions & 16 deletions

File tree

DESCRIPTION

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Package: fly
22
Title: Airphoto Footprint Estimation and Coverage Selection
3-
Version: 0.2.0
3+
Version: 0.2.1
44
Authors@R: c(
55
person("Allan", "Irvine", , "al@newgraphenvironment.com", role = c("aut", "cre"),
66
comment = c(ORCID = "0000-0002-3495-2128")),
@@ -26,6 +26,8 @@ Imports:
2626
stringr
2727
Suggests:
2828
bookdown,
29+
furrr,
30+
future,
2931
testthat (>= 3.0.0),
3032
knitr,
3133
rmarkdown

NEWS.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# fly (development version)
22

3+
## 0.2.1 (2026-03-11)
4+
5+
- Add `workers` parameter to `fly_fetch()` for parallel downloads via `furrr`/`future` ([#21](https://github.com/NewGraphEnvironment/fly/issues/21))
6+
- Add `furrr` and `future` to Suggests
7+
38
## 0.2.0 (2026-03-11)
49

510
- **BREAKING:** Remove `fly_query_habitat()`, `fly_query_lakes()`, `fly_trim_habitat()` — migrate to [fresh](https://github.com/NewGraphEnvironment/fresh) ([#19](https://github.com/NewGraphEnvironment/fly/issues/19))

R/fly_fetch.R

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
#' not exist.
1515
#' @param overwrite If `FALSE` (default), skip files that already exist
1616
#' in `dest_dir`.
17+
#' @param workers Number of parallel download workers. `1` (default) runs
18+
#' sequentially. Values greater than 1 use [furrr::future_map_dfr()] with
19+
#' [future::multisession] — requires `furrr` and `future` packages.
1720
#' @return A tibble with columns `airp_id`, `url`, `dest`, and `success`.
1821
#'
1922
#' @details
@@ -28,6 +31,10 @@
2831
#' Photos with missing (`NA` or empty) URLs are skipped and reported as
2932
#' `success = FALSE` in the output.
3033
#'
34+
#' When `workers > 1`, a [future::multisession] plan is set for the
35+
#' duration of the call and restored on exit. Each download is independent
36+
#' so parallelism is safe.
37+
#'
3138
#' @examples
3239
#' centroids <- sf::st_read(system.file("testdata/photo_centroids.gpkg", package = "fly"))
3340
#'
@@ -38,7 +45,8 @@
3845
#'
3946
#' @export
4047
fly_fetch <- function(photos_sf, type = "thumbnail",
41-
dest_dir = "photos", overwrite = FALSE) {
48+
dest_dir = "photos", overwrite = FALSE,
49+
workers = 1) {
4250
type <- match.arg(type, c("thumbnail", "flight_log", "calibration", "georef"))
4351

4452
url_col <- switch(type,
@@ -63,29 +71,51 @@ fly_fetch <- function(photos_sf, type = "thumbnail",
6371
seq_len(nrow(photos_sf))
6472
}
6573

66-
results <- dplyr::tibble(
67-
airp_id = ids,
68-
url = urls,
69-
dest = NA_character_,
70-
success = FALSE
71-
)
74+
rows <- lapply(seq_along(urls), function(i) {
75+
list(airp_id = ids[i], url = urls[i])
76+
})
7277

73-
for (i in seq_len(nrow(results))) {
74-
u <- results$url[i]
75-
if (is.na(u) || u == "") next
78+
# Self-contained download function that workers can serialize
79+
dl_fn <- function(row) {
80+
airp_id <- row$airp_id
81+
u <- row$url
82+
83+
if (is.na(u) || u == "") {
84+
return(dplyr::tibble(
85+
airp_id = airp_id, url = u,
86+
dest = NA_character_, success = FALSE
87+
))
88+
}
7689

7790
dest_file <- file.path(dest_dir, basename(u))
78-
results$dest[i] <- dest_file
7991

8092
if (!overwrite && file.exists(dest_file)) {
81-
results$success[i] <- TRUE
82-
next
93+
return(dplyr::tibble(
94+
airp_id = airp_id, url = u,
95+
dest = dest_file, success = TRUE
96+
))
8397
}
8498

85-
results$success[i] <- tryCatch({
99+
ok <- tryCatch({
86100
utils::download.file(u, dest_file, mode = "wb", quiet = TRUE)
87101
file.exists(dest_file) && file.size(dest_file) > 0
88102
}, error = function(e) FALSE)
103+
104+
dplyr::tibble(
105+
airp_id = airp_id, url = u,
106+
dest = dest_file, success = ok
107+
)
108+
}
109+
110+
if (workers > 1) {
111+
rlang::check_installed(c("furrr", "future"),
112+
reason = "for parallel downloads (workers > 1)")
113+
old_plan <- future::plan(future::multisession, workers = workers)
114+
on.exit(future::plan(old_plan), add = TRUE)
115+
results <- furrr::future_map_dfr(rows, dl_fn,
116+
.options = furrr::furrr_options(packages = "dplyr"))
117+
} else {
118+
results <- purrr::map_dfr(rows, dl_fn)
89119
}
90120

91121
n_ok <- sum(results$success)

man/fly_fetch.Rd

Lines changed: 10 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

tests/testthat/test-fly_fetch.R

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,21 @@ test_that("fly_fetch rejects invalid type", {
5858
expect_error(fly_fetch(centroids, type = "bogus"))
5959
})
6060

61+
test_that("fly_fetch supports parallel downloads with workers > 1", {
62+
skip_if_not_installed("furrr")
63+
skip_if_not_installed("future")
64+
centroids <- sf::st_read(testdata_path("photo_centroids.gpkg"), quiet = TRUE)
65+
dest <- file.path(tempdir(), "fly_test_parallel")
66+
unlink(dest, recursive = TRUE)
67+
68+
result <- fly_fetch(centroids[1:2, ], type = "thumbnail",
69+
dest_dir = dest, workers = 2)
70+
expect_s3_class(result, "tbl_df")
71+
expect_equal(nrow(result), 2)
72+
downloaded <- result[result$success, ]
73+
expect_true(all(file.exists(downloaded$dest)))
74+
})
75+
6176
test_that("fly_fetch maps type to correct URL column", {
6277
centroids <- sf::st_read(testdata_path("photo_centroids.gpkg"), quiet = TRUE)
6378
dest <- file.path(tempdir(), "fly_test_flight_log")

0 commit comments

Comments
 (0)