Improving load_aneel()

[bernardo-sieira]

About the energy_development_budget dataset:

We used to download the data from a googledrive directory, now unavaiable.
Considering we are trying to submit and update the package version on CRAN, the solution being adopted is simple and temporary. We will solely provide the CDE data as in the .csv files avaibale here. They are anual and only cover Rede-Aberta.

After the CRAN submission (and approval), it is proposed to implement a broader function with more options.

I recommend to give the user a new input ithat refers to the months wanted (to be used paired to the years) :

MONTH-YEAR

The function must read the .zip files avaible at the same address. A reading code could possibly be the following:

download_energy_development_budget <- function(years = NULL, months = NULL) {
  # Default months to all
  if (is.null(months)) months <- 1:12

  # Portuguese month abbreviations used in the file names
  month_abbr_pt <- c(
    "jan", "fev", "mar", "abr", "mai", "jun",
    "jul", "ago", "set", "out", "nov", "dez"
  )

  ## Query the CKAN API to get all resources for this dataset
  api_url <- "https://dadosabertos.aneel.gov.br/api/3/action/package_show"
  response <- httr::GET(api_url, query = list(id = "beneficiarios-da-cde"))

  if (httr::status_code(response) != 200) {
    stop("Failed to query ANEEL open data API. HTTP status: ", httr::status_code(response))
  }

  api_data <- httr::content(response, as = "parsed", simplifyVector = TRUE)
  resources <- api_data$result$resources

  # Keep only resources that have a download URL ending in .zip or .csv
  resources <- resources[grepl("\\.(zip|csv)$", resources$url, ignore.case = TRUE), ]

  if (nrow(resources) == 0) {
    stop("No downloadable resources found for the energy_development_budget dataset on ANEEL open data portal.")
  }

  ## Parse year and month from resource names/URLs
  # File names follow patterns like: cde-beneficiarios-01jan2024.zip
  # or may contain year info in the name field
  resources$parsed_year <- NA_integer_
  resources$parsed_month <- NA_integer_

  for (i in seq_len(nrow(resources))) {
    url_lower <- tolower(resources$url[i])
    name_lower <- tolower(resources$name[i])
    text <- paste(url_lower, name_lower)

    # Try to match pattern like "01jan2024" or "jan2024" or "012024"
    for (m in seq_along(month_abbr_pt)) {
      pattern <- paste0("(\\d{0,2})", month_abbr_pt[m], "(\\d{4})")
      match <- regmatches(text, regexpr(pattern, text))
      if (length(match) > 0 && nchar(match) > 0) {
        yr <- as.integer(sub(paste0(".*", month_abbr_pt[m], "(\\d{4}).*"), "\\1", match))
        resources$parsed_year[i] <- yr
        resources$parsed_month[i] <- m
        break
      }
    }

    # Fallback: try to extract a 4-digit year from the name if not yet parsed
    if (is.na(resources$parsed_year[i])) {
      yr_match <- regmatches(text, regexpr("\\b(20\\d{2})\\b", text))
      if (length(yr_match) > 0 && nchar(yr_match) > 0) {
        resources$parsed_year[i] <- as.integer(yr_match)
      }
    }
  }

  # Remove resources we couldn't parse a year for
  resources <- resources[!is.na(resources$parsed_year), ]

  # Default years to all available if NULL
  if (is.null(years)) {
    years <- sort(unique(resources$parsed_year))
  }

  # Filter to requested years and months
  resources <- resources[
    resources$parsed_year %in% years &
      (is.na(resources$parsed_month) | resources$parsed_month %in% months),
  ]

  if (nrow(resources) == 0) {
    stop(
      "No resources found for the requested years (",
      paste(years, collapse = ", "),
      ") and months (",
      paste(months, collapse = ", "),
      "). Check available data at https://dadosabertos.aneel.gov.br/dataset/beneficiarios-da-cde"
    )
  }

  message(
    "Downloading ", nrow(resources),
    " file(s) from ANEEL open data portal. This may take a while."
  )

  ## Download and read each resource
  dir <- tempdir()

  dat_list <- purrr::map(seq_len(nrow(resources)), function(idx) {
    resource_url <- resources$url[idx]
    file_extension <- tolower(sub(".*\\.", ".", resource_url))

    temp <- tempfile(fileext = file_extension, tmpdir = dir)

    tryCatch(
      {
        utils::download.file(url = resource_url, destfile = temp, mode = "wb", quiet = TRUE)

        if (file_extension == ".zip") {
          # Unzip and read CSV(s) inside
          exdir <- file.path(dir, paste0("cde_", idx))
          dir.create(exdir, showWarnings = FALSE, recursive = TRUE)
          utils::unzip(temp, exdir = exdir)

          csv_files <- list.files(exdir, pattern = "\\.csv$", full.names = TRUE, recursive = TRUE)
          if (length(csv_files) == 0) {
            warning("No CSV found inside ZIP: ", resource_url)
            return(NULL)
          }

          dfs <- lapply(csv_files, function(f) {
            tryCatch(
              data.table::fread(f, encoding = "Latin-1"),
              error = function(e) {
                warning("Error reading ", f, ": ", e$message)
                NULL
              }
            )
          })

          dfs <- dfs[!sapply(dfs, is.null)]
          if (length(dfs) == 0) return(NULL)
          do.call(rbind, dfs)
        } else {
          # Assume CSV
          data.table::fread(temp, encoding = "Latin-1")
        }
      },
      error = function(e) {
        warning("Failed to download/read: ", resource_url, " - ", e$message)
        NULL
      }
    )
  })

  # Remove NULLs and bind
  dat_list <- dat_list[!sapply(dat_list, is.null)]

  if (length(dat_list) == 0) {
    stop("All downloads failed. Please check your internet connection and try again.")
  }

  dat <- data.table::rbindlist(dat_list, fill = TRUE) %>%
    tibble::as_tibble()

  message("Successfully loaded ", nrow(dat), " rows from ", length(dat_list), " file(s).")

  return(dat)
}

[bernardo-sieira]

It is also necessary to fit the most recent change to the package format: 1. the download method applied isn't the one we tend to use. Instead of navigating the website it has direct links for each year. 2. proper documentation should be improved.