diff --git a/.gitignore b/.gitignore index 090f765..d1b215b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ sample-project/analysis_results.rds sample-project/report.txt sample-project/.bakepipe.state +# Targets backend files +_targets.R +_targets/ + # R files .Rproj.user .Rhistory diff --git a/DESCRIPTION b/DESCRIPTION index 4faa028..ef96f9f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,7 +14,7 @@ License: MIT + file LICENSE Encoding: UTF-8 Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.2 -Imports: callr, fs +Imports: callr, fs, targets Suggests: testthat (>= 3.0.0) Config/testthat/edition: 3 URL: https://vangberg.github.io/bakepipe/, https://github.com/vangberg/bakepipe diff --git a/NAMESPACE b/NAMESPACE index d65711b..947cb1f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,10 +4,9 @@ export(clean) export(external_in) export(file_in) export(file_out) +export(generate_targets_file) export(run) export(status) importFrom(callr,r) importFrom(fs,path_rel) importFrom(stats,setNames) -importFrom(utils,read.csv) -importFrom(utils,write.csv) diff --git a/R/clean.R b/R/clean.R index f68d000..e0eca2e 100644 --- a/R/clean.R +++ b/R/clean.R @@ -4,7 +4,8 @@ #' intermediate files. This provides a complete clean of generated artifacts. #' The pipeline can be regenerated by running run() again. #' -#' @param verbose Logical. If TRUE (default), prints progress messages to console. +#' @param verbose Logical. If TRUE (default), prints progress messages +#' to console. #' @return Character vector of file paths that were actually removed #' @export #' @examples @@ -13,94 +14,50 @@ #' dir.create(temp_dir) #' sample_proj <- system.file("extdata", "sample-project", package = "bakepipe") #' file.copy(sample_proj, temp_dir, recursive = TRUE) -#' +#' #' # Change to the sample project directory #' old_wd <- getwd() #' setwd(file.path(temp_dir, "sample-project")) -#' +#' #' # Run the pipeline first to create output files -#' run(verbose = FALSE) -#' +#' run() +#' #' # Now clean up the generated files #' removed_files <- clean() #' print(removed_files) -#' +#' #' # Restore working directory and clean up #' setwd(old_wd) #' unlink(temp_dir, recursive = TRUE) clean <- function(verbose = TRUE) { - # Parse all scripts to get dependencies - dependencies <- parse() - - # If no scripts found, return empty vector - if (length(dependencies$scripts) == 0) { - if (verbose) { - message("\n[CLEAN] \033[1;36mBakepipe Clean\033[0m") - message("\033[33m No output files found to clean\033[0m\n") + deps <- parse() + + # Collect all output files + files <- unique(unlist(lapply(deps$scripts, function(s) s$outputs))) + + # Add targets artifacts + files <- c(files, "_targets.R") + + # Track removed files + removed_files <- character() + removed <- 0 + for (f in files) { + if (file.exists(f)) { + file.remove(f) + removed_files <- c(removed_files, f) + removed <- removed + 1 } - return(character(0)) } - # Get all outputs from the top-level outputs list - all_outputs <- dependencies$outputs - - # Only try to remove files that actually exist - existing_files <- all_outputs[file.exists(all_outputs)] - - # Print header - if (verbose) { - message("\n[CLEAN] \033[1;36mBakepipe Clean\033[0m") - - if (length(existing_files) == 0) { - message("\033[32m[OK] No output files to clean - all clean!\033[0m\n") - return(character(0)) - } - - message(paste0("\033[33m Found ", length(existing_files), " output file", - if(length(existing_files) > 1) "s" else "", " to remove\033[0m\n")) - } else if (length(existing_files) == 0) { - return(character(0)) + # Remove _targets directory + if (dir.exists("_targets")) { + unlink("_targets", recursive = TRUE) + removed <- removed + 1 } - # Calculate max filename width for alignment - max_width <- max(nchar(existing_files)) - - # Remove the files with progress indicators - removed_files <- character(0) - failed_files <- character(0) - - for (file_path in existing_files) { - if (file.remove(file_path)) { - removed_files <- c(removed_files, file_path) - if (verbose) { - message(sprintf("\033[31m[RM] %s\033[0m", file_path)) - } - } else { - failed_files <- c(failed_files, file_path) - if (verbose) { - message(sprintf("\033[33m[!] %-*s \033[2m(failed to remove)\033[0m", max_width, file_path)) - } - } - } - - # Print summary - if (verbose) { - message("\n\033[1;36m[SUMMARY]\033[0m") - - if (length(removed_files) > 0) { - message(sprintf("\033[32m Removed %d file%s\033[0m", - length(removed_files), - if(length(removed_files) > 1) "s" else "")) - } - - if (length(failed_files) > 0) { - message(sprintf("\033[33m Failed to remove %d file%s\033[0m", - length(failed_files), - if(length(failed_files) > 1) "s" else "")) - } - - message("") + if (verbose && removed > 0) { + message(sprintf("✓ removed %d artifact%s", removed, if (removed != 1) "s" else "")) } - removed_files -} \ No newline at end of file + invisible(removed_files) +} diff --git a/R/generate_targets_file.R b/R/generate_targets_file.R new file mode 100644 index 0000000..30bd385 --- /dev/null +++ b/R/generate_targets_file.R @@ -0,0 +1,189 @@ +#' Generate _targets.R file from bakepipe scripts +#' +#' Parses all R scripts in the project and generates a _targets.R file +#' that defines targets for the targets package. This allows bakepipe +#' to use targets as a backend for pipeline execution. +#' +#' @return Invisibly returns the path to the generated _targets.R file +#' @importFrom callr r +#' @export +generate_targets_file <- function() { + # Parse all scripts to get dependencies + parsed <- parse() + + # Get project root + project_root <- root() + targets_file <- file.path(project_root, "_targets.R") + + # Start building the targets file content + lines <- character(0) + + # Add library calls + lines <- c(lines, "library(targets)") + lines <- c(lines, "") + + # Add the list of targets + lines <- c(lines, "list(") + + # Generate targets for each script and its dependencies + target_lines <- character(0) + + # Track all files that need file targets + all_file_targets <- character(0) + + # First pass: add all external input file targets + for (script_name in names(parsed$scripts)) { + script_info <- parsed$scripts[[script_name]] + + for (ext_file in script_info$externals) { + ext_target_name <- path_to_target_name(ext_file, "") + if (!(ext_target_name %in% all_file_targets)) { + target_lines <- c( + target_lines, + sprintf( + ' tar_target(%s, "%s", format = "file"),', + ext_target_name, + ext_file + ) + ) + all_file_targets <- c(all_file_targets, ext_target_name) + } + } + } + + # Second pass: generate one target per script + for (script_name in names(parsed$scripts)) { + script_info <- parsed$scripts[[script_name]] + + # Generate target name from script path + output_target_name <- path_to_target_name(script_name, "output") + + # Collect dependencies + deps <- character(0) + + # Track the script itself so changes are detected + script_target_name <- path_to_target_name(script_name, "script") + target_lines <- c( + target_lines, + sprintf( + ' tar_target(%s, "%s", format = "file"),', + script_target_name, + script_name + ) + ) + deps <- c(deps, script_target_name) + + # Add external input dependencies + for (ext_file in script_info$externals) { + ext_target_name <- path_to_target_name(ext_file, "") + deps <- c(deps, ext_target_name) + } + + # Add input file dependencies (from other scripts) + for (input_file in script_info$inputs) { + # Search through all scripts to find which one produces this input + for (producer_script in names(parsed$scripts)) { + if (input_file %in% parsed$scripts[[producer_script]]$outputs) { + # Found the producer script - depend on its output target + input_target_name <- path_to_target_name(producer_script, "output") + deps <- c(deps, input_target_name) + break + } + } + } + + # Build the script target + target_start <- c( + sprintf(" tar_target("), + sprintf(" %s,", output_target_name), + sprintf(" {") + ) + + # Add dependencies + dep_lines <- character(0) + for (dep in deps) { + dep_lines <- c(dep_lines, sprintf(" %s", dep)) + } + + # Add script execution via callr for isolation + exec_lines <- c( + sprintf(" callr::r("), + sprintf(" func = function(script_path) {"), + sprintf(" source(script_path, local = TRUE)"), + sprintf(" },"), + sprintf(' args = list(script_path = "%s")', script_name), + sprintf(" )") + ) + + # Add output vector if script has outputs + output_lines <- character(0) + if (length(script_info$outputs) > 0) { + output_files_r <- paste0('"', script_info$outputs, '"', collapse = ", ") + output_lines <- c(output_lines, sprintf(" c(%s)", output_files_r)) + } else { + # No outputs - return empty character vector + output_lines <- c(output_lines, sprintf(" character(0)")) + } + + target_end <- c( + sprintf(" },"), + sprintf(" format = \"file\""), + sprintf(" ),") + ) + + # Combine all parts + full_target <- c(target_start, dep_lines, exec_lines, output_lines, target_end) + target_lines <- c(target_lines, full_target) + + all_file_targets <- c(all_file_targets, output_target_name) + } + + # Remove trailing comma from last target + if (length(target_lines) > 0) { + last_line <- target_lines[length(target_lines)] + target_lines[length(target_lines)] <- sub(",$", "", last_line) + } + + # Combine everything + lines <- c(lines, target_lines) + lines <- c(lines, ")") + + # Write to file + writeLines(lines, targets_file) + + invisible(targets_file) +} + +#' Convert file path to valid R target name +#' +#' Converts a file path to a valid R identifier for use as a target name. +#' Special characters are replaced with underscores. +#' +#' @param path File path +#' @param prefix Prefix to add (e.g., "script", "run", or "") +#' @return Valid R identifier +#' @keywords internal +path_to_target_name <- function(path, prefix = "") { + # Convert to lowercase and replace special chars with underscores + name <- tolower(path) + + # Replace directory separators, dots (except in extensions), hyphens + # and other special characters with underscores + name <- gsub("[/\\.-]", "_", name) + + # Remove any remaining non-alphanumeric characters + name <- gsub("[^a-z0-9_]", "_", name) + + # Remove consecutive underscores + name <- gsub("_+", "_", name) + + # Remove leading/trailing underscores + name <- gsub("^_|_$", "", name) + + # Add prefix if provided + if (nzchar(prefix)) { + name <- paste(prefix, name, sep = "_") + } + + name +} diff --git a/R/graph.R b/R/graph.R deleted file mode 100644 index c9912be..0000000 --- a/R/graph.R +++ /dev/null @@ -1,455 +0,0 @@ -#' Create dependency graph from parsed script data -#' -#' Builds a Directed Acyclic Graph (DAG) where all files are nodes. -#' Node types are determined from parse data: -#' - Inputs: files only in parse_data$inputs (external inputs) -#' - Outputs: files in parse_data$outputs (includes intermediates) -#' - Scripts: script file names -#' -#' @param parse_data List from parse() function with 'scripts', 'inputs', 'outputs' -#' @param state_obj Optional. Data frame from read_state() function with 'file' -#' and 'stale' columns. If provided, will mark nodes as stale/fresh. -#' @return List containing: -#' \itemize{ -#' \item{nodes: Data frame with 'file', 'type', and 'stale' columns} -#' \item{edges: Data frame with 'from' and 'to' columns} -#' } -#' @importFrom stats setNames -graph <- function(parse_data, state_obj = NULL) { - if (length(parse_data$scripts) == 0) { - return(list( - nodes = data.frame(file = character(0), type = character(0), - stale = logical(0), stringsAsFactors = FALSE), - edges = data.frame(from = character(0), to = character(0), - stringsAsFactors = FALSE) - )) - } - - # Build edges first - edges <- build_file_edges(parse_data$scripts) - - # Collect all files as nodes and determine types from graph structure - nodes <- build_file_nodes(parse_data, edges, state_obj) - - # Create graph object - graph_obj <- list( - nodes = nodes, - edges = edges - ) - - # Validate artifact producers (each artifact has exactly one producer) - validate_artifact_producers(graph_obj, parse_data) - - - # Detect cycles - detect_cycles(graph_obj) - - # Propagate staleness to descendants - graph_obj <- propagate_staleness(graph_obj) - - return(graph_obj) -} - -#' Validate that each artifact has exactly one producer -#' -#' Ensures that every artifact (file referenced by file_in()) has exactly one producer. -#' This means each artifact should have exactly one script that produces it - not zero -#' (orphaned) and not more than one (multiple producers). -#' -#' @param graph_obj Graph object from graph() function -#' @param parse_data Parse result with scripts, inputs, outputs -#' @keywords internal -validate_artifact_producers <- function(graph_obj, parse_data) { - nodes <- graph_obj$nodes - edges <- graph_obj$edges - - # Get inputs from parse_data (these need producers) - inputs <- setdiff(parse_data$inputs, parse_data$outputs) - # Find input nodes that need producers - input_nodes <- nodes$file[nodes$type == "artifact" & nodes$file %in% inputs] - - # Find all artifact nodes for multiple producer check - artifact_nodes <- nodes$file[nodes$type == "artifact"] - - # Check for orphaned inputs (zero producers) - orphaned_inputs <- character(0) - for (input_file in input_nodes) { - script_producers <- edges$from[edges$to == input_file & - edges$from %in% nodes$file[nodes$type == "script"]] - if (length(script_producers) == 0) { - orphaned_inputs <- c(orphaned_inputs, input_file) - } - } - - # Check for multiple producers - multiple_producer_artifacts <- character(0) - for (artifact_file in artifact_nodes) { - script_producers <- edges$from[edges$to == artifact_file & - edges$from %in% nodes$file[nodes$type == "script"]] - if (length(script_producers) > 1) { - multiple_producer_artifacts <- c(multiple_producer_artifacts, artifact_file) - } - } - - # Report orphaned inputs - if (length(orphaned_inputs) > 0) { - stop("\n\033[31m[INVALID]\033[0m Pipeline validation failed\n", - "The following file_in() calls reference files that are not produced", - " by any file_out() call:\n", - paste("\033[33m -", orphaned_inputs, "\033[0m", collapse = "\n"), - "\n\n", - "Either:\n", - "1. Add a script that produces these files with file_out(), or\n", - "2. Change file_in() to external_in() if these are external files", - " provided by the user\n", - "Pipeline validation failed: ", - paste(orphaned_inputs, collapse = ", "), call. = FALSE) - } - - # Report multiple producers - if (length(multiple_producer_artifacts) > 0) { - producer_details <- character(0) - for (artifact in multiple_producer_artifacts) { - producers <- edges$from[edges$to == artifact & - edges$from %in% nodes$file[nodes$type == "script"]] - producer_details <- c(producer_details, - sprintf("\033[33m - %s\033[0m produced by: %s", - artifact, paste(producers, collapse = ", "))) - } - stop("\n\033[31m[INVALID]\033[0m Pipeline validation failed\n", - "The following artifacts have multiple producers:\n", - paste(producer_details, collapse = "\n"), - "\n\nEach artifact should have exactly one producer script.\n", - "Pipeline validation failed: ", - paste(multiple_producer_artifacts, collapse = ", "), call. = FALSE) - } - - TRUE -} - - -#' Build file nodes from graph structure and parse data -#' -#' Creates nodes for all files with types determined from parse data: -#' - Scripts: script file names -#' - Inputs: files only in inputs (external inputs) -#' - Outputs: files in outputs (includes intermediates) -#' -#' @param parse_data Parse result with scripts, inputs, outputs -#' @param edges Data frame with 'from' and 'to' columns -#' @param state_obj Optional state object from read_state() -#' @return Data frame with 'file', 'type', and 'stale' columns -#' @keywords internal -build_file_nodes <- function(parse_data, edges, state_obj = NULL) { - # Get all unique files mentioned in the graph - all_files <- unique(c(edges$from, edges$to, names(parse_data$scripts))) - - # Determine file types - scripts <- names(parse_data$scripts) - artifacts <- unique(c(parse_data$inputs, parse_data$outputs)) - externals <- parse_data$externals - - # Create types vector - file_types <- character(length(all_files)) - for (i in seq_along(all_files)) { - file <- all_files[i] - if (file %in% scripts) { - file_types[i] <- "script" - } else if (file %in% externals) { - file_types[i] <- "external" - } else if (file %in% artifacts) { - file_types[i] <- "artifact" - } else { - file_types[i] <- "unknown" - } - } - - # Create nodes data frame - all default to stale = TRUE - nodes <- data.frame( - file = all_files, - type = file_types, - stale = rep(TRUE, length(all_files)), - stringsAsFactors = FALSE - ) - - # Update staleness based on state_obj - if (!is.null(state_obj)) { - for (i in seq_len(nrow(nodes))) { - file <- nodes$file[i] - if (file %in% state_obj$file) { - nodes$stale[i] <- state_obj$stale[state_obj$file == file][1] - } - } - } - - nodes -} - - -#' Build edges between files for the new graph structure -#' -#' Creates edges directly between files: input -> script -> output -#' This creates a linear chain for each script's file dependencies. -#' -#' @param scripts_data Named list of scripts from parse()$scripts -#' @return Data frame with 'from' and 'to' columns -#' @keywords internal -build_file_edges <- function(scripts_data) { - edges <- data.frame(from = character(0), to = character(0), - stringsAsFactors = FALSE) - - for (script_name in names(scripts_data)) { - script_data <- scripts_data[[script_name]] - - # Create edges from input files to script - for (input_file in script_data$inputs) { - edges <- rbind(edges, data.frame( - from = input_file, - to = script_name, - stringsAsFactors = FALSE - )) - } - - # Create edges from external files to script - for (external_file in script_data$externals) { - edges <- rbind(edges, data.frame( - from = external_file, - to = script_name, - stringsAsFactors = FALSE - )) - } - - # Create edges from script to output files - for (output_file in script_data$outputs) { - edges <- rbind(edges, data.frame( - from = script_name, - to = output_file, - stringsAsFactors = FALSE - )) - } - } - - edges -} - -#' Detect cycles in the dependency graph using DFS -#' -#' @param graph_obj Graph object from graph() function -#' @keywords internal -detect_cycles <- function(graph_obj) { - nodes <- graph_obj$nodes$file - edges <- graph_obj$edges - - # DFS state: 0 = unvisited, 1 = visiting, 2 = visited - state <- setNames(rep(0, length(nodes)), nodes) - - # Get neighbors from edges - get_neighbors <- function(node) { - edges$to[edges$from == node] - } - - # Recursive DFS function - dfs <- function(node) { - if (state[node] == 1) { - # Back edge found - cycle detected - stop("Cycle detected in dependency graph involving node: ", node) - } - - if (state[node] == 2) { - # Already processed - return() - } - - # Mark as visiting - state[node] <<- 1 - - # Visit all neighbors - for (neighbor in get_neighbors(node)) { - dfs(neighbor) - } - - # Mark as visited - state[node] <<- 2 - } - - # Run DFS from all unvisited nodes - for (node in nodes) { - if (state[node] == 0) { - dfs(node) - } - } -} - -#' Topological sort of the dependency graph -#' -#' Returns files in topological order using Kahn's algorithm. Files will -#' appear in an order where all dependencies come before the file. -#' Only returns scripts in execution order when filtering by type. -#' -#' @param graph_obj Graph object from graph() function -#' @param scripts_only Logical. If TRUE, returns only script nodes in order -#' @return Character vector of file names in topological order -topological_sort <- function(graph_obj, scripts_only = FALSE) { - nodes <- graph_obj$nodes$file - edges <- graph_obj$edges - - if (length(nodes) == 0) { - return(character(0)) - } - - # Calculate in-degree for each node - in_degree <- setNames(rep(0, length(nodes)), nodes) - for (i in seq_len(nrow(edges))) { - to_node <- edges$to[i] - in_degree[to_node] <- in_degree[to_node] + 1 - } - - # Get neighbors from edges - get_neighbors <- function(node) { - edges$to[edges$from == node] - } - - # Initialize queue with nodes that have no incoming edges - queue <- names(in_degree)[in_degree == 0] - result <- character(0) - - while (length(queue) > 0) { - # Remove node with no incoming edges - current <- queue[1] - queue <- queue[-1] - result <- c(result, current) - - # Remove edges from current node - for (neighbor in get_neighbors(current)) { - in_degree[neighbor] <- in_degree[neighbor] - 1 - if (in_degree[neighbor] == 0) { - queue <- c(queue, neighbor) - } - } - } - - # If result doesn't contain all nodes, there was a cycle - if (length(result) != length(nodes)) { - stop("Cannot perform topological sort: graph contains cycles") - } - - # Filter to scripts only if requested - if (scripts_only) { - script_nodes <- graph_obj$nodes$file[graph_obj$nodes$type == "script"] - result <- result[result %in% script_nodes] - } - - result -} - -#' Find all descendants of a file in the dependency graph -#' -#' Returns all files that depend on the given file by following -#' the directed edges. Useful for marking files as stale when an upstream -#' dependency changes. -#' -#' @param graph_obj Graph object from graph() function -#' @param node Starting file to find descendants from -#' @param scripts_only Logical. If TRUE, returns only script descendants -#' @return Character vector of all descendant file names -find_descendants <- function(graph_obj, node, scripts_only = FALSE) { - nodes <- graph_obj$nodes$file - edges <- graph_obj$edges - - if (!node %in% nodes) { - stop("Node '", node, "' not found in graph") - } - - # Get neighbors from edges - get_neighbors <- function(n) { - edges$to[edges$from == n] - } - - visited <- character(0) - to_visit <- get_neighbors(node) - - while (length(to_visit) > 0) { - current <- to_visit[1] - to_visit <- to_visit[-1] - - if (!current %in% visited) { - visited <- c(visited, current) - # Add neighbors to visit queue - to_visit <- c(to_visit, get_neighbors(current)) - } - } - - # Filter to scripts only if requested - if (scripts_only) { - script_nodes <- graph_obj$nodes$file[graph_obj$nodes$type == "script"] - visited <- visited[visited %in% script_nodes] - } - - sort(unique(visited)) -} - -#' Propagate staleness through the dependency graph -#' -#' Implements the logic: -#' - If node is stale AND output: mark parent + descendants as stale -#' - If node is stale otherwise: mark self + descendants as stale -#' -#' @param graph_obj Graph object with nodes and edges data frames -#' @keywords internal -propagate_staleness <- function(graph_obj) { - nodes <- graph_obj$nodes - edges <- graph_obj$edges - - # Get neighbors and parents from edges - get_neighbors <- function(node) { - edges$to[edges$from == node] - } - - get_parents <- function(node) { - edges$from[edges$to == node] - } - - # DFS to mark descendants as stale - mark_descendants_stale <- function(node, visited = character(0)) { - if (node %in% visited) { - return(visited) - } - - visited <- c(visited, node) - - # Mark current node as stale - nodes$stale[nodes$file == node] <<- TRUE - - # Recursively mark all descendants - for (neighbor in get_neighbors(node)) { - visited <- mark_descendants_stale(neighbor, visited) - } - - visited - } - - # Process each node with new logic - for (i in seq_len(nrow(nodes))) { - node_name <- nodes$file[i] - node_is_stale <- nodes$stale[i] - node_type <- nodes$type[i] - - if (node_is_stale) { - if (node_type == "artifact") { - # If node is stale AND output: mark parent + descendants as stale - parents <- get_parents(node_name) - for (parent in parents) { - mark_descendants_stale(parent) - } - mark_descendants_stale(node_name) - } else { - # If node is stale otherwise: mark self + descendants as stale - mark_descendants_stale(node_name) - } - } - } - - # Update the graph object and return it - graph_obj$nodes <- nodes - graph_obj -} diff --git a/R/run.R b/R/run.R index 35f9ad9..3ff6037 100644 --- a/R/run.R +++ b/R/run.R @@ -4,7 +4,8 @@ #' scripts that are stale (have changed or have stale dependencies) for #' incremental execution. #' -#' @param verbose Logical. If TRUE (default), prints progress messages to console. +#' @param verbose Logical. If TRUE (default), prints progress messages +#' to console. #' @return Character vector of files that were created or updated #' @examples #' # Copy sample project to temp directory @@ -27,183 +28,31 @@ #' setwd(old_wd) #' unlink(temp_dir, recursive = TRUE) #' @export -#' @importFrom callr r run <- function(verbose = TRUE) { - # Parse scripts to get dependencies - pipeline_data <- parse() + # Parse dependencies to know what files will be created + deps <- parse() - # Handle empty pipeline - if (length(pipeline_data$scripts) == 0) { - if (verbose) { - message("\n[PIPELINE] \033[1;36mBakepipe Pipeline\033[0m") - message("\033[33m No scripts found in pipeline\033[0m\n") - } - return(character(0)) - } - - # Read current state - state_file <- file.path(root(), ".bakepipe.state") - state_obj <- read_state(state_file) - - # Create dependency graph with state information - graph_obj <- graph(pipeline_data, state_obj) - - # Get scripts in topological order - topo_order <- topological_sort(graph_obj, scripts_only = TRUE) - - # All files in topo_order are already scripts - script_names <- names(pipeline_data$scripts) - all_scripts <- topo_order[topo_order %in% script_names] - - # Only run stale scripts for incremental execution - # Get stale scripts from the nodes data frame - stale_scripts <- graph_obj$nodes$file[graph_obj$nodes$stale] - scripts_to_run <- all_scripts[all_scripts %in% stale_scripts] - scripts_to_skip <- all_scripts[!all_scripts %in% stale_scripts] - - # Print header - if (verbose) { - message("\n[PIPELINE] \033[1;36mBakepipe Pipeline\033[0m") - if (length(scripts_to_run) > 0) { - message(paste0( - "\033[32m Running ", length(scripts_to_run), " script", - if (length(scripts_to_run) > 1) "s" else "", "\033[0m" - )) - } - if (length(scripts_to_skip) > 0) { - message(paste0( - "\033[33m Skipping ", length(scripts_to_skip), " fresh script", - if (length(scripts_to_skip) > 1) "s" else "", "\033[0m" - )) - } - message("") - } - - # Calculate max script name width for alignment - max_width <- max(nchar(all_scripts)) - - # Print messages about scripts being skipped - if (verbose) { - for (script_name in scripts_to_skip) { - message(sprintf("\033[90m[OK] %-*s \033[2m(fresh)\033[0m", max_width, script_name)) - } - } - - # Track files created during execution - created_files <- character(0) - script_times <- numeric(0) - - # Execute each script in order - for (script_name in scripts_to_run) { - # Get script info - script_info <- pipeline_data$scripts[[script_name]] - - # Check if all input files exist - for (input_file in script_info$inputs) { - if (!file.exists(input_file)) { - stop( - "Input file '", input_file, "' required by '", script_name, - "' does not exist" - ) - } - } - - # Store output files to track what gets created - output_files <- script_info$outputs - - # Execute the script with timing - start_time <- Sys.time() - - tryCatch( - { - # Run script in isolated R process using callr - result <- callr::rscript( - script = script_name, - show = FALSE, - stderr = "2>&1" - ) - }, - error = function(e) { - # Extract the actual error message from the callr error - if (inherits(e, "callr_error") && !is.null(e$stderr)) { - # Include full stderr for debugging - stop("Error executing script '", script_name, "': ", e$stderr) - } - # Fallback to original error message - stop("Error executing script '", script_name, "': ", e$message) - } - ) - - end_time <- Sys.time() - elapsed <- as.numeric(difftime(end_time, start_time, units = "secs")) - script_times <- c(script_times, elapsed) - names(script_times)[length(script_times)] <- script_name - - # Show completion with timing - if (elapsed < 1) { - time_str <- sprintf("%.0fms", elapsed * 1000) - } else { - time_str <- sprintf("%.1fs", elapsed) - } - if (verbose) { - message(sprintf( - "\033[32m[OK] %-*s \033[2m(%s)\033[0m", - max_width, script_name, time_str - )) - } - - # Check that expected output files were created - for (output_file in output_files) { - if (file.exists(output_file)) { - created_files <- c(created_files, output_file) - } else { - warning( - "Script '", script_name, "' was expected to create '", - output_file, "' but file does not exist" - ) - } - } - } + generate_targets_file() - # Print summary - if (verbose) { - if (length(scripts_to_run) > 0 || length(created_files) > 0) { - message("\n\033[1;36m[SUMMARY]\033[0m") + # Control targets output verbosity + callr_args <- list(show = verbose, spinner = verbose) + targets::tar_make(callr_arguments = callr_args) - if (length(scripts_to_run) > 0) { - total_time <- sum(script_times) - if (total_time < 1) { - time_str <- sprintf("%.0fms", total_time * 1000) - } else { - time_str <- sprintf("%.1fs", total_time) - } - message(sprintf( - "\033[32m Executed %d script%s in %s\033[0m", - length(scripts_to_run), - if (length(scripts_to_run) > 1) "s" else "", - time_str - )) - } + # Get which targets ran in this execution + progress <- targets::tar_progress(fields = "progress") + ran_targets <- progress$name[progress$progress == "completed"] - if (length(created_files) > 0) { - message(sprintf( - "\033[36m Created/updated %d file%s:\033[0m", - length(created_files), - if (length(created_files) > 1) "s" else "" - )) - for (file in sort(unique(created_files))) { - message(sprintf("\033[2m - %s\033[0m", file)) - } - } - message("") - } else { - message("\n\033[32m[OK] All scripts are up to date!\033[0m\n") + # Map back to output files from the scripts that ran + output_files <- character(0) + for (script_name in names(deps$scripts)) { + output_target_name <- path_to_target_name(script_name, "output") + if (output_target_name %in% ran_targets) { + output_files <- c(output_files, deps$scripts[[script_name]]$outputs) } } - # Update state file after execution - write_state(state_file, pipeline_data) + # Remove duplicates if any + output_files <- unique(output_files) - # Return unique list of created files - unique(created_files) + invisible(output_files) } diff --git a/R/scripts.R b/R/scripts.R index d4b6efa..de2324f 100644 --- a/R/scripts.R +++ b/R/scripts.R @@ -16,8 +16,8 @@ scripts <- function() { ignore.case = TRUE ) - # Filter out _bakepipe.R file - r_files <- r_files[!grepl("_bakepipe\\.R$", r_files, ignore.case = TRUE)] + # Filter out _bakepipe.R and _targets.R files, and renv + r_files <- r_files[!grepl("_bakepipe\\.R$|_targets\\.R$|renv/", r_files, ignore.case = TRUE)] # Return normalized paths normalizePath(r_files) diff --git a/R/state.R b/R/state.R deleted file mode 100644 index ef43570..0000000 --- a/R/state.R +++ /dev/null @@ -1,115 +0,0 @@ -#' Read pipeline state from disk -#' -#' Reads the .bakepipe.state file and computes current checksums to -#' determine which files are stale. A file is considered stale if its -#' current checksum differs from the stored checksum. -#' -#' @param state_file Path to the state file (typically ".bakepipe.state") -#' @return Data frame with columns 'file' and 'stale' (logical) -#' @importFrom utils read.csv write.csv -read_state <- function(state_file) { - # Initialize empty state if file doesn't exist - if (!file.exists(state_file)) { - return(data.frame( - file = character(0), - stale = logical(0), - stringsAsFactors = FALSE - )) - } - - # Read existing state file - state_data <- utils::read.csv(state_file, stringsAsFactors = FALSE) - - # Get current checksums for all files in state - current_checksums <- character(0) - for (file_path in state_data$file) { - if (file.exists(file_path)) { - current_checksums[file_path] <- compute_file_checksum(file_path) - } else { - current_checksums[file_path] <- NA_character_ - } - } - - # Determine which files are stale based on checksum comparison - stale <- logical(nrow(state_data)) - for (i in seq_len(nrow(state_data))) { - file_path <- state_data$file[i] - stored_checksum <- state_data$checksum[i] - current_checksum <- current_checksums[file_path] - - # File is stale if checksum differs or file is missing - stale[i] <- is.na(current_checksum) || current_checksum != stored_checksum - } - - # Return data frame with file and stale columns - data.frame( - file = state_data$file, - stale = stale, - stringsAsFactors = FALSE - ) -} - -#' Write pipeline state to disk -#' -#' Writes the current state of all files in the pipeline to a CSV file. -#' This includes scripts and all their input/output files with their -#' current checksums and timestamps. -#' -#' @param state_file Path to the state file to write -#' (typically ".bakepipe.state") -#' @param parse_data List from parse() function with 'scripts', 'inputs', 'outputs' -write_state <- function(state_file, parse_data) { - # Collect all unique files from parse_data - all_files <- character(0) - - # Add script names - all_files <- c(all_files, names(parse_data$scripts)) - - # Add all inputs, outputs, and externals - all_files <- c(all_files, parse_data$inputs) - all_files <- c(all_files, parse_data$outputs) - all_files <- c(all_files, parse_data$externals) - - all_files <- unique(all_files) - - # Create state data frame - state_data <- data.frame( - file = all_files, - checksum = character(length(all_files)), - last_modified = character(length(all_files)), - stringsAsFactors = FALSE - ) - - # Compute checksums and timestamps for existing files - for (i in seq_len(nrow(state_data))) { - file_path <- state_data$file[i] - - if (file.exists(file_path)) { - state_data$checksum[i] <- compute_file_checksum(file_path) - file_info <- file.info(file_path) - state_data$last_modified[i] <- as.character(file_info$mtime) - } else { - # For missing files, use NA checksum and current timestamp - state_data$checksum[i] <- NA_character_ - state_data$last_modified[i] <- as.character(Sys.time()) - } - } - - # Write to CSV file - utils::write.csv(state_data, state_file, row.names = FALSE) -} - -#' Compute MD5 checksum for a file -#' -#' @param file_path Path to the file -#' @return Character string containing the MD5 checksum -#' @keywords internal -compute_file_checksum <- function(file_path) { - if (!file.exists(file_path)) { - return(NA_character_) - } - - # Use tools::md5sum for consistency with base R - checksum <- tools::md5sum(file_path) - as.character(checksum) -} diff --git a/R/status.R b/R/status.R index b95497b..6a42669 100644 --- a/R/status.R +++ b/R/status.R @@ -5,6 +5,7 @@ #' @param verbose Logical. If TRUE (default), prints status information to console. #' @return NULL (invisibly). This function is called for its side effect of #' displaying pipeline status information to the console. +#' @importFrom stats setNames #' @examples #' # Copy sample project to temp directory #' temp_dir <- tempfile() @@ -27,77 +28,100 @@ #' unlink(temp_dir, recursive = TRUE) #' @export status <- function(verbose = TRUE) { - # Parse the pipeline to get script dependencies pipeline_data <- parse() - # Handle empty pipeline if (length(pipeline_data$scripts) == 0) { if (verbose) { - message("\n[STATUS] \033[1;36mBakepipe Status\033[0m") - message("\033[33m No scripts found in pipeline\033[0m\n") + message("No scripts found") } return(invisible(NULL)) } - # Display header and Scripts table with state information + generate_targets_file() + if (verbose) { - message("\n[STATUS] \033[1;36mBakepipe Status\033[0m") - display_scripts_table(pipeline_data) + display_scripts_table_targets(pipeline_data) } invisible(NULL) } -#' Display the scripts table +#' Display the scripts table using targets backend #' @param pipeline_data Parsed pipeline data -display_scripts_table <- function(pipeline_data) { - # Read state information - state_obj <- read_state(file.path(root(), ".bakepipe.state")) +display_scripts_table_targets <- function(pipeline_data) { + # Print header + message("Bakepipe Status") + message("") + + # Get outdated targets from targets package + outdated <- tryCatch( + { + targets::tar_outdated(callr_function = NULL) + }, + error = function(e) { + # If targets hasn't run yet, all are outdated + character(0) + } + ) - # Create graph with state information - graph_obj <- graph(pipeline_data, state_obj) - topo_order <- topological_sort(graph_obj, scripts_only = TRUE) + # Get script names in order they appear in pipeline_data + script_names <- names(pipeline_data$scripts) + + # Create a map from output target names back to script names + target_to_script <- setNames(script_names, sapply(script_names, function(s) { + path_to_target_name(s, "output") + })) - # Scripts are already in the correct order - scripts <- topo_order + # Determine state for each script based on output target + state_vec <- character(length(script_names)) + for (i in seq_along(script_names)) { + script <- script_names[i] + # Convert script name to output target name + target_name <- path_to_target_name(script, "output") - # Determine state for each script - state_list <- lapply(scripts, function(script) { - script_stale <- graph_obj$nodes$stale[graph_obj$nodes$file == script] - if (length(script_stale) > 0 && script_stale) { - "stale" + # Check if this output target is outdated + if (target_name %in% outdated) { + state_vec[i] <- "stale" } else { - "fresh" + state_vec[i] <- "fresh" } - }) + } - state_vec <- unlist(state_list) - # Count scripts by state fresh_count <- sum(state_vec == "fresh") stale_count <- sum(state_vec == "stale") - + # Display summary - message(paste0("\033[32m ", fresh_count, " fresh script", if(fresh_count != 1) "s" else "", "\033[0m"), appendLF = FALSE) if (stale_count > 0) { - message(paste0(" - \033[33m", stale_count, " stale script", if(stale_count != 1) "s" else "", "\033[0m"), appendLF = FALSE) + message(sprintf("\033[32m%d fresh\033[0m, \033[33m%d stale\033[0m", fresh_count, stale_count)) + } else { + message(sprintf("\033[32m✓ %d script%s up to date\033[0m", fresh_count, if (fresh_count != 1) "s" else "")) } - message("\n") - - # Calculate max script name width for alignment - max_width <- max(nchar(scripts)) + message("") - # Display each script with status indicator - for (i in seq_along(scripts)) { - script <- scripts[i] + # Display each script with status + for (i in seq_along(script_names)) { + script <- script_names[i] state <- state_vec[i] + script_info <- pipeline_data$scripts[[script]] if (state == "fresh") { - message(sprintf("\033[90m[OK] %-*s \033[2m(fresh)\033[0m", max_width, script)) + message(sprintf("\033[32m✓\033[0m %s \033[32m(fresh)\033[0m", script)) } else { - message(sprintf("\033[33m[!] %-*s \033[2m(stale)\033[0m", max_width, script)) + message(sprintf("\033[33m!\033[0m %s \033[33m(stale)\033[0m", script)) + } + + # Show inputs, externals, outputs on separate lines if they exist + if (length(script_info$inputs) > 0) { + message(sprintf(" inputs: %s", paste(script_info$inputs, collapse = ", "))) + } + + if (length(script_info$externals) > 0) { + message(sprintf(" externals: %s", paste(script_info$externals, collapse = ", "))) + } + + if (length(script_info$outputs) > 0) { + message(sprintf(" outputs: %s", paste(script_info$outputs, collapse = ", "))) } } - - message("") } diff --git a/README.md b/README.md index 94ad0d8..28efd19 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Bakepipe is an R library that turns your script-based workflows into reproducible pipelines. It's designed for scientists and analysts who use R and prefer to keep their workflows in scripts, but need better management of file dependencies. +Under the hood, Bakepipe uses [targets](https://docs.ropensci.org/targets/) to manage your pipeline — much like how [jj](https://github.com/martinvonz/jj) is built on git. Bakepipe provides a simpler, more accessible interface to pipeline management without requiring you to refactor your scripts into functions. + **Key features:** - **Automatic dependency detection** - determines script execution order from `file_in()` and `file_out()` calls @@ -171,13 +173,13 @@ Bakepipe determines the correct execution order through static analysis of your ### Are outputs cached? -Yes! Bakepipe automatically performs incremental execution by tracking file checksums in a `.bakepipe.state` file. After the first run, Bakepipe will only re-run scripts that are "stale" - meaning either: +Yes! Bakepipe automatically performs incremental execution using the targets backend to track file metadata. After the first run, Bakepipe will only re-run scripts that are "stale" - meaning either: - The script itself has been modified - Any of the script's input files have been modified - Any upstream dependencies have been modified -This makes subsequent runs much faster, as only the necessary scripts are executed. Fresh scripts are skipped with a visual indicator showing they're up to date. +This makes subsequent runs much faster, as only the necessary scripts are executed. Fresh scripts are skipped with a visual indicator showing they're up to date. The targets metadata is stored in the `_targets/` directory (which you should add to `.gitignore`). ### How does Bakepipe compare to other pipeline tools? diff --git a/inst/extdata/sample-project/_targets.R b/inst/extdata/sample-project/_targets.R new file mode 100644 index 0000000..12f1393 --- /dev/null +++ b/inst/extdata/sample-project/_targets.R @@ -0,0 +1,37 @@ +library(targets) + +list( + tar_target(input_csv, "input.csv", format = "file"), + tar_target(script_01_process_r, "01_process.R", format = "file"), + tar_target( + output_01_process_r, + { + script_01_process_r + input_csv + callr::r( + func = function(script_path) { + source(script_path, local = TRUE) + }, + args = list(script_path = "01_process.R") + ) + c("processed.csv") + }, + format = "file" + ), + tar_target(script_02_summarize_r, "02_summarize.R", format = "file"), + tar_target( + output_02_summarize_r, + { + script_02_summarize_r + output_01_process_r + callr::r( + func = function(script_path) { + source(script_path, local = TRUE) + }, + args = list(script_path = "02_summarize.R") + ) + c("summary.csv") + }, + format = "file" + ) +) diff --git a/man/build_file_edges.Rd b/man/build_file_edges.Rd deleted file mode 100644 index 6d08633..0000000 --- a/man/build_file_edges.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{build_file_edges} -\alias{build_file_edges} -\title{Build edges between files for the new graph structure} -\usage{ -build_file_edges(scripts_data) -} -\arguments{ -\item{scripts_data}{Named list of scripts from parse()$scripts} -} -\value{ -Data frame with 'from' and 'to' columns -} -\description{ -Creates edges directly between files: input -> script -> output -This creates a linear chain for each script's file dependencies. -} -\keyword{internal} diff --git a/man/build_file_nodes.Rd b/man/build_file_nodes.Rd deleted file mode 100644 index 5b3ecad..0000000 --- a/man/build_file_nodes.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{build_file_nodes} -\alias{build_file_nodes} -\title{Build file nodes from graph structure and parse data} -\usage{ -build_file_nodes(parse_data, edges, state_obj = NULL) -} -\arguments{ -\item{parse_data}{Parse result with scripts, inputs, outputs} - -\item{edges}{Data frame with 'from' and 'to' columns} - -\item{state_obj}{Optional state object from read_state()} -} -\value{ -Data frame with 'file', 'type', and 'stale' columns -} -\description{ -Creates nodes for all files with types determined from parse data: -\itemize{ -\item Scripts: script file names -\item Inputs: files only in inputs (external inputs) -\item Outputs: files in outputs (includes intermediates) -} -} -\keyword{internal} diff --git a/man/clean.Rd b/man/clean.Rd index d96948f..19abec7 100644 --- a/man/clean.Rd +++ b/man/clean.Rd @@ -7,7 +7,8 @@ clean(verbose = TRUE) } \arguments{ -\item{verbose}{Logical. If TRUE (default), prints progress messages to console.} +\item{verbose}{Logical. If TRUE (default), prints progress messages +to console.} } \value{ Character vector of file paths that were actually removed @@ -29,7 +30,7 @@ old_wd <- getwd() setwd(file.path(temp_dir, "sample-project")) # Run the pipeline first to create output files -run(verbose = FALSE) +run() # Now clean up the generated files removed_files <- clean() diff --git a/man/compute_file_checksum.Rd b/man/compute_file_checksum.Rd deleted file mode 100644 index 5937357..0000000 --- a/man/compute_file_checksum.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/state.R -\name{compute_file_checksum} -\alias{compute_file_checksum} -\title{Compute MD5 checksum for a file} -\usage{ -compute_file_checksum(file_path) -} -\arguments{ -\item{file_path}{Path to the file} -} -\value{ -Character string containing the MD5 checksum -} -\description{ -Compute MD5 checksum for a file -} -\keyword{internal} diff --git a/man/detect_cycles.Rd b/man/detect_cycles.Rd deleted file mode 100644 index 448ddd7..0000000 --- a/man/detect_cycles.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{detect_cycles} -\alias{detect_cycles} -\title{Detect cycles in the dependency graph using DFS} -\usage{ -detect_cycles(graph_obj) -} -\arguments{ -\item{graph_obj}{Graph object from graph() function} -} -\description{ -Detect cycles in the dependency graph using DFS -} -\keyword{internal} diff --git a/man/display_scripts_table.Rd b/man/display_scripts_table.Rd deleted file mode 100644 index 20a13b0..0000000 --- a/man/display_scripts_table.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/status.R -\name{display_scripts_table} -\alias{display_scripts_table} -\title{Display the scripts table} -\usage{ -display_scripts_table(pipeline_data) -} -\arguments{ -\item{pipeline_data}{Parsed pipeline data} -} -\description{ -Display the scripts table -} diff --git a/man/display_scripts_table_targets.Rd b/man/display_scripts_table_targets.Rd new file mode 100644 index 0000000..340dbf8 --- /dev/null +++ b/man/display_scripts_table_targets.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/status.R +\name{display_scripts_table_targets} +\alias{display_scripts_table_targets} +\title{Display the scripts table using targets backend} +\usage{ +display_scripts_table_targets(pipeline_data) +} +\arguments{ +\item{pipeline_data}{Parsed pipeline data} +} +\description{ +Display the scripts table using targets backend +} diff --git a/man/find_descendants.Rd b/man/find_descendants.Rd deleted file mode 100644 index b5d7a6a..0000000 --- a/man/find_descendants.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{find_descendants} -\alias{find_descendants} -\title{Find all descendants of a file in the dependency graph} -\usage{ -find_descendants(graph_obj, node, scripts_only = FALSE) -} -\arguments{ -\item{graph_obj}{Graph object from graph() function} - -\item{node}{Starting file to find descendants from} - -\item{scripts_only}{Logical. If TRUE, returns only script descendants} -} -\value{ -Character vector of all descendant file names -} -\description{ -Returns all files that depend on the given file by following -the directed edges. Useful for marking files as stale when an upstream -dependency changes. -} diff --git a/man/generate_targets_file.Rd b/man/generate_targets_file.Rd new file mode 100644 index 0000000..07a7d72 --- /dev/null +++ b/man/generate_targets_file.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/generate_targets_file.R +\name{generate_targets_file} +\alias{generate_targets_file} +\title{Generate _targets.R file from bakepipe scripts} +\usage{ +generate_targets_file() +} +\value{ +Invisibly returns the path to the generated _targets.R file +} +\description{ +Parses all R scripts in the project and generates a _targets.R file +that defines targets for the targets package. This allows bakepipe +to use targets as a backend for pipeline execution. +} diff --git a/man/graph.Rd b/man/graph.Rd deleted file mode 100644 index 217f1f7..0000000 --- a/man/graph.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{graph} -\alias{graph} -\title{Create dependency graph from parsed script data} -\usage{ -graph(parse_data, state_obj = NULL) -} -\arguments{ -\item{parse_data}{List from parse() function with 'scripts', 'inputs', 'outputs'} - -\item{state_obj}{Optional. Data frame from read_state() function with 'file' -and 'stale' columns. If provided, will mark nodes as stale/fresh.} -} -\value{ -List containing: -\itemize{ -\item{nodes: Data frame with 'file', 'type', and 'stale' columns} -\item{edges: Data frame with 'from' and 'to' columns} -} -} -\description{ -Builds a Directed Acyclic Graph (DAG) where all files are nodes. -Node types are determined from parse data: -\itemize{ -\item Inputs: files only in parse_data$inputs (external inputs) -\item Outputs: files in parse_data$outputs (includes intermediates) -\item Scripts: script file names -} -} diff --git a/man/path_to_target_name.Rd b/man/path_to_target_name.Rd new file mode 100644 index 0000000..d9cf390 --- /dev/null +++ b/man/path_to_target_name.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/generate_targets_file.R +\name{path_to_target_name} +\alias{path_to_target_name} +\title{Convert file path to valid R target name} +\usage{ +path_to_target_name(path, prefix = "") +} +\arguments{ +\item{path}{File path} + +\item{prefix}{Prefix to add (e.g., "script", "run", or "")} +} +\value{ +Valid R identifier +} +\description{ +Converts a file path to a valid R identifier for use as a target name. +Special characters are replaced with underscores. +} +\keyword{internal} diff --git a/man/propagate_staleness.Rd b/man/propagate_staleness.Rd deleted file mode 100644 index cb35735..0000000 --- a/man/propagate_staleness.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{propagate_staleness} -\alias{propagate_staleness} -\title{Propagate staleness through the dependency graph} -\usage{ -propagate_staleness(graph_obj) -} -\arguments{ -\item{graph_obj}{Graph object with nodes and edges data frames} -} -\description{ -Implements the logic: -\itemize{ -\item If node is stale AND output: mark parent + descendants as stale -\item If node is stale otherwise: mark self + descendants as stale -} -} -\keyword{internal} diff --git a/man/read_state.Rd b/man/read_state.Rd deleted file mode 100644 index fa8e568..0000000 --- a/man/read_state.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/state.R -\name{read_state} -\alias{read_state} -\title{Read pipeline state from disk} -\usage{ -read_state(state_file) -} -\arguments{ -\item{state_file}{Path to the state file (typically ".bakepipe.state")} -} -\value{ -Data frame with columns 'file' and 'stale' (logical) -} -\description{ -Reads the .bakepipe.state file and computes current checksums to -determine which files are stale. A file is considered stale if its -current checksum differs from the stored checksum. -} diff --git a/man/run.Rd b/man/run.Rd index a68320a..89d99a2 100644 --- a/man/run.Rd +++ b/man/run.Rd @@ -7,7 +7,8 @@ run(verbose = TRUE) } \arguments{ -\item{verbose}{Logical. If TRUE (default), prints progress messages to console.} +\item{verbose}{Logical. If TRUE (default), prints progress messages +to console.} } \value{ Character vector of files that were created or updated diff --git a/man/topological_sort.Rd b/man/topological_sort.Rd deleted file mode 100644 index ed3aa4b..0000000 --- a/man/topological_sort.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{topological_sort} -\alias{topological_sort} -\title{Topological sort of the dependency graph} -\usage{ -topological_sort(graph_obj, scripts_only = FALSE) -} -\arguments{ -\item{graph_obj}{Graph object from graph() function} - -\item{scripts_only}{Logical. If TRUE, returns only script nodes in order} -} -\value{ -Character vector of file names in topological order -} -\description{ -Returns files in topological order using Kahn's algorithm. Files will -appear in an order where all dependencies come before the file. -Only returns scripts in execution order when filtering by type. -} diff --git a/man/validate_artifact_producers.Rd b/man/validate_artifact_producers.Rd deleted file mode 100644 index e914885..0000000 --- a/man/validate_artifact_producers.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/graph.R -\name{validate_artifact_producers} -\alias{validate_artifact_producers} -\title{Validate that each artifact has exactly one producer} -\usage{ -validate_artifact_producers(graph_obj, parse_data) -} -\arguments{ -\item{graph_obj}{Graph object from graph() function} - -\item{parse_data}{Parse result with scripts, inputs, outputs} -} -\description{ -Ensures that every artifact (file referenced by file_in()) has exactly one producer. -This means each artifact should have exactly one script that produces it - not zero -(orphaned) and not more than one (multiple producers). -} -\keyword{internal} diff --git a/man/write_state.Rd b/man/write_state.Rd deleted file mode 100644 index e3b0fc6..0000000 --- a/man/write_state.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/state.R -\name{write_state} -\alias{write_state} -\title{Write pipeline state to disk} -\usage{ -write_state(state_file, parse_data) -} -\arguments{ -\item{state_file}{Path to the state file to write -(typically ".bakepipe.state")} - -\item{parse_data}{List from parse() function with 'scripts', 'inputs', 'outputs'} -} -\description{ -Writes the current state of all files in the pipeline to a CSV file. -This includes scripts and all their input/output files with their -current checksums and timestamps. -} diff --git a/plans/001-targets-frontend.md b/plans/001-targets-frontend.md new file mode 100644 index 0000000..4a5277d --- /dev/null +++ b/plans/001-targets-frontend.md @@ -0,0 +1,70 @@ +# Targets frontend + +I think the Bakepipe API is really nice, but I don't think it is sensible +to reinvent the wheel when it comes to defining targets and their dependencies. Could we make Bakepipe a simple frontend for +the targets package instead? + +## Approach + +**Keep Bakepipe's API**, use targets as backend: +- Users still write scripts with `file_in()`, `file_out()`, `external_in()` +- Generate `_targets.R` from parsed dependencies +- Wrap targets functions in `bakepipe::run()`, `status()`, `clean()` + +## Generated Target Structure + +Minimal targets: one per script + external inputs + +```r +# Script 1: 01_process.R +# data <- read.csv(external_in("input.csv")) +# write.csv(subset_a, file_out("output_a.csv")) +# write.csv(subset_b, file_out("output_b.csv")) + +tar_target(input_csv, "input.csv", format = "file") + +tar_target( + output_01_process, + { + input_csv + source("01_process.R") + c("output_a.csv", "output_b.csv") + }, + format = "file" +) + +# Script 2: 02_analyze.R +# data <- read.csv(file_in("output_a.csv")) +# write.csv(results, file_out("analysis.csv")) + +tar_target( + output_02_analyze, + { + output_01_process + source("02_analyze.R") + c("analysis.csv") + }, + format = "file" +) +``` + +**Key design**: +- One target per script, returns vector of output files +- Dependencies listed explicitly at top of target +- No separate script/run/output targets +- Manual edits to any output file invalidate the target + +## Implementation + +- New `generate_targets_file()` - uses existing `parse()` to create `_targets.R` +- Modify `run()` → call `targets::tar_make()` +- Modify `status()` → call `targets::tar_outdated()` + pretty print +- Modify `clean()` → call `targets::tar_destroy()` +- Regenerate `_targets.R` before each operation + +## Benefits + +- Leverage targets' mature dependency tracking +- Reduce ~1100 lines to ~200 line translation layer +- Gain targets ecosystem (parallel execution, visualization, etc.) +- Users can graduate to native targets if needed diff --git a/plans/002-targets-implementation.md b/plans/002-targets-implementation.md new file mode 100644 index 0000000..078cd4f --- /dev/null +++ b/plans/002-targets-implementation.md @@ -0,0 +1,92 @@ +# Implementation Plan: Targets Backend with File Vector Returns + +**Updated Design**: Each script target returns a vector of output files, so manual edits to any output invalidate the target. + +## Phase 1: Core Target Generation + +### 1.1 Enhance `parse()` to track output files +- Modify existing `parse_script()` to collect all `file_out()` calls +- Store as `outputs` vector in dependency structure +- Test: Verify `parse()` correctly extracts multiple outputs + +### 1.2 Create `generate_targets_file()` +- Input: Parsed dependencies +- Output: `_targets.R` with vector-returning targets +- For each script, generate: + ```r + tar_target( + output_, + { + + source("