diff --git a/.python-version b/.python-version index e4fba21..6324d40 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12 +3.14 diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/intermediates/.gitignore b/data/intermediates/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/intermediates/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/outputs/.gitignore b/data/outputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/outputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/notebooks/checks.qmd b/notebooks/checks.qmd index 2b7933e..3229ed9 100644 --- a/notebooks/checks.qmd +++ b/notebooks/checks.qmd @@ -61,6 +61,18 @@ from enum import Enum from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -81,7 +93,18 @@ PRINCIPLES = { @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -111,6 +134,19 @@ Now, let's define a few checks: ```{python} def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -127,9 +163,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. - Pass/Fail, no exceptions. + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -147,6 +191,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -187,6 +245,20 @@ started writing code: #| sorting-hat: keep def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -241,6 +313,20 @@ of things like READMEs, notebooks, or markdown files: ```{python} def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -286,6 +372,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", @@ -337,4 +437,3 @@ def check_project_structure(directory: Path) -> CheckResult: In the next module, we define the ManifestChecker class, which will run all of these checks and more, and return a structured report that can be displayed in `stagecoach inspect`. - diff --git a/notebooks/cli.qmd b/notebooks/cli.qmd index e381cd7..6082566 100644 --- a/notebooks/cli.qmd +++ b/notebooks/cli.qmd @@ -30,17 +30,18 @@ Once the CLI is defined as "hailing" the stagecoach, stagecoach will run `issue_manifest` to create the manifest, and then proceed with the rest of the stagecoach. +Here's the code for the CLI. First, hailing the stagecoach: + ```{python} #| sorting-hat: keep from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -50,6 +51,16 @@ app = typer.Typer( ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -66,6 +77,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -74,6 +90,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -87,6 +112,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: @@ -94,6 +120,8 @@ def hail( raise typer.Exit(code=1) ``` +Inspecting the manifest: + ```{python} @app.command() def inspect( @@ -116,6 +144,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -137,12 +172,36 @@ def inspect( raise typer.Exit(code=1) ``` +And, staging: + ```{python} @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + manifest_path=manifest_path + ).stage() + if not staged: + raise typer.Exit(code=1) + ``` ```{python} @@ -150,4 +209,3 @@ def stage(): if __name__ == "__main__": app() ``` - diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index a71cb61..295f044 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -27,11 +27,13 @@ This manifest is a JSON file that users fill in with their credentials and desired data from the Frontier Gold Mine. Once the user returns the manifest, and the `Stagecoach` recognizes that the user wants access to data via `globus`, -and passes this information to the customs handler. +`Dataverse`, or from the Gold Mine, the `Stagecoach` will call the +`customs` handler and pass this information along. The customs handler then validates the credentials to ensure that the user has the correct permissions to access the data. This validation process -uses the `Globus SDK` to check the credentials against the -Globus service. If the credentials are valid, the customs handler +uses the `Globus SDK`, `Dataverse API`, or a few simple local +file permission checks to verify the credentials against the +data provider service. If the credentials are valid, the customs handler returns a success message, allowing the `Stagecoach` to proceed with the data delivery. If the credentials are invalid, the handler returns an error message, and the `Stagecoach` can @@ -40,7 +42,7 @@ credentials in the manifest. This process ensures that only authorized users can access the data, maintaining the security of The Frontier's Gold Mine resources. -## Tinkering with the SDK +## Globus SDK Authentication and Validation Globus SDK in Python works as below. Once you've set up an App in the [Developer panel](https://globus-sdk-python.readthedocs.io/en/stable/user_guide/getting_started/register_app.html) of the Globus website, @@ -185,121 +187,288 @@ the globus section of a stagecoach manifest. The manifest will have the following shape: -```yaml -globus: - use_globus: true - globus_source_endpoint: fancy-uuid - globus_source_path: /path/on/source/endpoint - globus_destination_endpoint: fancy-uuid - globus_destination_path: /path/on/destination/endpoint +```{python} +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) +``` + +In the customs manager, we will have to do two things: +authenticate with the Globus client, and send the actual +data. After wrestling with this for a few tries, and bouncing +design ideas off of chatgpt and jupyter notebooks, I've settled on +adding a context helper to do the UserApp creation: + +```{python} +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client ``` -We'll need to validate that the user has the kind of access -necessary. The return will be a dataclass that the stagecoach -can accept the requisite transfer object from and execute. +This will allow us to create transfer clients with the `with` +statement, allowing us to iterate over the items +in the Globus request individually, granting each of +them a unique request and clearance: ```{python} -#| sorting-hat: keep -from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None +class Clearance: + """ + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. + """ + + source: str cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +def check_globus_clearance( + globus_info: dict + ) -> Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. """ try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) + +``` + +If the checks pass, we can move on to building the transfer +logic. This is built mostly on what we learned above: + +```{python} +def build_globus_transfer( + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", + ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label=label, + ) + + for item in globus_info["items"]: - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + + transfer.add_item( + str(source_path), + str(destination_path), + recursive=item.get("recursive", True), + ) + + return transfer +``` + +Now with that, the customs manager can first +check for clearance, and additionally build the transfer +behind the scenes. + +## Gold Mine + +Accessing the Gold Mine can only be done from FASRC, for now. +It will implement a simple check for whether the user has access to +the specified path. This will reuse the `Clearance` class +from earlier: + +```{python} +import glob +import os +from pathlib import Path + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) ``` +## Dataverse + +🚧 Not yet implemented 🚧 + +## Conclusion When that sends back a clearance object with `cleared=True`, the `Stagecoach` can proceed with the transfer using the provided `transfer_client`. diff --git a/notebooks/issue_manifest.qmd b/notebooks/issue_manifest.qmd index 2c4101f..7ea7926 100644 --- a/notebooks/issue_manifest.qmd +++ b/notebooks/issue_manifest.qmd @@ -83,9 +83,23 @@ from rich.console import Console from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -93,7 +107,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") ``` We can see this function in action below: @@ -120,75 +134,105 @@ to the user's specified location. # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, # "stagecoach_version": 0.1, # # critical: we must know where the frontier starts # "root": "/n/holylabs/cgolden_lab/Lab/frontier" # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" -# }, -# # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None -# } +# "governance": "governance", # }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA -# "citizen": { -# # your name must match your name in town +# "citizen": { # "name": None, - -# # email may be used in future for authentications -# "email": None +# "email": None, # }, # "project": { -# # this must match the name of your project repository # "project_name": None, - -# # must match above # "project_working_dir": str(here()), - -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus +# "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) ``` If a citizen can pass the citizenship check, then @@ -206,7 +250,14 @@ from typing import Any from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -231,8 +282,19 @@ import questionary from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. - """Prompt the user and fill in manifest fields.""" + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -257,30 +319,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest ``` @@ -300,7 +369,6 @@ Next, writing the manifest should be straight forward: ```{python} #| sorting-hat: keep -import questionary from pathlib import Path from rich.console import Console @@ -310,7 +378,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -324,12 +410,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") ``` To test: @@ -354,7 +440,6 @@ Looks great! We now have the pieces to issue a manifest: #| sorting-hat: keep from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -368,36 +453,38 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") ``` @@ -407,4 +494,3 @@ to ensure they have correct access to the Gold Mine. Next, check out the CLI module to see how we "hail," a stagecoach. - diff --git a/notebooks/manifest_checker.qmd b/notebooks/manifest_checker.qmd index 60f96f0..7953c9c 100644 --- a/notebooks/manifest_checker.qmd +++ b/notebooks/manifest_checker.qmd @@ -31,10 +31,23 @@ from stagecoach.checks import * class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -43,7 +56,12 @@ class ManifestChecker: def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -58,11 +76,13 @@ class ManifestChecker: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() @@ -88,4 +108,3 @@ else: ``` - diff --git a/notebooks/outpost.qmd b/notebooks/outpost.qmd new file mode 100644 index 0000000..971240a --- /dev/null +++ b/notebooks/outpost.qmd @@ -0,0 +1,101 @@ +--- +title: "Outpost" +subtitle: "Scaffold a Fake Frontier Directory to Run `stagecoach` Outside of FASRC" +filters: + - sorting-hat + - ripper +extensions: + sorting-hat: + keep: + - python + verbose: true + ripper: + include-yaml: false + output-name: "outpost" + output-dir: "../src/stagecoach" + script-links-position: "none" +params: + export: true +format: gfm +--- + +We have a problem. The `stagecoach` tool is designed to run on the FASRC cluster, and it +relies on the `sheriff` to run a citizenship test (i.e. determine that you +are bought in to use The Frontier). This means that if you want to +run `stagecoach` outside of FASRC, you need to have those documents, +which will be a barrier for many users. + +Here, we introduce the `outpost` module, which will temporarily +scaffold a fake Frontier directory that the sheriff will +accept as valid. This +will allow users to run `stagecoach` on their local machines, +without needing to have the actual citizenship documents. + +We're going ot implement this as a CLI flag, `--outpost`, +which will trigger the creation of this fake directory. +Additionally, ChatGPT recommends using the `contextlib` +module to create a context manager that will handle the +setup and teardown of this fake directory, ensuring that it is cleaned up after use. + +The `outpost` module will have a `create_outpost` function +context. + +```{python} +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier +``` + + diff --git a/notebooks/stage.qmd b/notebooks/stage.qmd new file mode 100644 index 0000000..c3ce9a7 --- /dev/null +++ b/notebooks/stage.qmd @@ -0,0 +1,328 @@ +--- +title: "Stage Data" +subtitle: "Staging: Delivering the Data to the User" + +filters: + - sorting-hat + - ripper +extensions: + sorting-hat: + keep: + - python + ripper: + include-yaml: false + output-name: "stage" + output-dir: "../src/stagecoach" +format: gfm +--- + +Finally, it's time to deliver the user's data to them. +In this module, we define several staging methods that +the stagecoach can use. + +There are a couple of things that will need to happen to +make this work. One of them will be creating the +directories for data usage: + +```{python} +import os +import glob +from pathlib import Path +from stagecoach.ui import info, error, success, warning +from rich.console import Console +``` + +```{python} +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") + + success(console, "Stage directories are set up.") + +``` + +Let's create a quick `StageResult` class to represent +the results of staging operations, so we can report them +nicely in the UI and potentially use them in the future for more detailed reporting: + +```{python} +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) +``` + +Next, we can work on fetching the data from Globus. Fortunately, much of the +work has been done for us in the `customs` module, so +we just need to call the appropriate functions. + +```{python} +from stagecoach.customs import ( + + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> StageResult: + """ + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) + +``` + +Next we implement symlink staging via FASRC: + +```{python} +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) +``` + +That should do it for now! We will implement the dataverse staging +later on. diff --git a/notebooks/stagecoach.qmd b/notebooks/stagecoach.qmd index 8b434d1..f03f0c6 100644 --- a/notebooks/stagecoach.qmd +++ b/notebooks/stagecoach.qmd @@ -69,13 +69,15 @@ the data to the user. #| sorting-hat: keep import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -91,7 +93,16 @@ from stagecoach.ui import ( class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -121,30 +132,86 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: - """Create a StageCoach manifest.""" + outpost: bool = False, + ) -> bool: + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path banner(console, "🚂 Hailing the Stagecoach...") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") + + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) + + outpost = create_outpost() if outpost else nullcontext() - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) + + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: - """Inspect a StageCoach manifest.""" + ) -> bool: + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -152,16 +219,38 @@ class StageCoach: checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -170,30 +259,112 @@ class StageCoach: "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + failure_panel(console, "⚠️ Staging failed. Please try again.") + return False + ``` Click on each section below to learn how stagecoach works in more detail, and to see examples of how to use it. 1. [Manifest File](#manifest-file) - diff --git a/notebooks/ui.qmd b/notebooks/ui.qmd index 05ccff3..4d552d4 100644 --- a/notebooks/ui.qmd +++ b/notebooks/ui.qmd @@ -40,6 +40,16 @@ HANDBOOK_URL = ( def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -50,22 +60,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -75,6 +135,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -82,6 +150,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -89,6 +170,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: @@ -111,4 +203,4 @@ def check_result(console: Console, result) -> None: f"[dim]principle {principle}[/dim]\n" f" {escape(result.message)}" ) -``` \ No newline at end of file +``` diff --git a/pyproject.toml b/pyproject.toml index cc279f2..6405d5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "dataclasses>=0.8", + "globus-sdk>=4.5.0", "pyprojroot>=0.3.0", "pyyaml>=6.0.3", "questionary>=2.1.1", @@ -28,4 +29,4 @@ build-backend = "uv_build" "stagecoach.templates" = ["*.yaml"] [tool.uv.sources] -sheriff = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git", rev = "v0.1.2" } +sheriff = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git", rev = "v0.1.1" } diff --git a/src/stagecoach/checks.py b/src/stagecoach/checks.py index 905a571..23a6a8c 100644 --- a/src/stagecoach/checks.py +++ b/src/stagecoach/checks.py @@ -4,6 +4,18 @@ from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -24,7 +36,18 @@ class Severity(Enum): @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -43,6 +66,19 @@ class CheckResult: def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -59,9 +95,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. + + Parameters + ---------- + directory : Path + Project directory to inspect. - Pass/Fail, no exceptions. + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -79,6 +123,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -110,6 +168,20 @@ def check_environment_exists(directory: Path) -> CheckResult: def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -160,6 +232,20 @@ def check_code_exists(directory: Path) -> CheckResult: def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -205,6 +291,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", diff --git a/src/stagecoach/cli.py b/src/stagecoach/cli.py index 9a1a79a..c1cbf55 100644 --- a/src/stagecoach/cli.py +++ b/src/stagecoach/cli.py @@ -1,11 +1,10 @@ from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -15,6 +14,16 @@ ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -31,6 +40,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -39,6 +53,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -52,6 +75,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: @@ -80,6 +104,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -102,10 +133,31 @@ def inspect( @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + manifest_path=manifest_path + ).stage() + if not staged: + raise typer.Exit(code=1) if __name__ == "__main__": diff --git a/src/stagecoach/customs.py b/src/stagecoach/customs.py index c01c563..d53da69 100644 --- a/src/stagecoach/customs.py +++ b/src/stagecoach/customs.py @@ -1,98 +1,244 @@ -from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) + + +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client + + +from dataclasses import dataclass, field +from typing import Any @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None +class Clearance: + """ + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. + """ + + source: str cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +def check_globus_clearance( + globus_info: dict + ) -> Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. """ try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) + + +def build_globus_transfer( + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", + ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label=label, + ) + + for item in globus_info["items"]: - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + + transfer.add_item( + str(source_path), + str(destination_path), + recursive=item.get("recursive", True), + ) + + return transfer + + +import glob +import os +from pathlib import Path + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) diff --git a/src/stagecoach/issue_manifest.py b/src/stagecoach/issue_manifest.py index a8f4e8a..fa9144f 100644 --- a/src/stagecoach/issue_manifest.py +++ b/src/stagecoach/issue_manifest.py @@ -2,9 +2,23 @@ from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -12,7 +26,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") # mysheriff = Sheriff() @@ -23,75 +37,105 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, # "stagecoach_version": 0.1, # # critical: we must know where the frontier starts # "root": "/n/holylabs/cgolden_lab/Lab/frontier" # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" -# }, -# # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None -# } +# "governance": "governance", # }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA -# "citizen": { -# # your name must match your name in town +# "citizen": { # "name": None, - -# # email may be used in future for authentications -# "email": None +# "email": None, # }, # "project": { -# # this must match the name of your project repository # "project_name": None, - -# # must match above # "project_working_dir": str(here()), - -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus +# "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) import yaml @@ -99,7 +143,14 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -109,8 +160,19 @@ def load_template() -> dict[str, Any]: from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. - """Prompt the user and fill in manifest fields.""" + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -135,30 +197,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest @@ -168,7 +237,6 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: # filled_manifest = fill_manifest_interactively(mymanifest) -import questionary from pathlib import Path from rich.console import Console @@ -178,7 +246,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -192,12 +278,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") # from tempfile import TemporaryDirectory @@ -212,7 +298,6 @@ def write_manifest( from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -226,33 +311,35 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") diff --git a/src/stagecoach/manifest_checker.py b/src/stagecoach/manifest_checker.py index bfffe12..7f7b0c2 100644 --- a/src/stagecoach/manifest_checker.py +++ b/src/stagecoach/manifest_checker.py @@ -4,10 +4,23 @@ class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -16,7 +29,12 @@ def __init__(self, manifest_file: str | Path, severity_level: Severity): def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -31,11 +49,13 @@ def run_all(self) -> list[CheckResult]: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() diff --git a/src/stagecoach/outpost.py b/src/stagecoach/outpost.py new file mode 100644 index 0000000..9ff118a --- /dev/null +++ b/src/stagecoach/outpost.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier diff --git a/src/stagecoach/stage.py b/src/stagecoach/stage.py new file mode 100644 index 0000000..c7c02bb --- /dev/null +++ b/src/stagecoach/stage.py @@ -0,0 +1,281 @@ +import os +import glob +from pathlib import Path +from stagecoach.ui import info, error, success, warning +from rich.console import Console + + +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") + + success(console, "Stage directories are set up.") + + +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + + +from stagecoach.customs import ( + + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> StageResult: + """ + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) + + +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) diff --git a/src/stagecoach/stagecoach.py b/src/stagecoach/stagecoach.py index 51302f0..23c7df6 100644 --- a/src/stagecoach/stagecoach.py +++ b/src/stagecoach/stagecoach.py @@ -1,11 +1,13 @@ import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -21,7 +23,16 @@ class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -51,30 +62,86 @@ def hail( sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: - """Create a StageCoach manifest.""" + outpost: bool = False, + ) -> bool: + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path banner(console, "🚂 Hailing the Stagecoach...") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") + + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + outpost = create_outpost() if outpost else nullcontext() + + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) + + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: - """Inspect a StageCoach manifest.""" + ) -> bool: + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -82,16 +149,38 @@ def inspect( checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -100,24 +189,106 @@ def inspect( "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + failure_panel(console, "⚠️ Staging failed. Please try again.") + return False diff --git a/src/stagecoach/templates/manifest.yml b/src/stagecoach/templates/manifest.yml index 2c50c22..f1dcbf0 100644 --- a/src/stagecoach/templates/manifest.yml +++ b/src/stagecoach/templates/manifest.yml @@ -1,6 +1,3 @@ -citizen: - email: null - name: null frontier: name: golden-lab root: /n/holylabs/cgolden_lab/Lab/frontier @@ -8,19 +5,44 @@ frontier: stagecoach_version: 0.1 paths: goldmine: goldmine - governance: governance - town: town works: works + town: town + governance: governance +citizen: + name: null + email: null project: - input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs - output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs project_name: null project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach -remote: - globus: - globus_destination_endpoint: null - globus_destination_path: null - globus_source_endpoint: null - globus_source_path: null - globus_username: null - use_globus: false + input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs + intermediate_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/intermediates + output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs + sandbox_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/sandbox +sources: + 01_gold_mine: + enabled: false + items: + - name: example_goldmine_item + path_regex: null + required: true + 02_globus: + enabled: false + source_endpoint: null + destination_endpoint: null + items: + - name: example_globus_item + source_path: null + destination_path: null + files_regex: [] + recursive: true + required: true + 03_dataverse: + enabled: false + server_url: null + dataset_pid: null + dataset_version: latest + api_token_file: null + items: + - name: example_dataverse_item + files_regex: [] + required: true diff --git a/src/stagecoach/ui.py b/src/stagecoach/ui.py index 87d06ae..0ceea35 100644 --- a/src/stagecoach/ui.py +++ b/src/stagecoach/ui.py @@ -12,6 +12,16 @@ def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -22,22 +32,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -47,6 +107,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -54,6 +122,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -61,6 +142,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: diff --git a/stagecoach_manifest.yml b/stagecoach_manifest.yml index dd3e798..f6eb965 100644 --- a/stagecoach_manifest.yml +++ b/stagecoach_manifest.yml @@ -1,25 +1,46 @@ -citizen: - email: ttapera@hsph.harvard.edu - name: tinashe frontier: name: golden-lab - root: /n/holylabs/cgolden_lab/Lab/frontier schema_version: 1.0 + root: /n/holylabs/cgolden_lab/Lab/frontier paths: goldmine: goldmine - governance: governance - town: town works: works + town: town + governance: governance +citizen: + name: tinashe + email: ttapera@hsph.harvard.edu project: - input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs - output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs project_name: stagecoach - project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach -remote: - globus: - globus_destination_endpoint: 1156ed9e-6984-11ea-af52-0201714f6eab - globus_destination_path: /n/holylabs/cgolden_lab/Lab/projects/test_tinashe/_pkgdown.yml - globus_source_endpoint: 1156ed9e-6984-11ea-af52-0201714f6eab - globus_source_path: /n/holylabs/cgolden_lab/Lab/projects/sandbox/ReaProject/MadagascarWeatherStations/_pkgdown.yml - globus_username: foo - use_globus: true + project_working_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach + input_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/inputs + intermediate_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/intermediates + output_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/outputs + sandbox_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/sandbox +sources: + 01_gold_mine: + enabled: true + items: + - name: example_goldmine_item + path_regex: "/n/holylabs/cgolden_lab/Lab/frontier/gold_mine/01_ore/golden_googledrive_rclone/CLTN/4. Datasets/Data" + required: true + 02_globus: + enabled: true + source_endpoint: "1156ed9e-6984-11ea-af52-0201714f6eab" + destination_endpoint: "1156ed9e-6984-11ea-af52-0201714f6eab" + items: + - name: example_globus_item + source_path: "/n/holylabs/cgolden_lab/Lab/projects/era5_database/era5_sandbox/notes" + files_regex: ["*.ipynb"] + recursive: true + required: true + 03_dataverse: + enabled: false + server_url: null + dataset_pid: null + dataset_version: latest + api_token_file: null + items: + - name: example_dataverse_item + files_regex: [] + required: true diff --git a/user_guide/notebooks/checks.md b/user_guide/notebooks/checks.md index 0c78478..bff38d0 100644 --- a/user_guide/notebooks/checks.md +++ b/user_guide/notebooks/checks.md @@ -39,6 +39,18 @@ from enum import Enum from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -59,7 +71,18 @@ PRINCIPLES = { @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -87,6 +110,19 @@ Now, let’s define a few checks: ``` python def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -103,9 +139,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. - Pass/Fail, no exceptions. + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -123,6 +167,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -160,6 +218,20 @@ case a user hasn’t yet started writing code: ``` python def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -214,6 +286,20 @@ things like READMEs, notebooks, or markdown files: ``` python def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -259,6 +345,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", diff --git a/user_guide/notebooks/cli.md b/user_guide/notebooks/cli.md index be54eef..a581427 100644 --- a/user_guide/notebooks/cli.md +++ b/user_guide/notebooks/cli.md @@ -9,15 +9,16 @@ Once the CLI is defined as “hailing” the stagecoach, stagecoach will run `issue_manifest` to create the manifest, and then proceed with the rest of the stagecoach. +Here’s the code for the CLI. First, hailing the stagecoach: + ``` python from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -27,6 +28,16 @@ app = typer.Typer( ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -43,6 +54,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -51,6 +67,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -64,6 +89,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: @@ -71,6 +97,8 @@ def hail( raise typer.Exit(code=1) ``` +Inspecting the manifest: + ``` python @app.command() def inspect( @@ -93,6 +121,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -114,12 +149,35 @@ def inspect( raise typer.Exit(code=1) ``` +And, staging: + ``` python @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + manifest_path=manifest_path + ).stage() + if not staged: + raise typer.Exit(code=1) ``` ``` python diff --git a/user_guide/notebooks/customs.md b/user_guide/notebooks/customs.md index 198801e..932734e 100644 --- a/user_guide/notebooks/customs.md +++ b/user_guide/notebooks/customs.md @@ -8,20 +8,22 @@ validation is going to be called “customs”. As a reminder, the works by issuing users a manifest. This manifest is a JSON file that users fill in with their credentials and desired data from the Frontier Gold Mine. Once the user returns the manifest, and the `Stagecoach` -recognizes that the user wants access to data via `globus`, and passes -this information to the customs handler. The customs handler then -validates the credentials to ensure that the user has the correct -permissions to access the data. This validation process uses the -`Globus SDK` to check the credentials against the Globus service. If the -credentials are valid, the customs handler returns a success message, -allowing the `Stagecoach` to proceed with the data delivery. If the -credentials are invalid, the handler returns an error message, and the -`Stagecoach` can inform the user of the issue and prompt them to correct -their credentials in the manifest. This process ensures that only -authorized users can access the data, maintaining the security of The -Frontier’s Gold Mine resources. - -## Tinkering with the SDK +recognizes that the user wants access to data via `globus`, `Dataverse`, +or from the Gold Mine, the `Stagecoach` will call the `customs` handler +and pass this information along. The customs handler then validates the +credentials to ensure that the user has the correct permissions to +access the data. This validation process uses the `Globus SDK`, +`Dataverse API`, or a few simple local file permission checks to verify +the credentials against the data provider service. If the credentials +are valid, the customs handler returns a success message, allowing the +`Stagecoach` to proceed with the data delivery. If the credentials are +invalid, the handler returns an error message, and the `Stagecoach` can +inform the user of the issue and prompt them to correct their +credentials in the manifest. This process ensures that only authorized +users can access the data, maintaining the security of The Frontier’s +Gold Mine resources. + +## Globus SDK Authentication and Validation Globus SDK in Python works as below. Once you’ve set up an App in the [Developer @@ -94,111 +96,285 @@ sheriff to validate the globus section of a stagecoach manifest. The manifest will have the following shape: -We’ll need to validate that the user has the kind of access necessary. -The return will be a dataclass that the stagecoach can accept the -requisite transfer object from and execute. +``` python +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) +``` + +In the customs manager, we will have to do two things: authenticate with +the Globus client, and send the actual data. After wrestling with this +for a few tries, and bouncing design ideas off of chatgpt and jupyter +notebooks, I’ve settled on adding a context helper to do the UserApp +creation: + +``` python +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client +``` + +This will allow us to create transfer clients with the `with` statement, +allowing us to iterate over the items in the Globus request +individually, granting each of them a unique request and clearance: ``` python -from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None +class Clearance: + """ + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. + """ + + source: str cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +def check_globus_clearance( + globus_info: dict + ) -> Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. """ try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) +``` + +If the checks pass, we can move on to building the transfer logic. This +is built mostly on what we learned above: + +``` python +def build_globus_transfer( + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", + ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label=label, + ) + + for item in globus_info["items"]: - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + + transfer.add_item( + str(source_path), + str(destination_path), + recursive=item.get("recursive", True), + ) + + return transfer +``` + +Now with that, the customs manager can first check for clearance, and +additionally build the transfer behind the scenes. + +## Gold Mine + +Accessing the Gold Mine can only be done from FASRC, for now. It will +implement a simple check for whether the user has access to the +specified path. This will reuse the `Clearance` class from earlier: + +``` python +import glob +import os +from pathlib import Path + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) ``` +## Dataverse + +🚧 Not yet implemented 🚧 + +## Conclusion + When that sends back a clearance object with `cleared=True`, the `Stagecoach` can proceed with the transfer using the provided `transfer_client`. If `cleared=False`, the `Stagecoach` can inform the diff --git a/user_guide/notebooks/issue_manifest.md b/user_guide/notebooks/issue_manifest.md index 860a506..bd38f95 100644 --- a/user_guide/notebooks/issue_manifest.md +++ b/user_guide/notebooks/issue_manifest.md @@ -46,9 +46,23 @@ from rich.console import Console from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -56,7 +70,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") ``` We can see this function in action below: @@ -78,75 +92,105 @@ and figure out how to get the data to the user’s specified location. # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, # "stagecoach_version": 0.1, # # critical: we must know where the frontier starts # "root": "/n/holylabs/cgolden_lab/Lab/frontier" # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" -# }, -# # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None -# } +# "governance": "governance", # }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA -# "citizen": { -# # your name must match your name in town +# "citizen": { # "name": None, - -# # email may be used in future for authentications -# "email": None +# "email": None, # }, # "project": { -# # this must match the name of your project repository # "project_name": None, - -# # must match above # "project_working_dir": str(here()), - -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus +# "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) ``` If a citizen can pass the citizenship check, then the stagecoach can @@ -161,7 +205,14 @@ from typing import Any from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -187,8 +238,19 @@ import questionary from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. - """Prompt the user and fill in manifest fields.""" + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -213,30 +275,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest ``` @@ -252,7 +321,6 @@ In practice, this process looks like: Next, writing the manifest should be straight forward: ``` python -import questionary from pathlib import Path from rich.console import Console @@ -262,7 +330,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -276,12 +362,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") ``` To test: @@ -302,7 +388,6 @@ Looks great! We now have the pieces to issue a manifest: ``` python from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -316,36 +401,38 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") ``` Issuing a manifest writes the manifest to the file system, where the diff --git a/user_guide/notebooks/manifest_checker.md b/user_guide/notebooks/manifest_checker.md index 6783bbb..dd94380 100644 --- a/user_guide/notebooks/manifest_checker.md +++ b/user_guide/notebooks/manifest_checker.md @@ -14,10 +14,23 @@ from stagecoach.checks import * class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -26,7 +39,12 @@ class ManifestChecker: def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -41,11 +59,13 @@ class ManifestChecker: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() diff --git a/user_guide/notebooks/outpost.md b/user_guide/notebooks/outpost.md new file mode 100644 index 0000000..36d3b73 --- /dev/null +++ b/user_guide/notebooks/outpost.md @@ -0,0 +1,79 @@ +# Outpost + + +We have a problem. The `stagecoach` tool is designed to run on the FASRC +cluster, and it relies on the `sheriff` to run a citizenship test +(i.e. determine that you are bought in to use The Frontier). This means +that if you want to run `stagecoach` outside of FASRC, you need to have +those documents, which will be a barrier for many users. + +Here, we introduce the `outpost` module, which will temporarily scaffold +a fake Frontier directory that the sheriff will accept as valid. This +will allow users to run `stagecoach` on their local machines, without +needing to have the actual citizenship documents. + +We’re going ot implement this as a CLI flag, `--outpost`, which will +trigger the creation of this fake directory. Additionally, ChatGPT +recommends using the `contextlib` module to create a context manager +that will handle the setup and teardown of this fake directory, ensuring +that it is cleaned up after use. + +The `outpost` module will have a `create_outpost` function context. + +``` python +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier +``` diff --git a/user_guide/notebooks/stage.md b/user_guide/notebooks/stage.md new file mode 100644 index 0000000..dd2c9f5 --- /dev/null +++ b/user_guide/notebooks/stage.md @@ -0,0 +1,315 @@ +# Stage Data + + +Finally, it’s time to deliver the user’s data to them. In this module, +we define several staging methods that the stagecoach can use. + +There are a couple of things that will need to happen to make this work. +One of them will be creating the directories for data usage: + +``` python +import os +import glob +from pathlib import Path +from stagecoach.ui import info, error, success, warning +from rich.console import Console +``` + +``` python +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") + + success(console, "Stage directories are set up.") +``` + +Let’s create a quick `StageResult` class to represent the results of +staging operations, so we can report them nicely in the UI and +potentially use them in the future for more detailed reporting: + +``` python +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) +``` + +Next, we can work on fetching the data from Globus. Fortunately, much of +the work has been done for us in the `customs` module, so we just need +to call the appropriate functions. + +``` python +from stagecoach.customs import ( + + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> StageResult: + """ + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) +``` + +Next we implement symlink staging via FASRC: + +``` python +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) +``` + +That should do it for now! We will implement the dataverse staging later +on. + +# Script file + +The code for this document can be found here: + +- [../src/stagecoach/stage.py](../src/stagecoach/stage.py) diff --git a/user_guide/notebooks/stagecoach.md b/user_guide/notebooks/stagecoach.md index 76cbe1b..4994f7a 100644 --- a/user_guide/notebooks/stagecoach.md +++ b/user_guide/notebooks/stagecoach.md @@ -51,13 +51,15 @@ defined in the `stagecoach` class: ``` python import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -73,7 +75,16 @@ from stagecoach.ui import ( class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -103,30 +114,86 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: - """Create a StageCoach manifest.""" + outpost: bool = False, + ) -> bool: + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path banner(console, "🚂 Hailing the Stagecoach...") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) + + outpost = create_outpost() if outpost else nullcontext() + + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) + + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: - """Inspect a StageCoach manifest.""" + ) -> bool: + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -134,16 +201,38 @@ class StageCoach: checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -152,27 +241,109 @@ class StageCoach: "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + failure_panel(console, "⚠️ Staging failed. Please try again.") + return False ``` Click on each section below to learn how stagecoach works in more diff --git a/user_guide/notebooks/ui.md b/user_guide/notebooks/ui.md index 917b390..db4600f 100644 --- a/user_guide/notebooks/ui.md +++ b/user_guide/notebooks/ui.md @@ -20,6 +20,16 @@ HANDBOOK_URL = ( def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -30,22 +40,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -55,6 +115,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -62,6 +130,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -69,6 +150,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: diff --git a/uv.lock b/uv.lock index 4412201..63d4a8d 100644 --- a/uv.lock +++ b/uv.lock @@ -1756,10 +1756,9 @@ wheels = [ [[package]] name = "sheriff" -version = "0.1.2" -source = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.2#8ce66b626ed2d23dc8e9909b0114d90d19ba6797" } +version = "0.1.1" +source = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.1#491af8005d20b9ea6d5c0df58226e504107aac41" } dependencies = [ - { name = "globus-sdk" }, { name = "great-docs" }, { name = "ipykernel" }, { name = "jupyter" }, @@ -1808,6 +1807,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "dataclasses" }, + { name = "globus-sdk" }, { name = "pyprojroot" }, { name = "pyyaml" }, { name = "questionary" }, @@ -1819,11 +1819,12 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "dataclasses", specifier = ">=0.8" }, + { name = "globus-sdk", specifier = ">=4.5.0" }, { name = "pyprojroot", specifier = ">=0.3.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "questionary", specifier = ">=2.1.1" }, { name = "rich", specifier = ">=15.0.0" }, - { name = "sheriff", git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.2" }, + { name = "sheriff", git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.1" }, { name = "typer", specifier = ">=0.25.1" }, ]