From ef3b2cb19735dd1f3143bfb25acba5c3b06ca3e4 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Tue, 12 May 2026 11:25:24 -0400 Subject: [PATCH 01/10] Stage Data progress --- data/inputs/.gitignore | 2 + data/intermediates/.gitignore | 2 + data/outputs/.gitignore | 2 + notebooks/customs.qmd | 2 +- notebooks/issue_manifest.qmd | 75 ++++++++++++++++--------- notebooks/stage.qmd | 78 ++++++++++++++++++++++++++ src/stagecoach/issue_manifest.py | 75 ++++++++++++++++--------- src/stagecoach/stage.py | 48 ++++++++++++++++ src/stagecoach/templates/manifest.yml | 16 +++++- stagecoach_manifest.yml | 30 +++++++--- user_guide/notebooks/issue_manifest.md | 75 ++++++++++++++++--------- user_guide/notebooks/stage.md | 66 ++++++++++++++++++++++ 12 files changed, 375 insertions(+), 96 deletions(-) create mode 100644 data/inputs/.gitignore create mode 100644 data/intermediates/.gitignore create mode 100644 data/outputs/.gitignore create mode 100644 notebooks/stage.qmd create mode 100644 src/stagecoach/stage.py create mode 100644 user_guide/notebooks/stage.md diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/intermediates/.gitignore b/data/intermediates/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/intermediates/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/outputs/.gitignore b/data/outputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/outputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index c601425..99974f2 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -185,7 +185,7 @@ the globus section of a stagecoach manifest. The manifest will have the following shape: ```yaml -globus: +02_globus: use_globus: true globus_source_endpoint: fancy-uuid globus_source_path: /path/on/source/endpoint diff --git a/notebooks/issue_manifest.qmd b/notebooks/issue_manifest.qmd index d16b78e..9ae0384 100644 --- a/notebooks/issue_manifest.qmd +++ b/notebooks/issue_manifest.qmd @@ -98,6 +98,7 @@ to the user's specified location. # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here # import os # import yaml @@ -120,43 +121,54 @@ to the user's specified location. # "governance": "governance" # }, # # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { +# "sources": { +# "02_globus": { # # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus # "use_globus": False, # "globus_username": None, # "globus_source_endpoint": None, # "globus_source_path": None, # "globus_destination_endpoint": None, -# "globus_destination_path": None +# "globus_destination_path": None, +# "globus_files": [] +# }, +# "03_dataverse": { +# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets +# "use_dataverse": False, +# "dataverse_server_url": None, +# "dataverse_dataset_pid": None, +# "dataverse_dataset_version": "latest", +# "dataverse_api_token_file": None, +# "dataverse_files": [] +# }, +# "01_gold_mine": { +# # paths to data in the gold mine; only necessary if you want to access data in the gold mine +# "gold_mine_paths": [] # } # }, # # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA # "citizen": { - # # your name must match your name in town # "name": None, - # # email may be used in future for authentications # "email": None # }, - # "project": { # # this must match the name of your project repository # "project_name": None, - # # must match above # "project_working_dir": str(here()), - # # this is the name of the folder that stagecoach will ferry # # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - # # this is the name of the folder that stagecoach # # will ferry your output data from; # # only necessary if you want to ferry output data back to the frontier, # # eg via globus # "output_data_dir": str(here() / "data" / "outputs"), +# "intermediate_data_dir": str(here() / "data" / "intermediates"), +# "output_data_dir": str(here() / "data" / "outputs"), +# "sandbox_dir": str(here() / "sandbox") # } # } @@ -234,30 +246,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest ``` @@ -365,16 +384,16 @@ def issue_manifest( """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") ``` diff --git a/notebooks/stage.qmd b/notebooks/stage.qmd new file mode 100644 index 0000000..6e3895b --- /dev/null +++ b/notebooks/stage.qmd @@ -0,0 +1,78 @@ +--- +title: "Stage Data" +subtitle: "Staging: Delivering the Data to the User" + +filters: + - sorting-hat + - ripper +extensions: + sorting-hat: + keep: + - python + ripper: + include-yaml: false + output-name: "stage" + output-dir: "../src/stagecoach" +format: gfm +--- + +Finally, it's time to deliver the user's data to them. +In this module, we define several staging methods that +the stagecoach can use. + +There are a couple of things that will need to happen to +make this work. One of them will be creating the +directories for data usage: + +```{python} +import os +from pathlib import Path +from stagecoach.ui import info +from rich.console import Console +``` + +```{python} +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the stage directories if they don't exist. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + if not os.path.exists(input_dir): + os.makedirs(input_dir) + info(console, f"Created input data directory at {input_dir}") + else: + info(console, f"Input data directory already exists at {input_dir}") + + if not os.path.exists(intermediate_dir): + os.makedirs(intermediate_dir) + info(console, f"Created intermediate data directory at {intermediate_dir}") + else: + info(console, f"Intermediate data directory already exists at {intermediate_dir}") + + if not os.path.exists(sandbox_dir): + os.makedirs(sandbox_dir) + info(console, f"Created sandbox directory at {sandbox_dir}") + else: + info(console, f"Sandbox directory already exists at {sandbox_dir}") + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + info(console, f"Created output data directory at {output_dir}") + else: + info(console, f"Output data directory already exists at {output_dir}") + + if gitignore: + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + paths = [Path(p) / ".gitignore" for p in paths if p] # Remove any None values + + for p in paths: + with open(p, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p.parent}") +``` + diff --git a/src/stagecoach/issue_manifest.py b/src/stagecoach/issue_manifest.py index 69de0f6..6707fb0 100644 --- a/src/stagecoach/issue_manifest.py +++ b/src/stagecoach/issue_manifest.py @@ -23,6 +23,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here # import os # import yaml @@ -45,43 +46,54 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: # "governance": "governance" # }, # # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { +# "sources": { +# "02_globus": { # # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus # "use_globus": False, # "globus_username": None, # "globus_source_endpoint": None, # "globus_source_path": None, # "globus_destination_endpoint": None, -# "globus_destination_path": None +# "globus_destination_path": None, +# "globus_files": [] +# }, +# "03_dataverse": { +# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets +# "use_dataverse": False, +# "dataverse_server_url": None, +# "dataverse_dataset_pid": None, +# "dataverse_dataset_version": "latest", +# "dataverse_api_token_file": None, +# "dataverse_files": [] +# }, +# "01_gold_mine": { +# # paths to data in the gold mine; only necessary if you want to access data in the gold mine +# "gold_mine_paths": [] # } # }, # # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA # "citizen": { - # # your name must match your name in town # "name": None, - # # email may be used in future for authentications # "email": None # }, - # "project": { # # this must match the name of your project repository # "project_name": None, - # # must match above # "project_working_dir": str(here()), - # # this is the name of the folder that stagecoach will ferry # # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - # # this is the name of the folder that stagecoach # # will ferry your output data from; # # only necessary if you want to ferry output data back to the frontier, # # eg via globus # "output_data_dir": str(here() / "data" / "outputs"), +# "intermediate_data_dir": str(here() / "data" / "intermediates"), +# "output_data_dir": str(here() / "data" / "outputs"), +# "sandbox_dir": str(here() / "sandbox") # } # } @@ -134,30 +146,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest @@ -245,13 +264,13 @@ def issue_manifest( """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") diff --git a/src/stagecoach/stage.py b/src/stagecoach/stage.py new file mode 100644 index 0000000..d13ff38 --- /dev/null +++ b/src/stagecoach/stage.py @@ -0,0 +1,48 @@ +from stagecoach.ui import info +from rich.console import Console +import os + + +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the stage directories if they don't exist. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + if not os.path.exists(input_dir): + os.makedirs(input_dir) + info(console, f"Created input data directory at [bold]{input_dir}[/bold]") + else: + info(console, f"Input data directory already exists at [bold]{input_dir}[/bold]") + + if not os.path.exists(intermediate_dir): + os.makedirs(intermediate_dir) + info(console, f"Created intermediate data directory at [bold]{intermediate_dir}[/bold]") + else: + info(console, f"Intermediate data directory already exists at [bold]{intermediate_dir}[/bold]") + + if not os.path.exists(sandbox_dir): + os.makedirs(sandbox_dir) + info(console, f"Created sandbox directory at [bold]{sandbox_dir}[/bold]") + else: + info(console, f"Sandbox directory already exists at [bold]{sandbox_dir}[/bold]") + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + info(console, f"Created output data directory at [bold]{output_dir}[/bold]") + else: + info(console, f"Output data directory already exists at [bold]{output_dir}[/bold]") + + # if gitignore: + # paths = [''.join('.gitignore') for dir in [input_dir, intermediate_dir, output_dir, sandbox_dir]] + # gitignore_path = os.path.join(sandbox_dir, ".gitignore") + # if not os.path.exists(gitignore_path): + # with open(gitignore_path, "w") as f: + # f.write("*\n") + # info(console, f"Created .gitignore in sandbox directory at [bold]{gitignore_path}[/bold]") + # else: + # info(console, f".gitignore already exists in sandbox directory at [bold]{gitignore_path}[/bold]") diff --git a/src/stagecoach/templates/manifest.yml b/src/stagecoach/templates/manifest.yml index 76ad937..4495bb8 100644 --- a/src/stagecoach/templates/manifest.yml +++ b/src/stagecoach/templates/manifest.yml @@ -12,14 +12,26 @@ paths: works: works project: input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs + intermediate_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/intermediates output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs project_name: null project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach -remote: - globus: + sandbox_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/sandbox +sources: + 01_gold_mine: + gold_mine_paths: [] + 02_globus: globus_destination_endpoint: null globus_destination_path: null + globus_files: [] globus_source_endpoint: null globus_source_path: null globus_username: null use_globus: false + 03_dataverse: + dataverse_api_token_file: null + dataverse_dataset_pid: null + dataverse_dataset_version: latest + dataverse_files: [] + dataverse_server_url: null + use_dataverse: false diff --git a/stagecoach_manifest.yml b/stagecoach_manifest.yml index dd3e798..daeac4a 100644 --- a/stagecoach_manifest.yml +++ b/stagecoach_manifest.yml @@ -1,5 +1,5 @@ citizen: - email: ttapera@hsph.harvard.edu + email: '' name: tinashe frontier: name: golden-lab @@ -12,14 +12,26 @@ paths: works: works project: input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs + intermediate_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/intermediates output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs project_name: stagecoach project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach -remote: - globus: - globus_destination_endpoint: 1156ed9e-6984-11ea-af52-0201714f6eab - globus_destination_path: /n/holylabs/cgolden_lab/Lab/projects/test_tinashe/_pkgdown.yml - globus_source_endpoint: 1156ed9e-6984-11ea-af52-0201714f6eab - globus_source_path: /n/holylabs/cgolden_lab/Lab/projects/sandbox/ReaProject/MadagascarWeatherStations/_pkgdown.yml - globus_username: foo - use_globus: true + sandbox_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/sandbox +sources: + 01_gold_mine: + gold_mine_paths: [] + 02_globus: + globus_destination_endpoint: null + globus_destination_path: null + globus_files: [] + globus_source_endpoint: null + globus_source_path: null + globus_username: null + use_globus: false + 03_dataverse: + dataverse_api_token_file: null + dataverse_dataset_pid: null + dataverse_dataset_version: latest + dataverse_files: [] + dataverse_server_url: null + use_dataverse: false diff --git a/user_guide/notebooks/issue_manifest.md b/user_guide/notebooks/issue_manifest.md index a229c1e..df3920c 100644 --- a/user_guide/notebooks/issue_manifest.md +++ b/user_guide/notebooks/issue_manifest.md @@ -59,6 +59,7 @@ and figure out how to get the data to the user’s specified location. # we don't export this portion of script # because it will cause side effects to anyone who # loads it at runtime + # from pyprojroot import here # import os # import yaml @@ -81,43 +82,54 @@ and figure out how to get the data to the user’s specified location. # "governance": "governance" # }, # # we only have two compute options for now, but this may expand in the future -# "remote": { -# "globus": { +# "sources": { +# "02_globus": { # # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus # "use_globus": False, # "globus_username": None, # "globus_source_endpoint": None, # "globus_source_path": None, # "globus_destination_endpoint": None, -# "globus_destination_path": None +# "globus_destination_path": None, +# "globus_files": [] +# }, +# "03_dataverse": { +# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets +# "use_dataverse": False, +# "dataverse_server_url": None, +# "dataverse_dataset_pid": None, +# "dataverse_dataset_version": "latest", +# "dataverse_api_token_file": None, +# "dataverse_files": [] +# }, +# "01_gold_mine": { +# # paths to data in the gold mine; only necessary if you want to access data in the gold mine +# "gold_mine_paths": [] # } # }, # # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA # "citizen": { - # # your name must match your name in town # "name": None, - # # email may be used in future for authentications # "email": None # }, - # "project": { # # this must match the name of your project repository # "project_name": None, - # # must match above # "project_working_dir": str(here()), - # # this is the name of the folder that stagecoach will ferry # # your input data to # "input_data_dir": str(here() / "data" / "inputs"), - # # this is the name of the folder that stagecoach # # will ferry your output data from; # # only necessary if you want to ferry output data back to the frontier, # # eg via globus # "output_data_dir": str(here() / "data" / "outputs"), +# "intermediate_data_dir": str(here() / "data" / "intermediates"), +# "output_data_dir": str(here() / "data" / "outputs"), +# "sandbox_dir": str(here() / "sandbox") # } # } @@ -193,30 +205,37 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: default=str(Path(project_working_dir) / "data" / "outputs"), ).ask() - use_globus = questionary.confirm( - "Use Globus for transport?", - default=False, - ).ask() + # use_globus = questionary.confirm( + # "Use Globus for transport?", + # default=False, + # ).ask() - manifest.setdefault("citizen", {}) - manifest.setdefault("project", {}) - manifest.setdefault("remote", {}) - manifest["remote"].setdefault("globus", {}) + # manifest.setdefault("citizen", {}) + # manifest.setdefault("project", {}) + # manifest.setdefault("source", {}) + # manifest["source"].setdefault("globus", {}) manifest["citizen"]["name"] = citizen_name manifest["citizen"]["email"] = citizen_email manifest["project"]["project_name"] = project_name manifest["project"]["project_working_dir"] = project_working_dir manifest["project"]["input_data_dir"] = input_data_dir manifest["project"]["output_data_dir"] = output_data_dir - manifest["remote"]["globus"]["use_globus"] = use_globus - if use_globus: - manifest["remote"]["globus"]["globus_username"] = questionary.text( - "Globus username:" - ).ask() - manifest["remote"]["globus"]["globus_endpoint"] = questionary.text( - "Globus endpoint:" - ).ask() + # if use_globus: + # manifest["source"]["02_globus"]["globus_username"] = questionary.text( + # "Globus username:" + # ).ask() + # manifest["source"]["02_globus"]["globus_endpoint"] = questionary.text( + # "Globus endpoint:" + # ).ask() + + # if use_dataverse: + # manifest["source"]["03_dataverse"]["dataverse_server_url"] = questionary.text( + # "Dataverse server URL:" + # ).ask() + # manifest["source"]["03_dataverse"]["dataverse_dataset_pid"] = questionary.text( + # "Dataverse dataset PID:" + # ).ask() return manifest ``` @@ -316,16 +335,16 @@ def issue_manifest( """ check_citizenship(customs_sheriff, console) - info(console, "[bold]Loading manifest template...[/bold]") + info(console, "Loading manifest template...") manifest = load_template() - info(console, "[green]✔ Template loaded[/green]\n") + success(console, "Template loaded\n") if interactive: manifest = fill_manifest_interactively(manifest) write_manifest(manifest, output_path, overwrite=overwrite, console=console) - success(console, "Next step:\n[bold]stagecoach inspect[/bold]") + success(console, "Next step: stagecoach inspect") ``` Issuing a manifest writes the manifest to the file system, where the diff --git a/user_guide/notebooks/stage.md b/user_guide/notebooks/stage.md new file mode 100644 index 0000000..b915459 --- /dev/null +++ b/user_guide/notebooks/stage.md @@ -0,0 +1,66 @@ +# Stage Data + + +Finally, it’s time to deliver the user’s data to them. In this module, +we define several staging methods that the stagecoach can use. + +There are a couple of things that will need to happen to make this work. +One of them will be creating the directories for data usage: + +``` python +from stagecoach.ui import info +from rich.console import Console +import os +``` + +``` python +def create_stage_directories(manifest: dict, console: Console, gitignore=True): + """ + Create the stage directories if they don't exist. + """ + + input_dir = manifest.get("project", {}).get("input_data_dir") + intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") + output_dir = manifest.get("project", {}).get("output_data_dir") + sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + if not os.path.exists(input_dir): + os.makedirs(input_dir) + info(console, f"Created input data directory at [bold]{input_dir}[/bold]") + else: + info(console, f"Input data directory already exists at [bold]{input_dir}[/bold]") + + if not os.path.exists(intermediate_dir): + os.makedirs(intermediate_dir) + info(console, f"Created intermediate data directory at [bold]{intermediate_dir}[/bold]") + else: + info(console, f"Intermediate data directory already exists at [bold]{intermediate_dir}[/bold]") + + if not os.path.exists(sandbox_dir): + os.makedirs(sandbox_dir) + info(console, f"Created sandbox directory at [bold]{sandbox_dir}[/bold]") + else: + info(console, f"Sandbox directory already exists at [bold]{sandbox_dir}[/bold]") + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + info(console, f"Created output data directory at [bold]{output_dir}[/bold]") + else: + info(console, f"Output data directory already exists at [bold]{output_dir}[/bold]") + + # if gitignore: + # paths = [''.join('.gitignore') for dir in [input_dir, intermediate_dir, output_dir, sandbox_dir]] + # gitignore_path = os.path.join(sandbox_dir, ".gitignore") + # if not os.path.exists(gitignore_path): + # with open(gitignore_path, "w") as f: + # f.write("*\n") + # info(console, f"Created .gitignore in sandbox directory at [bold]{gitignore_path}[/bold]") + # else: + # info(console, f".gitignore already exists in sandbox directory at [bold]{gitignore_path}[/bold]") +``` + +# Script file + +The code for this document can be found here: + +- [../src/stagecoach/stage.py](../src/stagecoach/stage.py) From e5de19ea6163d5317037b60f7bdbe000d0577038 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 01:29:49 -0400 Subject: [PATCH 02/10] Significant progress to fix #2-Stage-Data, including manifest restructuring, CLI enhancements, and documentation updates. feat(manifest): restructure manifest for clarity and functionality * Reorganized the manifest structure for better readability and usability. * Added new fields for project details and source configurations. * Updated paths for data directories to reflect new project structure. * Enabled and configured sources for gold mine, globus, and dataverse. * Improved the citizen section for better user identification. feat(cli): enhance CLI commands for manifest handling * Added commands for inspecting and staging data. * Improved error handling and user feedback during CLI operations. * Integrated customs checks for globus and gold mine access. docs(guide): update user guide for new features * Revised sections on manifest creation and staging processes. * Added examples for new CLI commands and their usage. * Clarified the customs validation process for data access. style(notebooks): improve code formatting and readability * Refactored code snippets for better clarity and consistency. * Enhanced comments and documentation within the code. --- notebooks/cli.qmd | 14 +- notebooks/customs.qmd | 288 +++++++++++++++--------- notebooks/issue_manifest.qmd | 134 ++++++----- notebooks/stage.qmd | 245 +++++++++++++++++--- notebooks/stagecoach.qmd | 118 ++++++++-- src/stagecoach/cli.py | 7 +- src/stagecoach/customs.py | 235 +++++++++++-------- src/stagecoach/issue_manifest.py | 134 ++++++----- src/stagecoach/stage.py | 230 ++++++++++++++++--- src/stagecoach/stagecoach.py | 117 ++++++++-- src/stagecoach/templates/manifest.yml | 54 +++-- stagecoach_manifest.yml | 60 ++--- user_guide/notebooks/cli.md | 13 +- user_guide/notebooks/customs.md | 299 ++++++++++++++++--------- user_guide/notebooks/issue_manifest.md | 134 ++++++----- user_guide/notebooks/stage.md | 246 +++++++++++++++++--- user_guide/notebooks/stagecoach.md | 117 ++++++++-- 17 files changed, 1759 insertions(+), 686 deletions(-) diff --git a/notebooks/cli.qmd b/notebooks/cli.qmd index ccb4031..56e4d8c 100644 --- a/notebooks/cli.qmd +++ b/notebooks/cli.qmd @@ -29,6 +29,8 @@ Once the CLI is defined as "hailing" the stagecoach, stagecoach will run `issue_manifest` to create the manifest, and then proceed with the rest of the stagecoach. +Here's the code for the CLI. First, hailing the stagecoach: + ```{python} #| sorting-hat: keep @@ -93,6 +95,8 @@ def hail( raise typer.Exit(code=1) ``` +Inspecting the manifest: + ```{python} @app.command() def inspect( @@ -136,12 +140,20 @@ def inspect( raise typer.Exit(code=1) ``` +And, staging: + ```{python} @app.command() def stage(): console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + ).stage() + if not staged: + raise typer.Exit(code=1) + ``` ```{python} diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index 99974f2..b7f3932 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -26,11 +26,13 @@ This manifest is a JSON file that users fill in with their credentials and desired data from the Frontier Gold Mine. Once the user returns the manifest, and the `Stagecoach` recognizes that the user wants access to data via `globus`, -and passes this information to the customs handler. +`Dataverse`, or from the Gold Mine, the `Stagecoach` will call the +`customs` handler and pass this information along. The customs handler then validates the credentials to ensure that the user has the correct permissions to access the data. This validation process -uses the `Globus SDK` to check the credentials against the -Globus service. If the credentials are valid, the customs handler +uses the `Globus SDK`, `Dataverse API`, or a few simple local +file permission checks to verify the credentials against the +data provider service. If the credentials are valid, the customs handler returns a success message, allowing the `Stagecoach` to proceed with the data delivery. If the credentials are invalid, the handler returns an error message, and the `Stagecoach` can @@ -39,7 +41,7 @@ credentials in the manifest. This process ensures that only authorized users can access the data, maintaining the security of The Frontier's Gold Mine resources. -## Tinkering with the SDK +## Globus SDK Authentication and Validation Globus SDK in Python works as below. Once you've set up an App in the [Developer panel](https://globus-sdk-python.readthedocs.io/en/stable/user_guide/getting_started/register_app.html) of the Globus website, @@ -184,121 +186,199 @@ the globus section of a stagecoach manifest. The manifest will have the following shape: -```yaml -02_globus: - use_globus: true - globus_source_endpoint: fancy-uuid - globus_source_path: /path/on/source/endpoint - globus_destination_endpoint: fancy-uuid - globus_destination_path: /path/on/destination/endpoint +```{python} +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) +``` + +In the customs manager, we will have to do two things: +authenticate with the Globus client, and send the actual +data. After wrestling with this for a few tries, and bouncing +design ideas off of chatgpt and jupyter notebooks, I've settled on +adding a context helper to do the UserApp creation: + +```{python} +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client ``` -We'll need to validate that the user has the kind of access -necessary. The return will be a dataclass that the stagecoach -can accept the requisite transfer object from and execute. +This will allow us to create transfer clients with the `with` +statement, allowing us to iterate over the items +in the Globus request individually, granting each of +them a unique request and clearance: ```{python} -#| sorting-hat: keep +from dataclasses import dataclass, field +from typing import Any from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None - cleared: bool - -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +class Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Result of a Stagecoach source access check. """ + + source: str + cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + +def check_globus_clearance( + globus_info: dict + ) -> Clearance: try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) - - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) + +``` + +If the checks pass, we can move on to building the transfer +logic. This is built mostly on what we learned above: + +```{python} +def build_globus_transfer( + globus_info: dict, + clearance: Clearance + ) -> TransferData: + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label="Stagecoach transfer", + ) + + for item in globus_info["items"]: + transfer.add_item( + item["source_path"], + item["destination_path"], + recursive=item.get("recursive", True), + ) + + return transfer +``` + +Now with that, the customs manager can first +check for clearance, and additionally build the transfer +behind the scenes. + +## Gold Mine + +Accessing the Gold Mine can only be done from FASRC, for now. +It will implement a simple check for whether the user has access to +the specified path. This will reuse the `Clearance` class +from earlier: + +```{python} +import glob + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) ``` +## Dataverse + +🚧 Not yet implemented 🚧 + +## Conclusion When that sends back a clearance object with `cleared=True`, the `Stagecoach` can proceed with the transfer using the provided `transfer_client`. diff --git a/notebooks/issue_manifest.qmd b/notebooks/issue_manifest.qmd index 9ae0384..ac850ec 100644 --- a/notebooks/issue_manifest.qmd +++ b/notebooks/issue_manifest.qmd @@ -63,7 +63,7 @@ from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: """Validate Frontier citizenship using the Sheriff.""" - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -71,7 +71,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") ``` We can see this function in action below: @@ -100,84 +100,100 @@ to the user's specified location. # loads it at runtime # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, - -# # critical: we must know where the frontier starts -# "root": "/n/holylabs/cgolden_lab/Lab/frontier" +# "root": "/n/holylabs/cgolden_lab/Lab/frontier", # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" +# "governance": "governance", # }, -# # we only have two compute options for now, but this may expand in the future -# "sources": { -# "02_globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None, -# "globus_files": [] -# }, -# "03_dataverse": { -# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets -# "use_dataverse": False, -# "dataverse_server_url": None, -# "dataverse_dataset_pid": None, -# "dataverse_dataset_version": "latest", -# "dataverse_api_token_file": None, -# "dataverse_files": [] -# }, -# "01_gold_mine": { -# # paths to data in the gold mine; only necessary if you want to access data in the gold mine -# "gold_mine_paths": [] -# } -# }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA + # "citizen": { -# # your name must match your name in town # "name": None, -# # email may be used in future for authentications -# "email": None +# "email": None, # }, + # "project": { -# # this must match the name of your project repository # "project_name": None, -# # must match above # "project_working_dir": str(here()), -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus -# "output_data_dir": str(here() / "data" / "outputs"), # "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# "sandbox_dir": str(here() / "sandbox") -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) ``` If a citizen can pass the citizenship check, then @@ -320,12 +336,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") ``` To test: diff --git a/notebooks/stage.qmd b/notebooks/stage.qmd index 6e3895b..4ee5363 100644 --- a/notebooks/stage.qmd +++ b/notebooks/stage.qmd @@ -26,8 +26,10 @@ directories for data usage: ```{python} import os +import re +import glob from pathlib import Path -from stagecoach.ui import info +from stagecoach.ui import info, error, success, warning from rich.console import Console ``` @@ -41,38 +43,215 @@ def create_stage_directories(manifest: dict, console: Console, gitignore=True): intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") output_dir = manifest.get("project", {}).get("output_data_dir") sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") - if not os.path.exists(input_dir): - os.makedirs(input_dir) - info(console, f"Created input data directory at {input_dir}") - else: - info(console, f"Input data directory already exists at {input_dir}") - - if not os.path.exists(intermediate_dir): - os.makedirs(intermediate_dir) - info(console, f"Created intermediate data directory at {intermediate_dir}") - else: - info(console, f"Intermediate data directory already exists at {intermediate_dir}") - - if not os.path.exists(sandbox_dir): - os.makedirs(sandbox_dir) - info(console, f"Created sandbox directory at {sandbox_dir}") - else: - info(console, f"Sandbox directory already exists at {sandbox_dir}") - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - info(console, f"Created output data directory at {output_dir}") - else: - info(console, f"Output data directory already exists at {output_dir}") - - if gitignore: - paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] - paths = [Path(p) / ".gitignore" for p in paths if p] # Remove any None values - - for p in paths: - with open(p, "w") as f: - f.write("*\n!.gitignore\n") - info(console, f"Created .gitignore in {p.parent}") + success(console, "Stage directories are set up.") + ``` +Let's create a quick `StageResult` class to represent +the results of staging operations, so we can report them +nicely in the UI and potentially use them in the future for more detailed reporting: + +```{python} +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) +``` + +Next, we can work on fetching the data from Globus. Fortunately, much of the +work has been done for us in the `customs` module, so +we just need to call the appropriate functions. + +```{python} +from stagecoach.customs import ( + Clearance, + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> bool: + """ + Stage data from Globus based on the manifest. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(globus_info, clearance) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) + +``` + +Next we implement symlink staging via FASRC: + +```{python} +from pathlib import Path +import glob +import os + +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) +``` + +That should do it for now! We will implement the dataverse staging +later on. + + diff --git a/notebooks/stagecoach.qmd b/notebooks/stagecoach.qmd index 136e362..7b3a6f0 100644 --- a/notebooks/stagecoach.qmd +++ b/notebooks/stagecoach.qmd @@ -70,8 +70,9 @@ the data to the user. import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer +from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console from rich.panel import Panel @@ -120,7 +121,7 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: + ) -> bool: """Create a StageCoach manifest.""" sheriff = sheriff or self.sheriff console = console or self.console @@ -128,6 +129,8 @@ class StageCoach: banner(console, "🚂 Hailing the Stagecoach...") + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") + issue_manifest( customs_sheriff=sheriff, interactive=interactive, @@ -136,13 +139,15 @@ class StageCoach: console=console, ) - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: + ) -> bool: """Inspect a StageCoach manifest.""" console = console or self.console @@ -151,16 +156,31 @@ class StageCoach: checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + gold_mine_items = gold_mine_info.get("items", []) + for item in gold_mine_items: + item_name = item.get("name", "Unnamed Item") + item_path = item.get("path_regex", "No path provided") + # if not Path(item_path).exists() or + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -169,27 +189,87 @@ class StageCoach: "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + banner(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + + """Stage data according to a StageCoach manifest.""" + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + banner(console, "⚠️ Staging failed. Please try again.") + return False + ``` Click on each section below to learn how stagecoach works in more detail, and to see examples of how to use it. diff --git a/src/stagecoach/cli.py b/src/stagecoach/cli.py index 9a1a79a..58a2f01 100644 --- a/src/stagecoach/cli.py +++ b/src/stagecoach/cli.py @@ -105,7 +105,12 @@ def inspect( def stage(): console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + ).stage() + if not staged: + raise typer.Exit(code=1) if __name__ == "__main__": diff --git a/src/stagecoach/customs.py b/src/stagecoach/customs.py index c01c563..992db39 100644 --- a/src/stagecoach/customs.py +++ b/src/stagecoach/customs.py @@ -1,98 +1,155 @@ +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) + + +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client + + +from dataclasses import dataclass, field +from typing import Any from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None - cleared: bool - -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +class Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Result of a Stagecoach source access check. """ + + source: str + cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + +def check_globus_clearance( + globus_info: dict + ) -> Clearance: try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) - - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) + + +def build_globus_transfer( + globus_info: dict, + clearance: Clearance + ) -> TransferData: + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label="Stagecoach transfer", + ) + + for item in globus_info["items"]: + transfer.add_item( + item["source_path"], + item["destination_path"], + recursive=item.get("recursive", True), + ) + + return transfer + + +import glob + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) diff --git a/src/stagecoach/issue_manifest.py b/src/stagecoach/issue_manifest.py index 6707fb0..e3f4418 100644 --- a/src/stagecoach/issue_manifest.py +++ b/src/stagecoach/issue_manifest.py @@ -4,7 +4,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: """Validate Frontier citizenship using the Sheriff.""" - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -12,7 +12,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") # mysheriff = Sheriff() @@ -25,84 +25,100 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: # loads it at runtime # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, - -# # critical: we must know where the frontier starts -# "root": "/n/holylabs/cgolden_lab/Lab/frontier" +# "root": "/n/holylabs/cgolden_lab/Lab/frontier", # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" +# "governance": "governance", # }, -# # we only have two compute options for now, but this may expand in the future -# "sources": { -# "02_globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None, -# "globus_files": [] -# }, -# "03_dataverse": { -# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets -# "use_dataverse": False, -# "dataverse_server_url": None, -# "dataverse_dataset_pid": None, -# "dataverse_dataset_version": "latest", -# "dataverse_api_token_file": None, -# "dataverse_files": [] -# }, -# "01_gold_mine": { -# # paths to data in the gold mine; only necessary if you want to access data in the gold mine -# "gold_mine_paths": [] -# } -# }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA + # "citizen": { -# # your name must match your name in town # "name": None, -# # email may be used in future for authentications -# "email": None +# "email": None, # }, + # "project": { -# # this must match the name of your project repository # "project_name": None, -# # must match above # "project_working_dir": str(here()), -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus -# "output_data_dir": str(here() / "data" / "outputs"), # "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# "sandbox_dir": str(here() / "sandbox") -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) import yaml @@ -210,12 +226,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") # from tempfile import TemporaryDirectory diff --git a/src/stagecoach/stage.py b/src/stagecoach/stage.py index d13ff38..1111a79 100644 --- a/src/stagecoach/stage.py +++ b/src/stagecoach/stage.py @@ -1,6 +1,9 @@ -from stagecoach.ui import info -from rich.console import Console import os +import re +import glob +from pathlib import Path +from stagecoach.ui import info, error, success, warning +from rich.console import Console def create_stage_directories(manifest: dict, console: Console, gitignore=True): @@ -12,37 +15,194 @@ def create_stage_directories(manifest: dict, console: Console, gitignore=True): intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") output_dir = manifest.get("project", {}).get("output_data_dir") sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") - if not os.path.exists(input_dir): - os.makedirs(input_dir) - info(console, f"Created input data directory at [bold]{input_dir}[/bold]") - else: - info(console, f"Input data directory already exists at [bold]{input_dir}[/bold]") - - if not os.path.exists(intermediate_dir): - os.makedirs(intermediate_dir) - info(console, f"Created intermediate data directory at [bold]{intermediate_dir}[/bold]") - else: - info(console, f"Intermediate data directory already exists at [bold]{intermediate_dir}[/bold]") - - if not os.path.exists(sandbox_dir): - os.makedirs(sandbox_dir) - info(console, f"Created sandbox directory at [bold]{sandbox_dir}[/bold]") - else: - info(console, f"Sandbox directory already exists at [bold]{sandbox_dir}[/bold]") - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - info(console, f"Created output data directory at [bold]{output_dir}[/bold]") - else: - info(console, f"Output data directory already exists at [bold]{output_dir}[/bold]") - - # if gitignore: - # paths = [''.join('.gitignore') for dir in [input_dir, intermediate_dir, output_dir, sandbox_dir]] - # gitignore_path = os.path.join(sandbox_dir, ".gitignore") - # if not os.path.exists(gitignore_path): - # with open(gitignore_path, "w") as f: - # f.write("*\n") - # info(console, f"Created .gitignore in sandbox directory at [bold]{gitignore_path}[/bold]") - # else: - # info(console, f".gitignore already exists in sandbox directory at [bold]{gitignore_path}[/bold]") + success(console, "Stage directories are set up.") + + +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + + +from stagecoach.customs import ( + Clearance, + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> bool: + """ + Stage data from Globus based on the manifest. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(globus_info, clearance) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) + + +from pathlib import Path +import glob +import os + +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) diff --git a/src/stagecoach/stagecoach.py b/src/stagecoach/stagecoach.py index 51302f0..a82822c 100644 --- a/src/stagecoach/stagecoach.py +++ b/src/stagecoach/stagecoach.py @@ -1,8 +1,9 @@ import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer +from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console from rich.panel import Panel @@ -51,7 +52,7 @@ def hail( sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: + ) -> bool: """Create a StageCoach manifest.""" sheriff = sheriff or self.sheriff console = console or self.console @@ -59,6 +60,8 @@ def hail( banner(console, "🚂 Hailing the Stagecoach...") + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") + issue_manifest( customs_sheriff=sheriff, interactive=interactive, @@ -67,13 +70,15 @@ def hail( console=console, ) - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: + ) -> bool: """Inspect a StageCoach manifest.""" console = console or self.console @@ -82,16 +87,31 @@ def inspect( checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + gold_mine_items = gold_mine_info.get("items", []) + for item in gold_mine_items: + item_name = item.get("name", "Unnamed Item") + item_path = item.get("path_regex", "No path provided") + # if not Path(item_path).exists() or + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -100,24 +120,83 @@ def inspect( "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + banner(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + + """Stage data according to a StageCoach manifest.""" + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + banner(console, "⚠️ Staging failed. Please try again.") + return False diff --git a/src/stagecoach/templates/manifest.yml b/src/stagecoach/templates/manifest.yml index 4495bb8..3624f93 100644 --- a/src/stagecoach/templates/manifest.yml +++ b/src/stagecoach/templates/manifest.yml @@ -1,37 +1,47 @@ -citizen: - email: null - name: null frontier: name: golden-lab - root: /n/holylabs/cgolden_lab/Lab/frontier schema_version: 1.0 + root: /n/holylabs/cgolden_lab/Lab/frontier paths: goldmine: goldmine - governance: governance - town: town works: works + town: town + governance: governance +citizen: + name: null + email: null project: + project_name: null + project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs intermediate_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/intermediates output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs - project_name: null - project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach sandbox_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/sandbox sources: 01_gold_mine: - gold_mine_paths: [] + enabled: false + items: + - name: example_goldmine_item + path_regex: null + required: true 02_globus: - globus_destination_endpoint: null - globus_destination_path: null - globus_files: [] - globus_source_endpoint: null - globus_source_path: null - globus_username: null - use_globus: false + enabled: false + source_endpoint: null + destination_endpoint: null + items: + - name: example_globus_item + source_path: null + destination_path: null + files_regex: [] + recursive: true + required: true 03_dataverse: - dataverse_api_token_file: null - dataverse_dataset_pid: null - dataverse_dataset_version: latest - dataverse_files: [] - dataverse_server_url: null - use_dataverse: false + enabled: false + server_url: null + dataset_pid: null + dataset_version: latest + api_token_file: null + items: + - name: example_dataverse_item + files_regex: [] + required: true diff --git a/stagecoach_manifest.yml b/stagecoach_manifest.yml index daeac4a..cc4902c 100644 --- a/stagecoach_manifest.yml +++ b/stagecoach_manifest.yml @@ -1,37 +1,47 @@ -citizen: - email: '' - name: tinashe frontier: name: golden-lab - root: /n/holylabs/cgolden_lab/Lab/frontier schema_version: 1.0 + root: /n/holylabs/cgolden_lab/Lab/frontier paths: goldmine: goldmine - governance: governance - town: town works: works + town: town + governance: governance +citizen: + name: tinashe + email: ttapera@hsph.harvard.edu project: - input_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/inputs - intermediate_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/intermediates - output_data_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/data/outputs project_name: stagecoach - project_working_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach - sandbox_dir: /n/holylabs/cgolden_lab/Lab/frontier/stagecoach/sandbox + project_working_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach + input_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/inputs + intermediate_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/intermediates + output_data_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/outputs + sandbox_dir: /n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/sandbox sources: 01_gold_mine: - gold_mine_paths: [] + enabled: true + items: + - name: example_goldmine_item + path_regex: "/n/holylabs/cgolden_lab/Lab/frontier/gold_mine/01_ore/golden_googledrive_rclone/CLTN/4. Datasets/Data" + required: true 02_globus: - globus_destination_endpoint: null - globus_destination_path: null - globus_files: [] - globus_source_endpoint: null - globus_source_path: null - globus_username: null - use_globus: false + enabled: true + source_endpoint: "1156ed9e-6984-11ea-af52-0201714f6eab" + destination_endpoint: "1156ed9e-6984-11ea-af52-0201714f6eab" + items: + - name: example_globus_item + source_path: "/n/holylabs/cgolden_lab/Lab/projects/era5_database/era5_sandbox/notes" + destination_path: "/n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/inputs/inst" + files_regex: ["*.ipynb"] + recursive: true + required: true 03_dataverse: - dataverse_api_token_file: null - dataverse_dataset_pid: null - dataverse_dataset_version: latest - dataverse_files: [] - dataverse_server_url: null - use_dataverse: false + enabled: false + server_url: null + dataset_pid: null + dataset_version: latest + api_token_file: null + items: + - name: example_dataverse_item + files_regex: [] + required: true diff --git a/user_guide/notebooks/cli.md b/user_guide/notebooks/cli.md index be54eef..15e5c5b 100644 --- a/user_guide/notebooks/cli.md +++ b/user_guide/notebooks/cli.md @@ -9,6 +9,8 @@ Once the CLI is defined as “hailing” the stagecoach, stagecoach will run `issue_manifest` to create the manifest, and then proceed with the rest of the stagecoach. +Here’s the code for the CLI. First, hailing the stagecoach: + ``` python from pathlib import Path from enum import Enum @@ -71,6 +73,8 @@ def hail( raise typer.Exit(code=1) ``` +Inspecting the manifest: + ``` python @app.command() def inspect( @@ -114,12 +118,19 @@ def inspect( raise typer.Exit(code=1) ``` +And, staging: + ``` python @app.command() def stage(): console = Console() customs_sheriff = Sheriff(console) - pass + staged = StageCoach( + sheriff=customs_sheriff, + console=console, + ).stage() + if not staged: + raise typer.Exit(code=1) ``` ``` python diff --git a/user_guide/notebooks/customs.md b/user_guide/notebooks/customs.md index 198801e..fefec5a 100644 --- a/user_guide/notebooks/customs.md +++ b/user_guide/notebooks/customs.md @@ -8,20 +8,22 @@ validation is going to be called “customs”. As a reminder, the works by issuing users a manifest. This manifest is a JSON file that users fill in with their credentials and desired data from the Frontier Gold Mine. Once the user returns the manifest, and the `Stagecoach` -recognizes that the user wants access to data via `globus`, and passes -this information to the customs handler. The customs handler then -validates the credentials to ensure that the user has the correct -permissions to access the data. This validation process uses the -`Globus SDK` to check the credentials against the Globus service. If the -credentials are valid, the customs handler returns a success message, -allowing the `Stagecoach` to proceed with the data delivery. If the -credentials are invalid, the handler returns an error message, and the -`Stagecoach` can inform the user of the issue and prompt them to correct -their credentials in the manifest. This process ensures that only -authorized users can access the data, maintaining the security of The -Frontier’s Gold Mine resources. - -## Tinkering with the SDK +recognizes that the user wants access to data via `globus`, `Dataverse`, +or from the Gold Mine, the `Stagecoach` will call the `customs` handler +and pass this information along. The customs handler then validates the +credentials to ensure that the user has the correct permissions to +access the data. This validation process uses the `Globus SDK`, +`Dataverse API`, or a few simple local file permission checks to verify +the credentials against the data provider service. If the credentials +are valid, the customs handler returns a success message, allowing the +`Stagecoach` to proceed with the data delivery. If the credentials are +invalid, the handler returns an error message, and the `Stagecoach` can +inform the user of the issue and prompt them to correct their +credentials in the manifest. This process ensures that only authorized +users can access the data, maintaining the security of The Frontier’s +Gold Mine resources. + +## Globus SDK Authentication and Validation Globus SDK in Python works as below. Once you’ve set up an App in the [Developer @@ -94,111 +96,196 @@ sheriff to validate the globus section of a stagecoach manifest. The manifest will have the following shape: -We’ll need to validate that the user has the kind of access necessary. -The return will be a dataclass that the stagecoach can accept the -requisite transfer object from and execute. +``` python +# import yaml +# from pathlib import Path +# from pprint import pprint +# from pyprojroot import here + +# manifest = yaml.safe_load(Path(here() / "stagecoach_manifest.yml").read_text()) +# pprint(manifest['sources']) +``` + +In the customs manager, we will have to do two things: authenticate with +the Globus client, and send the actual data. After wrestling with this +for a few tries, and bouncing design ideas off of chatgpt and jupyter +notebooks, I’ve settled on adding a context helper to do the UserApp +creation: ``` python +from globus_sdk import GlobusAppConfig, TransferClient, UserApp, TransferData +from contextlib import contextmanager + +@contextmanager +def globus_transfer_client(): + with UserApp( + "Frontier-Stagecoach", + client_id="7723dff4-fa63-4639-903b-ba6541e24e98", + config=GlobusAppConfig(auto_redrive_gares=True), + ) as app: + with TransferClient(app=app) as client: + yield client +``` + +This will allow us to create transfer clients with the `with` statement, +allowing us to iterate over the items in the Globus request +individually, granting each of them a unique request and clearance: + +``` python +from dataclasses import dataclass, field +from typing import Any from rich.console import Console -from globus_sdk import GlobusAppConfig, UserApp, TransferClient, GlobusAPIError, TransferData -from dataclasses import dataclass @dataclass -class GlobusClearance: - transfer_id : dict | None - source_collection: str | None - destination_collection: str | None - cleared: bool - -def issue_globus_transfer( - globus_info: dict, - console: Console, - stagecoach_app_id: str = "7723dff4-fa63-4639-903b-ba6541e24e98", - issue_transfer: bool = False - ) -> GlobusClearance: +class Clearance: """ - Use the Globus SDK to validate globus credentials and optionally issue a transfer. - Args: - globus_info (dict): A dictionary containing the globus credentials and information. - console (Console): Rich console object for output messages. - stagecoach_app_id (str): The Globus application client ID for authentication. - issue_transfer (bool): Whether to actually issue a transfer after validation. Defaults to False. - Returns: - GlobusClearance: A dataclass indicating the validation result and associated objects. + Result of a Stagecoach source access check. """ + + source: str + cleared: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + +def check_globus_clearance( + globus_info: dict + ) -> Clearance: try: - - if not globus_info.get("use_globus", False): - console.print("Globus access not requested.") - return GlobusClearance( - transfer_id = None, - source_collection = None, - destination_collection = None, - cleared = True + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + for item in globus_info["items"]: + client.operation_stat( + globus_info["source_endpoint"], + path=item["source_path"], + ) + + return Clearance( + source="02_globus", + cleared=True, + message="Globus clearance passed.", + details={ + "source_endpoint": globus_info["source_endpoint"], + "destination_endpoint": globus_info["destination_endpoint"], + "items_checked": len(globus_info["items"]) + } ) - - app_name = "Frontier-Customs_" + globus_info.get("globus_username", "Globus-Validator") - - with UserApp( - app_name, - client_id=stagecoach_app_id, - config=GlobusAppConfig(auto_redrive_gares=True), - ) as app: - with TransferClient(app=app) as client: - src_collection = globus_info.get("globus_source_endpoint") - dst_collection = globus_info.get("globus_destination_endpoint") - src_path = globus_info.get("globus_source_path") - dst_path = globus_info.get("globus_destination_path") - - client.add_app_data_access_scope(src_collection) - client.add_app_data_access_scope(dst_collection) - - transfer_request = TransferData(src_collection, dst_collection) - transfer_request.add_item(src_path, dst_path) - - # Attempt to stat the source collection to validate access - resp = client.operation_stat(src_collection, src_path) - if resp.http_status == 200: - console.print("✅ Globus credentials validated successfully.") - else: - console.print(f"❌ Failed to validate globus credentials. Operation stat returned status: {resp.http_status}") - raise GlobusAPIError(resp) - - if issue_transfer: - task = client.submit_transfer(transfer_request) - console.print(f"Submitted transfer. Task ID: {task['task_id']}.") - return GlobusClearance( - transfer_id = task, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True - ) - else: - return GlobusClearance( - transfer_id = None, - source_collection = src_collection, - destination_collection = dst_collection, - cleared = True + + except Exception as exc: + return Clearance( + source="02_globus", + cleared=False, + message=str(exc), + details={ + "source_endpoint": globus_info.get("source_endpoint"), + "destination_endpoint": globus_info.get("destination_endpoint"), + } + ) +``` + +If the checks pass, we can move on to building the transfer logic. This +is built mostly on what we learned above: + +``` python +def build_globus_transfer( + globus_info: dict, + clearance: Clearance + ) -> TransferData: + if not clearance.cleared: + raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( + source_endpoint=globus_info["source_endpoint"], + destination_endpoint=globus_info["destination_endpoint"], + label="Stagecoach transfer", + ) + + for item in globus_info["items"]: + transfer.add_item( + item["source_path"], + item["destination_path"], + recursive=item.get("recursive", True), + ) + + return transfer +``` + +Now with that, the customs manager can first check for clearance, and +additionally build the transfer behind the scenes. + +## Gold Mine + +Accessing the Gold Mine can only be done from FASRC, for now. It will +implement a simple check for whether the user has access to the +specified path. This will reuse the `Clearance` class from earlier: + +``` python +import glob + +def check_gold_mine_clearance( + gold_mine_info: dict + ) -> Clearance: + + try: + items = gold_mine_info.get("items", []) + checked = [] + + for item in items: + name = item.get("name", "unnamed") + path_regex = item.get("path_regex") + + if not path_regex: + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: missing path_regex", + ) + + matches = [Path(p) for p in glob.glob(path_regex)] + + if not matches and item.get("required", True): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: no matches for {path_regex}", + ) + + for match in matches: + if not os.access(match, os.R_OK): + return Clearance( + source="01_gold_mine", + cleared=False, + message=f"{name}: path is not readable: {match}", ) - except GlobusAPIError as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False if there's an API error, along with a clearance object indicating failure - except Exception as e: - console.print_exception() - return GlobusClearance( - transfer_id = None, - source_collection = globus_info.get("globus_source_endpoint", None), - destination_collection = globus_info.get("globus_destination_endpoint", None), - cleared = False - ) # Return False for any other unexpected errors, along with a clearance object indicating failure + checked.append( + { + "name": name, + "path_regex": path_regex, + "matches": [str(p) for p in matches], + } + ) + + return Clearance( + source="01_gold_mine", + cleared=True, + message="Gold Mine clearance passed.", + details={"items_checked": checked}, + ) + + except Exception as exc: + return Clearance( + source="01_gold_mine", + cleared=False, + message=str(exc), + ) ``` +## Dataverse + +🚧 Not yet implemented 🚧 + +## Conclusion + When that sends back a clearance object with `cleared=True`, the `Stagecoach` can proceed with the transfer using the provided `transfer_client`. If `cleared=False`, the `Stagecoach` can inform the diff --git a/user_guide/notebooks/issue_manifest.md b/user_guide/notebooks/issue_manifest.md index df3920c..420de0d 100644 --- a/user_guide/notebooks/issue_manifest.md +++ b/user_guide/notebooks/issue_manifest.md @@ -29,7 +29,7 @@ from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: """Validate Frontier citizenship using the Sheriff.""" - console.print("[bold]Checking citizenship...[/bold]") + info(console, "Checking citizenship...") if not customs_sheriff.check_citizen(): raise RuntimeError( @@ -37,7 +37,7 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: "Please check your citizenship and try again." ) - console.print("[green]✔ Citizenship verified[/green]\n") + success(console, "Citizenship verified") ``` We can see this function in action below: @@ -61,84 +61,100 @@ and figure out how to get the data to the user’s specified location. # loads it at runtime # from pyprojroot import here -# import os -# import yaml -# manifest = { +# manifest_template = { # "frontier": { -# # the name of the frontier may be used in future # "name": "golden-lab", -# # the version may be used in future # "schema_version": 1.0, - -# # critical: we must know where the frontier starts -# "root": "/n/holylabs/cgolden_lab/Lab/frontier" +# "root": "/n/holylabs/cgolden_lab/Lab/frontier", # }, + # "paths": { -# # this is the general structure of the frontier # "goldmine": "goldmine", # "works": "works", # "town": "town", -# "governance": "governance" +# "governance": "governance", # }, -# # we only have two compute options for now, but this may expand in the future -# "sources": { -# "02_globus": { -# # globus credentials for ferrying data to and from the frontier; only necessary if you want to ferry data to and from the frontier via globus -# "use_globus": False, -# "globus_username": None, -# "globus_source_endpoint": None, -# "globus_source_path": None, -# "globus_destination_endpoint": None, -# "globus_destination_path": None, -# "globus_files": [] -# }, -# "03_dataverse": { -# # dataverse credentials for accessing dataverse datasets; only necessary if you want to access dataverse datasets -# "use_dataverse": False, -# "dataverse_server_url": None, -# "dataverse_dataset_pid": None, -# "dataverse_dataset_version": "latest", -# "dataverse_api_token_file": None, -# "dataverse_files": [] -# }, -# "01_gold_mine": { -# # paths to data in the gold mine; only necessary if you want to access data in the gold mine -# "gold_mine_paths": [] -# } -# }, -# # FILL IN THE BELOW SECTIONS WITH THE NECESSARY INFORMATION TO GET ACCESS TO THE DATA + # "citizen": { -# # your name must match your name in town # "name": None, -# # email may be used in future for authentications -# "email": None +# "email": None, # }, + # "project": { -# # this must match the name of your project repository # "project_name": None, -# # must match above # "project_working_dir": str(here()), -# # this is the name of the folder that stagecoach will ferry -# # your input data to # "input_data_dir": str(here() / "data" / "inputs"), -# # this is the name of the folder that stagecoach -# # will ferry your output data from; -# # only necessary if you want to ferry output data back to the frontier, -# # eg via globus -# "output_data_dir": str(here() / "data" / "outputs"), # "intermediate_data_dir": str(here() / "data" / "intermediates"), # "output_data_dir": str(here() / "data" / "outputs"), -# "sandbox_dir": str(here() / "sandbox") -# } +# "sandbox_dir": str(here() / "sandbox"), +# }, + +# "sources": { +# "01_gold_mine": { +# "enabled": False, + +# # Each item stages into: +# # /01_gold_mine// +# "items": [ +# { +# "name": "example_goldmine_item", +# "path_regex": None, +# "required": True, +# } +# ], +# }, + +# "02_globus": { +# "enabled": False, + +# # Globus collection UUIDs +# "source_endpoint": None, +# "destination_endpoint": None, + +# # Each item stages into: +# # /02_globus// +# "items": [ +# { +# "name": "example_globus_item", +# "source_path": None, +# "destination_path": None, # optional override +# "files_regex": [], +# "recursive": True, +# "required": True, +# } +# ], +# }, + +# "03_dataverse": { +# "enabled": False, + +# "server_url": None, +# "dataset_pid": None, +# "dataset_version": "latest", +# "api_token_file": None, + +# # Each item stages into: +# # /03_dataverse// +# "items": [ +# { +# "name": "example_dataverse_item", +# "files_regex": [], +# "required": True, +# } +# ], +# }, +# }, # } -# yaml_manifest = yaml.dump(manifest) +# import yaml +# from pathlib import Path -# output_path = here() / "src" / "stagecoach" / "templates" / "manifest.yml" +# output_path = Path("src/stagecoach/templates/manifest.yml") +# output_path.parent.mkdir(parents=True, exist_ok=True) -# with open(output_path, "w") as f: -# f.write(yaml_manifest) +# with output_path.open("w") as f: +# yaml.dump(manifest_template, f, sort_keys=False) ``` If a citizen can pass the citizenship check, then the stagecoach can @@ -275,12 +291,12 @@ def write_manifest( raise RuntimeError(f"Manifest not written: {output_path} already exists.") output_path.parent.mkdir(parents=True, exist_ok=True) - console.print(f"[bold]Writing manifest to:[/bold] {output_path}") + info(console, f"Writing manifest to: {output_path}") with console.status("[bold green]Saving manifest..."): with output_path.open("w") as f: yaml.dump(manifest, f, sort_keys=False) - console.print("[green]✔ Manifest created successfully[/green]\n") + success(console, "Manifest created successfully") ``` To test: diff --git a/user_guide/notebooks/stage.md b/user_guide/notebooks/stage.md index b915459..7546356 100644 --- a/user_guide/notebooks/stage.md +++ b/user_guide/notebooks/stage.md @@ -8,9 +8,12 @@ There are a couple of things that will need to happen to make this work. One of them will be creating the directories for data usage: ``` python -from stagecoach.ui import info -from rich.console import Console import os +import re +import glob +from pathlib import Path +from stagecoach.ui import info, error, success, warning +from rich.console import Console ``` ``` python @@ -23,42 +26,215 @@ def create_stage_directories(manifest: dict, console: Console, gitignore=True): intermediate_dir = manifest.get("project", {}).get("intermediate_data_dir") output_dir = manifest.get("project", {}).get("output_data_dir") sandbox_dir = manifest.get("project", {}).get("sandbox_dir") + + paths = [input_dir, intermediate_dir, output_dir, sandbox_dir] + + for p in paths: + if p is None: + error(console, f"Stage directory {p} is not defined in the manifest.") + raise ValueError("One or more stage directories are not defined in the manifest.") + if not os.path.exists(p): + os.makedirs(p) + info(console, f"Created stage directory at {p}") + else: + info(console, f"Stage directory already exists at {p}") + + if gitignore: + gitignore_path = Path(p) / ".gitignore" + if not gitignore_path.exists(): + with open(gitignore_path, "w") as f: + f.write("*\n!.gitignore\n") + info(console, f"Created .gitignore in {p}") + else: + info(console, f".gitignore already exists in {p}") - if not os.path.exists(input_dir): - os.makedirs(input_dir) - info(console, f"Created input data directory at [bold]{input_dir}[/bold]") - else: - info(console, f"Input data directory already exists at [bold]{input_dir}[/bold]") - - if not os.path.exists(intermediate_dir): - os.makedirs(intermediate_dir) - info(console, f"Created intermediate data directory at [bold]{intermediate_dir}[/bold]") - else: - info(console, f"Intermediate data directory already exists at [bold]{intermediate_dir}[/bold]") - - if not os.path.exists(sandbox_dir): - os.makedirs(sandbox_dir) - info(console, f"Created sandbox directory at [bold]{sandbox_dir}[/bold]") - else: - info(console, f"Sandbox directory already exists at [bold]{sandbox_dir}[/bold]") - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - info(console, f"Created output data directory at [bold]{output_dir}[/bold]") - else: - info(console, f"Output data directory already exists at [bold]{output_dir}[/bold]") - - # if gitignore: - # paths = [''.join('.gitignore') for dir in [input_dir, intermediate_dir, output_dir, sandbox_dir]] - # gitignore_path = os.path.join(sandbox_dir, ".gitignore") - # if not os.path.exists(gitignore_path): - # with open(gitignore_path, "w") as f: - # f.write("*\n") - # info(console, f"Created .gitignore in sandbox directory at [bold]{gitignore_path}[/bold]") - # else: - # info(console, f".gitignore already exists in sandbox directory at [bold]{gitignore_path}[/bold]") + success(console, "Stage directories are set up.") +``` + +Let’s create a quick `StageResult` class to represent the results of +staging operations, so we can report them nicely in the UI and +potentially use them in the future for more detailed reporting: + +``` python +from dataclasses import dataclass, field +from typing import Any + +@dataclass +class StageResult: + source: str + staged: bool + message: str = "" + details: dict[str, Any] = field(default_factory=dict) +``` + +Next, we can work on fetching the data from Globus. Fortunately, much of +the work has been done for us in the `customs` module, so we just need +to call the appropriate functions. + +``` python +from stagecoach.customs import ( + Clearance, + check_globus_clearance, + build_globus_transfer, + globus_transfer_client, + + check_gold_mine_clearance +) + +def stage_data_from_globus( + manifest: dict, + console: Console, + issue_transfer: bool = True +) -> bool: + """ + Stage data from Globus based on the manifest. + """ + + if not issue_transfer: + info(console, "Globus clearance check passed. Skipping transfer as per configuration.") + return StageResult( + source="02_globus", + staged=False, + message="Globus clearance passed. Transfer skipped.", + ) + + globus_info = manifest.get("sources", {}).get("02_globus", {}) + if not globus_info: + error(console, "No Globus information found in the manifest.") + raise ValueError("Globus information is missing from the manifest.") + clearance = check_globus_clearance(globus_info) + transfer = build_globus_transfer(globus_info, clearance) + try: + with globus_transfer_client() as client: + client.get_endpoint(globus_info["source_endpoint"]) + client.get_endpoint(globus_info["destination_endpoint"]) + task = client.submit_transfer(transfer) + info(console, f"Globus transfer submitted with task ID: {task['task_id']}") + + return StageResult( + source="02_globus", + staged=True, + message=f"Globus transfer submitted with task ID: {task['task_id']}", + details=task, + ) + except Exception as exc: + error(console, f"Error during Globus staging: {exc}") + return StageResult( + source="02_globus", + staged=False, + message="Globus staging failed with transfer client error.", + details=exc, + ) ``` +Next we implement symlink staging via FASRC: + +``` python +from pathlib import Path +import glob +import os + +def stage_data_from_fasrc( + manifest: dict, + console: Console, +) -> StageResult: + fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) + items = fasrc_info.get("items", []) + + task_list = { + "succeeded": [], + "failed": [], + "skipped": [], + } + + if not items: + return StageResult( + source="01_gold_mine", + staged=False, + message="No FASRC source items found in the manifest.", + details=task_list, + ) + + try: + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "01_gold_mine" + source_root.mkdir(parents=True, exist_ok=True) + + for item in items: + item_name = item.get("name") + path_regex = item.get("path_regex") + + if not item_name or not path_regex: + task_list["failed"].append(item) + continue + + info(console, f"Staging Gold Mine item: {item_name}") + + item_stage_dir = source_root / item_name + item_stage_dir.mkdir(parents=True, exist_ok=True) + + files_found = glob.glob(path_regex) + + if not files_found: + warning(console, f"No files found matching pattern: {path_regex}") + task_list["skipped"].append(path_regex) + continue + + for source_path_raw in files_found: + source_path = Path(source_path_raw).resolve() + destination_path = item_stage_dir / source_path.name + + if destination_path.exists() or destination_path.is_symlink(): + existing_target = None + + if destination_path.is_symlink(): + existing_target = Path(os.readlink(destination_path)).resolve() + + if existing_target == source_path: + task_list["skipped"].append(str(source_path)) + continue + + task_list["failed"].append(str(source_path)) + continue + + try: + os.symlink( + source_path, + destination_path, + target_is_directory=source_path.is_dir(), + ) + task_list["succeeded"].append(str(source_path)) + + except OSError: + task_list["failed"].append(str(source_path)) + + except Exception as exc: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine staging failed: {exc}", + details=task_list, + ) + + staged = len(task_list["succeeded"]) > 0 or len(task_list["skipped"]) > 0 + failed = len(task_list["failed"]) > 0 + + return StageResult( + source="01_gold_mine", + staged=staged and not failed, + message=( + "Symlink staging summary: " + f"{len(task_list['succeeded'])} succeeded, " + f"{len(task_list['skipped'])} skipped, " + f"{len(task_list['failed'])} failed." + ), + details=task_list, + ) +``` + +That should do it for now! We will implement the dataverse staging later +on. + # Script file The code for this document can be found here: diff --git a/user_guide/notebooks/stagecoach.md b/user_guide/notebooks/stagecoach.md index 76cbe1b..0aeaa37 100644 --- a/user_guide/notebooks/stagecoach.md +++ b/user_guide/notebooks/stagecoach.md @@ -53,8 +53,9 @@ defined in the `stagecoach` class: import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import GlobusClearance, issue_globus_transfer +from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console from rich.panel import Panel @@ -103,7 +104,7 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, - ) -> None: + ) -> bool: """Create a StageCoach manifest.""" sheriff = sheriff or self.sheriff console = console or self.console @@ -111,6 +112,8 @@ class StageCoach: banner(console, "🚂 Hailing the Stagecoach...") + banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") + issue_manifest( customs_sheriff=sheriff, interactive=interactive, @@ -119,13 +122,15 @@ class StageCoach: console=console, ) - banner(console, f"Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") + + return True def inspect( self, console: Console | None = None, level: Severity = Severity.ERROR - ) -> None: + ) -> bool: """Inspect a StageCoach manifest.""" console = console or self.console @@ -134,16 +139,31 @@ class StageCoach: checker = ManifestChecker(self.manifest_path, level) checks = checker.run_all() + for result in checks: + check_result(console, result) + manifest = yaml.safe_load(self.manifest_path.read_text()) - if manifest.get("remote", {}).get("globus", {}).get("use_globus"): + # check gold mine access + if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): + info(console, "Gold Mine access requested. Checking with customs...") + gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) + gold_mine_items = gold_mine_info.get("items", []) + for item in gold_mine_items: + item_name = item.get("name", "Unnamed Item") + item_path = item.get("path_regex", "No path provided") + # if not Path(item_path).exists() or + + # check globus + if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): info(console, "Globus access requested. Checking with customs...") - clearance = issue_globus_transfer( - globus_info=manifest.get("remote", {}).get("globus", {}), - console=console, - issue_transfer=False, - ) + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + clearance = check_globus_clearance( + globus_info=globus_info + ) + if clearance.cleared: success(console, "Globus credentials validated.") else: @@ -152,27 +172,86 @@ class StageCoach: "Globus credentials validation failed. Please check your credentials.", ) console.print(clearance) - - return False - - for result in checks: - check_result(console, result) + raise ValueError("Globus credentials validation failed.") + # check dataverse + if manifest.get("sources", {}).get("03_dataverse", {}).get("enabled"): + info(console, "Dataverse access requested. Checking with customs...") + warning(console, "Dataverse access checks are not yet implemented. Please ensure you have access to the Dataverse dataset before proceeding.") if not checker.passes(): console.print() - error(console, "Manifest checks failed. Please review the results above.") + banner(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False console.print() - success(console, "Manifest checks passed.") + banner(console, "✅ Manifest checks passed, you're cleared for staging! Run `stagecoach stage` to bring the data into your environment.") return True - def stage(self): - pass + def stage( + self, + console: Console | None = None, + ) -> bool: + + """Stage data according to a StageCoach manifest.""" + console = console or self.console + banner(console, "🚂 Staging the data...") + + manifest = yaml.safe_load(self.manifest_path.read_text()) + + # create stage directories + create_stage_directories(manifest, console, gitignore=True) + + # fasrc + fasrc = manifest.get("sources", {}).get("01_gold_mine", {}) + if fasrc['enabled']: + fasrc_stageresult = stage_data_from_fasrc(manifest, console) + if fasrc_stageresult.staged: + success(console, "Gold Mine data staged successfully.") + else: + error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + + # globus + globus = manifest.get("sources", {}).get("02_globus", {}) + if globus['enabled']: + globus_stageresult = stage_data_from_globus(manifest, console) + if globus_stageresult.staged: + success(console, "Globus data staged successfully.") + else: + error(console, f"Globus staging failed: {globus_stageresult.message}") + + # dataverse + dataverse = manifest.get("sources", {}).get("03_dataverse", {}) + if dataverse['enabled']: + warning(console, "Dataverse staging is not yet implemented. Please stage the data manually and place it in the appropriate directory.") + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging is not yet implemented.", + ) + else: + dataverse_stageresult = StageResult( + source="03_dataverse", + staged=True, + message="Dataverse staging not requested.", + ) + + final_staging = all( + result.staged + for result in [fasrc_stageresult, globus_stageresult, dataverse_stageresult] + ) + + if final_staging: + banner(console, "🪎 All data staged successfully! You're ready to start working!") + return True + else: + error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") + + banner(console, "⚠️ Staging failed. Please try again.") + return False ``` Click on each section below to learn how stagecoach works in more From 1086d795eed1bcc63248130ac11e2484bad89f98 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 02:10:37 -0400 Subject: [PATCH 03/10] feat(manifest_checker): enhance manifest validation checks - Update the ManifestChecker class to improve documentation and clarify parameters. - Add detailed descriptions for methods and attributes. - Ensure structured results are returned for better inspection output. feat(stage): improve staging operations and documentation - Refactor create_stage_directories to clarify its purpose and parameters. - Enhance stage_data_from_globus and stage_data_from_fasrc with detailed docstrings. - Introduce StageResult for summarizing staging outcomes. feat(stagecoach): refine StageCoach class and methods - Update StageCoach to improve manifest issuance and inspection processes. - Add detailed parameter descriptions for methods. - Enhance error handling and user feedback during staging. docs(ui): improve UI message functions documentation - Add docstrings to UI functions for better clarity on usage. - Ensure all message functions have consistent documentation style. docs(checks): enhance checks documentation - Improve docstrings for check functions to clarify their purpose and return types. - Ensure all checks are well-documented for user understanding. docs(customs): clarify customs checks and clearance processes - Add detailed descriptions for clearance checks and their parameters. - Ensure that all customs-related functions are well-documented. docs(issue_manifest): enhance manifest issue process documentation - Improve documentation for the issue_manifest function and its parameters. - Ensure clarity on the process of generating a Stagecoach manifest. docs(manifest_checker): improve documentation for manifest checks - Update the ManifestChecker class documentation to clarify its purpose and functionality. - Ensure all methods are well-documented for better user understanding. docs(stage): enhance staging documentation - Improve documentation for staging functions to clarify their purpose and parameters. - Ensure all staging-related functions are well-documented for user understanding. --- notebooks/checks.qmd | 121 ++++++++++++++++++++++- notebooks/cli.qmd | 38 ++++++- notebooks/customs.qmd | 71 ++++++++++++- notebooks/issue_manifest.qmd | 83 ++++++++++++---- notebooks/manifest_checker.qmd | 39 ++++++-- notebooks/stage.qmd | 89 ++++++++++++++--- notebooks/stagecoach.qmd | 90 ++++++++++++++--- notebooks/ui.qmd | 94 +++++++++++++++++- src/stagecoach/checks.py | 120 +++++++++++++++++++++- src/stagecoach/cli.py | 37 ++++++- src/stagecoach/customs.py | 71 ++++++++++++- src/stagecoach/issue_manifest.py | 82 ++++++++++++--- src/stagecoach/manifest_checker.py | 38 +++++-- src/stagecoach/stage.py | 87 ++++++++++++++-- src/stagecoach/stagecoach.py | 89 ++++++++++++++--- src/stagecoach/ui.py | 92 +++++++++++++++++ user_guide/notebooks/checks.md | 120 +++++++++++++++++++++- user_guide/notebooks/cli.md | 37 ++++++- user_guide/notebooks/customs.md | 71 ++++++++++++- user_guide/notebooks/issue_manifest.md | 82 ++++++++++++--- user_guide/notebooks/manifest_checker.md | 38 +++++-- user_guide/notebooks/stage.md | 87 ++++++++++++++-- user_guide/notebooks/stagecoach.md | 89 ++++++++++++++--- user_guide/notebooks/ui.md | 92 +++++++++++++++++ 24 files changed, 1684 insertions(+), 173 deletions(-) diff --git a/notebooks/checks.qmd b/notebooks/checks.qmd index 8c44bb8..51c6b2d 100644 --- a/notebooks/checks.qmd +++ b/notebooks/checks.qmd @@ -60,6 +60,18 @@ from enum import Enum from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -80,7 +92,18 @@ PRINCIPLES = { @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -110,6 +133,19 @@ Now, let's define a few checks: ```{python} def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -126,9 +162,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. + + Parameters + ---------- + directory : Path + Project directory to inspect. - Pass/Fail, no exceptions. + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -146,6 +190,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -186,6 +244,20 @@ started writing code: #| sorting-hat: keep def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -240,6 +312,20 @@ of things like READMEs, notebooks, or markdown files: ```{python} def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -285,6 +371,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", @@ -312,6 +412,20 @@ or a Python project structure with a `src` directory. #| sorting-hat: keep #| def check_project_structure(directory: Path) -> CheckResult: + """ + Check for a minimal project scaffold. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when the project contains either an R project file or a + Python-style ``src`` directory. + """ has_rproj = (directory / "project.Rproj").exists() has_src = (directory / "src").exists() and (directory / "src").is_dir() @@ -336,4 +450,3 @@ def check_project_structure(directory: Path) -> CheckResult: In the next module, we define the ManifestChecker class, which will run all of these checks and more, and return a structured report that can be displayed in `stagecoach inspect`. - diff --git a/notebooks/cli.qmd b/notebooks/cli.qmd index 56e4d8c..1bec0f0 100644 --- a/notebooks/cli.qmd +++ b/notebooks/cli.qmd @@ -37,11 +37,10 @@ Here's the code for the CLI. First, hailing the stagecoach: from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -51,6 +50,16 @@ app = typer.Typer( ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -75,6 +84,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -119,6 +137,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -145,6 +170,14 @@ And, staging: ```{python} @app.command() def stage(): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) staged = StageCoach( @@ -161,4 +194,3 @@ def stage(): if __name__ == "__main__": app() ``` - diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index b7f3932..061e140 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -208,6 +208,14 @@ from contextlib import contextmanager @contextmanager def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ with UserApp( "Frontier-Stagecoach", client_id="7723dff4-fa63-4639-903b-ba6541e24e98", @@ -225,12 +233,22 @@ them a unique request and clearance: ```{python} from dataclasses import dataclass, field from typing import Any -from rich.console import Console @dataclass class Clearance: """ - Result of a Stagecoach source access check. + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. """ source: str @@ -241,6 +259,19 @@ class Clearance: def check_globus_clearance( globus_info: dict ) -> Clearance: + """ + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. + """ try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) @@ -283,6 +314,26 @@ def build_globus_transfer( globus_info: dict, clearance: Clearance ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") transfer = TransferData( @@ -314,10 +365,26 @@ from earlier: ```{python} import glob +import os +from pathlib import Path def check_gold_mine_clearance( gold_mine_info: dict ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ try: items = gold_mine_info.get("items", []) diff --git a/notebooks/issue_manifest.qmd b/notebooks/issue_manifest.qmd index ac850ec..bba1d5b 100644 --- a/notebooks/issue_manifest.qmd +++ b/notebooks/issue_manifest.qmd @@ -61,7 +61,21 @@ from rich.console import Console from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ info(console, "Checking citizenship...") @@ -211,7 +225,14 @@ from typing import Any from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -236,8 +257,19 @@ import questionary from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. - """Prompt the user and fill in manifest fields.""" + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -312,7 +344,6 @@ Next, writing the manifest should be straight forward: ```{python} #| sorting-hat: keep -import questionary from pathlib import Path from rich.console import Console @@ -322,7 +353,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -366,7 +415,6 @@ Looks great! We now have the pieces to issue a manifest: #| sorting-hat: keep from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -380,23 +428,25 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) @@ -419,4 +469,3 @@ to ensure they have correct access to the Gold Mine. Next, check out the CLI module to see how we "hail," a stagecoach. - diff --git a/notebooks/manifest_checker.qmd b/notebooks/manifest_checker.qmd index 0a25252..56ffb1c 100644 --- a/notebooks/manifest_checker.qmd +++ b/notebooks/manifest_checker.qmd @@ -30,10 +30,23 @@ from stagecoach.checks import * class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -42,7 +55,12 @@ class ManifestChecker: def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -57,11 +75,13 @@ class ManifestChecker: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() @@ -87,4 +107,3 @@ else: ``` - diff --git a/notebooks/stage.qmd b/notebooks/stage.qmd index 4ee5363..3731ac8 100644 --- a/notebooks/stage.qmd +++ b/notebooks/stage.qmd @@ -26,7 +26,6 @@ directories for data usage: ```{python} import os -import re import glob from pathlib import Path from stagecoach.ui import info, error, success, warning @@ -36,7 +35,22 @@ from rich.console import Console ```{python} def create_stage_directories(manifest: dict, console: Console, gitignore=True): """ - Create the stage directories if they don't exist. + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. """ input_dir = manifest.get("project", {}).get("input_data_dir") @@ -79,6 +93,20 @@ from typing import Any @dataclass class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ source: str staged: bool message: str = "" @@ -91,7 +119,7 @@ we just need to call the appropriate functions. ```{python} from stagecoach.customs import ( - Clearance, + check_globus_clearance, build_globus_transfer, globus_transfer_client, @@ -103,9 +131,29 @@ def stage_data_from_globus( manifest: dict, console: Console, issue_transfer: bool = True -) -> bool: +) -> StageResult: """ - Stage data from Globus based on the manifest. + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. """ if not issue_transfer: @@ -149,14 +197,25 @@ def stage_data_from_globus( Next we implement symlink staging via FASRC: ```{python} -from pathlib import Path -import glob -import os - def stage_data_from_fasrc( manifest: dict, console: Console, ) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) items = fasrc_info.get("items", []) @@ -166,6 +225,16 @@ def stage_data_from_fasrc( "skipped": [], } + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + if not items: return StageResult( source="01_gold_mine", @@ -253,5 +322,3 @@ def stage_data_from_fasrc( That should do it for now! We will implement the dataverse staging later on. - - diff --git a/notebooks/stagecoach.qmd b/notebooks/stagecoach.qmd index 7b3a6f0..9625ff9 100644 --- a/notebooks/stagecoach.qmd +++ b/notebooks/stagecoach.qmd @@ -70,12 +70,11 @@ the data to the user. import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -91,7 +90,16 @@ from stagecoach.ui import ( class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -122,7 +130,27 @@ class StageCoach: console: Console | None = None, manifest_path: str | Path | None = None, ) -> bool: - """Create a StageCoach manifest.""" + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path @@ -148,7 +176,22 @@ class StageCoach: console: Console | None = None, level: Severity = Severity.ERROR ) -> bool: - """Inspect a StageCoach manifest.""" + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -165,11 +208,18 @@ class StageCoach: if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): info(console, "Gold Mine access requested. Checking with customs...") gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) - gold_mine_items = gold_mine_info.get("items", []) - for item in gold_mine_items: - item_name = item.get("name", "Unnamed Item") - item_path = item.get("path_regex", "No path provided") - # if not Path(item_path).exists() or + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + # check globus if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): @@ -197,7 +247,7 @@ class StageCoach: if not checker.passes(): console.print() - banner(console, "⚠️ Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False @@ -212,8 +262,19 @@ class StageCoach: self, console: Console | None = None, ) -> bool: - - """Stage data according to a StageCoach manifest.""" + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ console = console or self.console banner(console, "🚂 Staging the data...") @@ -267,7 +328,7 @@ class StageCoach: else: error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") - banner(console, "⚠️ Staging failed. Please try again.") + failure_panel(console, "⚠️ Staging failed. Please try again.") return False ``` @@ -275,4 +336,3 @@ class StageCoach: Click on each section below to learn how stagecoach works in more detail, and to see examples of how to use it. 1. [Manifest File](#manifest-file) - diff --git a/notebooks/ui.qmd b/notebooks/ui.qmd index 853b79e..80353ea 100644 --- a/notebooks/ui.qmd +++ b/notebooks/ui.qmd @@ -39,6 +39,16 @@ HANDBOOK_URL = ( def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -49,22 +59,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -74,6 +134,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -81,6 +149,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -88,6 +169,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: @@ -110,4 +202,4 @@ def check_result(console: Console, result) -> None: f"[dim]principle {principle}[/dim]\n" f" {escape(result.message)}" ) -``` \ No newline at end of file +``` diff --git a/src/stagecoach/checks.py b/src/stagecoach/checks.py index 390e313..3fe114f 100644 --- a/src/stagecoach/checks.py +++ b/src/stagecoach/checks.py @@ -4,6 +4,18 @@ from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -24,7 +36,18 @@ class Severity(Enum): @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -43,6 +66,19 @@ class CheckResult: def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -59,9 +95,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. - Pass/Fail, no exceptions. + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -79,6 +123,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -110,6 +168,20 @@ def check_environment_exists(directory: Path) -> CheckResult: def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -160,6 +232,20 @@ def check_code_exists(directory: Path) -> CheckResult: def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -205,6 +291,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", @@ -225,6 +325,20 @@ def check_readme_exists(directory: Path) -> CheckResult: def check_project_structure(directory: Path) -> CheckResult: + """ + Check for a minimal project scaffold. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when the project contains either an R project file or a + Python-style ``src`` directory. + """ has_rproj = (directory / "project.Rproj").exists() has_src = (directory / "src").exists() and (directory / "src").is_dir() diff --git a/src/stagecoach/cli.py b/src/stagecoach/cli.py index 58a2f01..a6254a7 100644 --- a/src/stagecoach/cli.py +++ b/src/stagecoach/cli.py @@ -1,11 +1,10 @@ from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -15,6 +14,16 @@ ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -39,6 +48,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -80,6 +98,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -103,6 +128,14 @@ def inspect( @app.command() def stage(): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) staged = StageCoach( diff --git a/src/stagecoach/customs.py b/src/stagecoach/customs.py index 992db39..499bf6a 100644 --- a/src/stagecoach/customs.py +++ b/src/stagecoach/customs.py @@ -12,6 +12,14 @@ @contextmanager def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ with UserApp( "Frontier-Stagecoach", client_id="7723dff4-fa63-4639-903b-ba6541e24e98", @@ -23,12 +31,22 @@ def globus_transfer_client(): from dataclasses import dataclass, field from typing import Any -from rich.console import Console @dataclass class Clearance: """ - Result of a Stagecoach source access check. + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. """ source: str @@ -39,6 +57,19 @@ class Clearance: def check_globus_clearance( globus_info: dict ) -> Clearance: + """ + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. + """ try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) @@ -76,6 +107,26 @@ def build_globus_transfer( globus_info: dict, clearance: Clearance ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") transfer = TransferData( @@ -95,10 +146,26 @@ def build_globus_transfer( import glob +import os +from pathlib import Path def check_gold_mine_clearance( gold_mine_info: dict ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ try: items = gold_mine_info.get("items", []) diff --git a/src/stagecoach/issue_manifest.py b/src/stagecoach/issue_manifest.py index e3f4418..9a25a89 100644 --- a/src/stagecoach/issue_manifest.py +++ b/src/stagecoach/issue_manifest.py @@ -2,7 +2,21 @@ from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ info(console, "Checking citizenship...") @@ -126,7 +140,14 @@ def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -136,8 +157,19 @@ def load_template() -> dict[str, Any]: from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. - """Prompt the user and fill in manifest fields.""" + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. + + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -202,7 +234,6 @@ def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: # filled_manifest = fill_manifest_interactively(mymanifest) -import questionary from pathlib import Path from rich.console import Console @@ -212,7 +243,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -246,7 +295,6 @@ def write_manifest( from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -260,23 +308,25 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) diff --git a/src/stagecoach/manifest_checker.py b/src/stagecoach/manifest_checker.py index bfffe12..7f7b0c2 100644 --- a/src/stagecoach/manifest_checker.py +++ b/src/stagecoach/manifest_checker.py @@ -4,10 +4,23 @@ class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -16,7 +29,12 @@ def __init__(self, manifest_file: str | Path, severity_level: Severity): def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -31,11 +49,13 @@ def run_all(self) -> list[CheckResult]: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() diff --git a/src/stagecoach/stage.py b/src/stagecoach/stage.py index 1111a79..a87601c 100644 --- a/src/stagecoach/stage.py +++ b/src/stagecoach/stage.py @@ -1,5 +1,4 @@ import os -import re import glob from pathlib import Path from stagecoach.ui import info, error, success, warning @@ -8,7 +7,22 @@ def create_stage_directories(manifest: dict, console: Console, gitignore=True): """ - Create the stage directories if they don't exist. + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. """ input_dir = manifest.get("project", {}).get("input_data_dir") @@ -45,6 +59,20 @@ def create_stage_directories(manifest: dict, console: Console, gitignore=True): @dataclass class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ source: str staged: bool message: str = "" @@ -52,7 +80,7 @@ class StageResult: from stagecoach.customs import ( - Clearance, + check_globus_clearance, build_globus_transfer, globus_transfer_client, @@ -64,9 +92,29 @@ def stage_data_from_globus( manifest: dict, console: Console, issue_transfer: bool = True -) -> bool: +) -> StageResult: """ - Stage data from Globus based on the manifest. + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. """ if not issue_transfer: @@ -106,14 +154,25 @@ def stage_data_from_globus( ) -from pathlib import Path -import glob -import os - def stage_data_from_fasrc( manifest: dict, console: Console, ) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) items = fasrc_info.get("items", []) @@ -123,6 +182,16 @@ def stage_data_from_fasrc( "skipped": [], } + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + if not items: return StageResult( source="01_gold_mine", diff --git a/src/stagecoach/stagecoach.py b/src/stagecoach/stagecoach.py index a82822c..998604e 100644 --- a/src/stagecoach/stagecoach.py +++ b/src/stagecoach/stagecoach.py @@ -1,12 +1,11 @@ import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -22,7 +21,16 @@ class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -53,7 +61,27 @@ def hail( console: Console | None = None, manifest_path: str | Path | None = None, ) -> bool: - """Create a StageCoach manifest.""" + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path @@ -79,7 +107,22 @@ def inspect( console: Console | None = None, level: Severity = Severity.ERROR ) -> bool: - """Inspect a StageCoach manifest.""" + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -96,11 +139,18 @@ def inspect( if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): info(console, "Gold Mine access requested. Checking with customs...") gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) - gold_mine_items = gold_mine_info.get("items", []) - for item in gold_mine_items: - item_name = item.get("name", "Unnamed Item") - item_path = item.get("path_regex", "No path provided") - # if not Path(item_path).exists() or + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + # check globus if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): @@ -128,7 +178,7 @@ def inspect( if not checker.passes(): console.print() - banner(console, "⚠️ Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False @@ -143,8 +193,19 @@ def stage( self, console: Console | None = None, ) -> bool: - - """Stage data according to a StageCoach manifest.""" + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ console = console or self.console banner(console, "🚂 Staging the data...") @@ -198,5 +259,5 @@ def stage( else: error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") - banner(console, "⚠️ Staging failed. Please try again.") + failure_panel(console, "⚠️ Staging failed. Please try again.") return False diff --git a/src/stagecoach/ui.py b/src/stagecoach/ui.py index 87d06ae..0ceea35 100644 --- a/src/stagecoach/ui.py +++ b/src/stagecoach/ui.py @@ -12,6 +12,16 @@ def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -22,22 +32,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -47,6 +107,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -54,6 +122,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -61,6 +142,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: diff --git a/user_guide/notebooks/checks.md b/user_guide/notebooks/checks.md index 76a2780..a01aac5 100644 --- a/user_guide/notebooks/checks.md +++ b/user_guide/notebooks/checks.md @@ -39,6 +39,18 @@ from enum import Enum from pathlib import Path class Severity(Enum): + """ + Severity levels for manifest validation results. + + Attributes + ---------- + PASS : str + The check succeeded. + WARNING : str + The check surfaced a non-blocking issue. + ERROR : str + The check found a blocking issue. + """ PASS = "pass" WARNING = "warning" ERROR = "error" @@ -59,7 +71,18 @@ PRINCIPLES = { @dataclass class CheckResult: """ - Result from a single Stagecoach manifest check. + Represent the outcome of a single manifest check. + + Attributes + ---------- + name : str + Stable identifier for the check. + state : Severity + Severity assigned to the check result. + message : str + Human-readable explanation of the outcome. + principle : int | list[int] + Frontier principle number, or numbers, associated with the check. """ name: str state: Severity @@ -87,6 +110,19 @@ Now, let’s define a few checks: ``` python def check_project_exists(directory: Path) -> CheckResult: + """ + Verify that the project directory exists. + + Parameters + ---------- + directory : Path + Directory declared as the project working directory. + + Returns + ------- + CheckResult + Passes when ``directory`` exists and is a directory. + """ if directory.exists() and directory.is_dir(): return CheckResult( name="project_exists", @@ -103,9 +139,17 @@ def check_project_exists(directory: Path) -> CheckResult: def check_git_repo_exists(directory: Path) -> CheckResult: """ - Check that the project directory is a git repository. + Verify that the project directory is under Git version control. + + Parameters + ---------- + directory : Path + Project directory to inspect. - Pass/Fail, no exceptions. + Returns + ------- + CheckResult + Passes when a ``.git`` directory is present. """ if (directory / ".git").exists(): return CheckResult( @@ -123,6 +167,20 @@ def check_git_repo_exists(directory: Path) -> CheckResult: ) def check_environment_exists(directory: Path) -> CheckResult: + """ + Look for a project environment specification or lockfile. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when at least one supported environment file exists and + warns otherwise. + """ candidates = [ "rv.lock", "renv.lock", @@ -160,6 +218,20 @@ case a user hasn’t yet started writing code: ``` python def check_code_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains analysis or source code files. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when at least one supported code or notebook file is found + outside ignored directories. + """ patterns = [ "**/*.py", "**/*.R", @@ -214,6 +286,20 @@ things like READMEs, notebooks, or markdown files: ``` python def check_narrative_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains narrated analysis notebooks. + + Parameters + ---------- + directory : Path + Project directory to inspect recursively. + + Returns + ------- + CheckResult + Passes when Quarto, R Markdown, or Jupyter notebooks are present + outside ignored directories. + """ patterns = [ "**/*.qmd", @@ -259,6 +345,20 @@ def check_narrative_exists(directory: Path) -> CheckResult: ) def check_readme_exists(directory: Path) -> CheckResult: + """ + Check whether the project contains a top-level README document. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when a supported README filename exists and errors + otherwise. + """ if (directory / "README.md").exists() or (directory / "README.Rmd").exists() or (directory / "README.qmd").exists() or (directory / "README").exists(): return CheckResult( name="readme_exists", @@ -284,6 +384,20 @@ project using an R project file, or a Python project structure with a ``` python def check_project_structure(directory: Path) -> CheckResult: + """ + Check for a minimal project scaffold. + + Parameters + ---------- + directory : Path + Project directory to inspect. + + Returns + ------- + CheckResult + Passes when the project contains either an R project file or a + Python-style ``src`` directory. + """ has_rproj = (directory / "project.Rproj").exists() has_src = (directory / "src").exists() and (directory / "src").is_dir() diff --git a/user_guide/notebooks/cli.md b/user_guide/notebooks/cli.md index 15e5c5b..1b1bdfc 100644 --- a/user_guide/notebooks/cli.md +++ b/user_guide/notebooks/cli.md @@ -15,11 +15,10 @@ Here’s the code for the CLI. First, hailing the stagecoach: from pathlib import Path from enum import Enum from rich.console import Console -from rich.panel import Panel from typing import Annotated from sheriff.sheriff import Sheriff from stagecoach.stagecoach import StageCoach -from stagecoach.checks import Severity, PRINCIPLES +from stagecoach.checks import Severity from stagecoach.ui import failure_panel import typer @@ -29,6 +28,16 @@ app = typer.Typer( ) class FailureLevel(str, Enum): + """ + Severity thresholds exposed by the CLI. + + Attributes + ---------- + WARNING : str + Treat warnings as command failures. + ERROR : str + Treat only errors as command failures. + """ WARNING = "warning" ERROR = "error" @@ -53,6 +62,15 @@ def hail( ) -> None: """ Create a Stagecoach manifest. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + output_path : Path, default=Path("stagecoach_manifest.yml") + Destination path for the generated manifest. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. """ console = Console() @@ -97,6 +115,13 @@ def inspect( ) -> None: """ Inspect a Stagecoach manifest. + + Parameters + ---------- + manifest_path : Path, default=Path("stagecoach_manifest.yml") + Path to the manifest to validate. + level : FailureLevel, default=FailureLevel.ERROR + Minimum severity that should cause the command to exit with failure. """ console = Console() @@ -123,6 +148,14 @@ And, staging: ``` python @app.command() def stage(): + """ + Stage data declared by the manifest. + + Returns + ------- + None + The command exits with a nonzero status when staging fails. + """ console = Console() customs_sheriff = Sheriff(console) staged = StageCoach( diff --git a/user_guide/notebooks/customs.md b/user_guide/notebooks/customs.md index fefec5a..da33b8e 100644 --- a/user_guide/notebooks/customs.md +++ b/user_guide/notebooks/customs.md @@ -118,6 +118,14 @@ from contextlib import contextmanager @contextmanager def globus_transfer_client(): + """ + Yield an authenticated Globus transfer client. + + Yields + ------ + TransferClient + Transfer client configured for the Frontier Stagecoach app. + """ with UserApp( "Frontier-Stagecoach", client_id="7723dff4-fa63-4639-903b-ba6541e24e98", @@ -134,12 +142,22 @@ individually, granting each of them a unique request and clearance: ``` python from dataclasses import dataclass, field from typing import Any -from rich.console import Console @dataclass class Clearance: """ - Result of a Stagecoach source access check. + Represent the outcome of a source access check. + + Attributes + ---------- + source : str + Source identifier being validated. + cleared : bool + Whether access validation succeeded. + message : str, default="" + Human-readable summary of the validation result. + details : dict[str, Any] + Source-specific metadata captured during validation. """ source: str @@ -150,6 +168,19 @@ class Clearance: def check_globus_clearance( globus_info: dict ) -> Clearance: + """ + Validate access to Globus endpoints and requested source paths. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + + Returns + ------- + Clearance + Clearance result describing whether Globus access checks passed. + """ try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) @@ -191,6 +222,26 @@ def build_globus_transfer( globus_info: dict, clearance: Clearance ) -> TransferData: + """ + Build a Globus transfer request from manifest settings. + + Parameters + ---------- + globus_info : dict + Manifest subsection describing Globus endpoints and staged items. + clearance : Clearance + Successful clearance result for the same Globus configuration. + + Returns + ------- + TransferData + Transfer request populated with all requested items. + + Raises + ------ + ValueError + Raised when ``clearance`` indicates that access checks failed. + """ if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") transfer = TransferData( @@ -220,10 +271,26 @@ specified path. This will reuse the `Clearance` class from earlier: ``` python import glob +import os +from pathlib import Path def check_gold_mine_clearance( gold_mine_info: dict ) -> Clearance: + """ + Validate readability of requested Gold Mine paths on FASRC. + + Parameters + ---------- + gold_mine_info : dict + Manifest subsection describing Gold Mine staging items. + + Returns + ------- + Clearance + Clearance result describing whether all required paths were found + and were readable. + """ try: items = gold_mine_info.get("items", []) diff --git a/user_guide/notebooks/issue_manifest.md b/user_guide/notebooks/issue_manifest.md index 420de0d..07d1753 100644 --- a/user_guide/notebooks/issue_manifest.md +++ b/user_guide/notebooks/issue_manifest.md @@ -27,7 +27,21 @@ from rich.console import Console from sheriff.sheriff import Sheriff def check_citizenship(customs_sheriff: Sheriff, console: Console) -> None: - """Validate Frontier citizenship using the Sheriff.""" + """ + Validate Frontier citizenship through the sheriff. + + Parameters + ---------- + customs_sheriff : Sheriff + Sheriff instance used to verify citizenship. + console : Console + Rich console used for progress messages. + + Raises + ------ + RuntimeError + Raised when the sheriff does not validate the current user. + """ info(console, "Checking citizenship...") @@ -169,7 +183,14 @@ from typing import Any from importlib.resources import files def load_template() -> dict[str, Any]: - """Load the packaged Stagecoach manifest template.""" + """ + Load the packaged Stagecoach manifest template. + + Returns + ------- + dict[str, Any] + Parsed manifest template bundled with the package. + """ template_path = files("stagecoach.templates").joinpath("manifest.yml") return yaml.safe_load(template_path.read_text()) @@ -195,8 +216,19 @@ import questionary from typing import Any def fill_manifest_interactively(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Prompt the user for manifest values at the command line. - """Prompt the user and fill in manifest fields.""" + Parameters + ---------- + manifest : dict[str, Any] + Manifest template to populate in place. + + Returns + ------- + dict[str, Any] + Updated manifest containing the collected user input. + """ citizen_name = questionary.text( "Citizen name (Your name as it appears in Town):" @@ -267,7 +299,6 @@ In practice, this process looks like: Next, writing the manifest should be straight forward: ``` python -import questionary from pathlib import Path from rich.console import Console @@ -277,7 +308,25 @@ def write_manifest( console: Console, overwrite: bool = False ) -> None: - """Write a manifest to disk.""" + """ + Write a manifest to disk as YAML. + + Parameters + ---------- + manifest : dict[str, Any] + Manifest data to serialize. + output_path : str | Path + Destination path for the YAML file. + console : Console + Rich console used for progress messages. + overwrite : bool, default=False + Whether to bypass the overwrite confirmation prompt. + + Raises + ------ + RuntimeError + Raised when the target file exists and overwrite is declined. + """ output_path = Path(output_path) @@ -317,7 +366,6 @@ Looks great! We now have the pieces to issue a manifest: ``` python from rich.console import Console -from rich.panel import Panel from stagecoach.ui import info, success from sheriff.sheriff import Sheriff from pyprojroot import here @@ -331,23 +379,25 @@ def issue_manifest( ) -> None: """ - Generate a Stagecoach manifest. - This function validates Frontier citizenship, loads a template manifest, - optionally fills it interactively, and writes the result to disk. + Generate a Stagecoach manifest for a project. Parameters ---------- - customs_sheriff + customs_sheriff : Sheriff Sheriff instance used to validate Frontier citizenship. - - interactive + console : Console + Rich console used for progress messages. + interactive : bool, default=True Whether to prompt the user for manifest fields. - - output_path + output_path : str | Path, default=here() / "stagecoach_manifest.yml" Destination path for the generated manifest. - - overwrite + overwrite : bool, default=False Whether to overwrite an existing manifest at the output path. + + Returns + ------- + None + This function is called for its side effect of writing the manifest. """ check_citizenship(customs_sheriff, console) diff --git a/user_guide/notebooks/manifest_checker.md b/user_guide/notebooks/manifest_checker.md index 6783bbb..dd94380 100644 --- a/user_guide/notebooks/manifest_checker.md +++ b/user_guide/notebooks/manifest_checker.md @@ -14,10 +14,23 @@ from stagecoach.checks import * class ManifestChecker: """ - Check whether a project directory satisfies minimum Frontier expectations. - The checker is intentionally lightweight. - Each check returns a structured result that Stagecoach can display - nicely in `stagecoach inspect`. + Run the standard Stagecoach manifest checks for a project. + + Parameters + ---------- + manifest_file : str | Path + Path to the manifest file that defines the project directory. + severity_level : Severity + Minimum severity that should cause ``passes`` to return ``False``. + + Attributes + ---------- + manifest : dict + Parsed manifest contents. + project_dir : str + Project working directory declared by the manifest. + severity_level : Severity + Failure threshold used by ``passes``. """ def __init__(self, manifest_file: str | Path, severity_level: Severity): self.manifest = yaml.safe_load(Path(manifest_file).read_text()) @@ -26,7 +39,12 @@ class ManifestChecker: def run_all(self) -> list[CheckResult]: """ - Run all manifest checks. + Run all configured project checks. + + Returns + ------- + list[CheckResult] + Results from each Stagecoach project validation check. """ return [ @@ -41,11 +59,13 @@ class ManifestChecker: def passes(self) -> bool: """ - Return True if no check result is at or above the configured - severity threshold. + Determine whether the manifest passes the configured threshold. - If severity_level is Severity.ERROR, warnings are allowed. - If severity_level is Severity.WARNING, warnings and errors fail. + Returns + ------- + bool + ``True`` when every check result falls below + ``self.severity_level`` and ``False`` otherwise. """ results = self.run_all() diff --git a/user_guide/notebooks/stage.md b/user_guide/notebooks/stage.md index 7546356..ee0a087 100644 --- a/user_guide/notebooks/stage.md +++ b/user_guide/notebooks/stage.md @@ -9,7 +9,6 @@ One of them will be creating the directories for data usage: ``` python import os -import re import glob from pathlib import Path from stagecoach.ui import info, error, success, warning @@ -19,7 +18,22 @@ from rich.console import Console ``` python def create_stage_directories(manifest: dict, console: Console, gitignore=True): """ - Create the stage directories if they don't exist. + Create the standard Stagecoach data directories. + + Parameters + ---------- + manifest : dict + Manifest containing the ``project`` directory configuration. + console : Console + Rich console used for progress messages. + gitignore : bool, default=True + Whether to create a minimal ``.gitignore`` file in each directory. + + Raises + ------ + ValueError + Raised when one or more required project directories are missing + from the manifest. """ input_dir = manifest.get("project", {}).get("input_data_dir") @@ -61,6 +75,20 @@ from typing import Any @dataclass class StageResult: + """ + Summarize the result of a staging operation. + + Attributes + ---------- + source : str + Source identifier, such as ``01_gold_mine`` or ``02_globus``. + staged : bool + Whether the staging operation completed successfully. + message : str, default="" + Human-readable summary of the outcome. + details : dict[str, Any] + Source-specific metadata about the staging attempt. + """ source: str staged: bool message: str = "" @@ -73,7 +101,7 @@ to call the appropriate functions. ``` python from stagecoach.customs import ( - Clearance, + check_globus_clearance, build_globus_transfer, globus_transfer_client, @@ -85,9 +113,29 @@ def stage_data_from_globus( manifest: dict, console: Console, issue_transfer: bool = True -) -> bool: +) -> StageResult: """ - Stage data from Globus based on the manifest. + Submit a Globus transfer for manifest-declared items. + + Parameters + ---------- + manifest : dict + Manifest containing the ``02_globus`` source configuration. + console : Console + Rich console used for progress messages. + issue_transfer : bool, default=True + Whether to submit the transfer after validating access. + + Returns + ------- + StageResult + Summary of the Globus staging attempt. + + Raises + ------ + ValueError + Raised when the manifest does not define the required Globus + source information. """ if not issue_transfer: @@ -130,14 +178,25 @@ def stage_data_from_globus( Next we implement symlink staging via FASRC: ``` python -from pathlib import Path -import glob -import os - def stage_data_from_fasrc( manifest: dict, console: Console, ) -> StageResult: + """ + Stage Gold Mine data by creating symlinks on FASRC. + + Parameters + ---------- + manifest : dict + Manifest containing the ``01_gold_mine`` source configuration. + console : Console + Rich console used for progress messages. + + Returns + ------- + StageResult + Summary of symlink creation outcomes for all requested items. + """ fasrc_info = manifest.get("sources", {}).get("01_gold_mine", {}) items = fasrc_info.get("items", []) @@ -147,6 +206,16 @@ def stage_data_from_fasrc( "skipped": [], } + clearance = check_gold_mine_clearance(fasrc_info) + + if not clearance.cleared: + return StageResult( + source="01_gold_mine", + staged=False, + message=f"Gold Mine clearance failed: {clearance.message}", + details=task_list + ) + if not items: return StageResult( source="01_gold_mine", diff --git a/user_guide/notebooks/stagecoach.md b/user_guide/notebooks/stagecoach.md index 0aeaa37..2ce04c4 100644 --- a/user_guide/notebooks/stagecoach.md +++ b/user_guide/notebooks/stagecoach.md @@ -53,12 +53,11 @@ defined in the `stagecoach` class: import yaml from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker -from stagecoach.customs import Clearance, check_globus_clearance, build_globus_transfer from stagecoach.checks import Severity +from stagecoach.customs import check_gold_mine_clearance, check_globus_clearance from stagecoach.stage import create_stage_directories, stage_data_from_fasrc, stage_data_from_globus, StageResult from sheriff.sheriff import Sheriff from rich.console import Console -from rich.panel import Panel from pathlib import Path from stagecoach.ui import ( @@ -74,7 +73,16 @@ from stagecoach.ui import ( class StageCoach: """ - Orchestrate Stagecoach workflows. + Orchestrate manifest issuance, inspection, and data staging. + + Parameters + ---------- + sheriff : Sheriff | None, default=None + Sheriff instance used for identity and policy checks. + manifest_path : str | Path, default="stagecoach_manifest.yml" + Path to the manifest used by instance methods. + console : Console | None, default=None + Rich console used for user-facing output. Methods ------- @@ -105,7 +113,27 @@ class StageCoach: console: Console | None = None, manifest_path: str | Path | None = None, ) -> bool: - """Create a StageCoach manifest.""" + """ + Create a manifest for a new Stagecoach workflow. + + Parameters + ---------- + interactive : bool, default=True + Whether to prompt for manifest fields interactively. + overwrite : bool, default=False + Whether to overwrite an existing manifest file. + sheriff : Sheriff | None, default=None + Optional sheriff override for this invocation. + console : Console | None, default=None + Optional console override for this invocation. + manifest_path : str | Path | None, default=None + Optional manifest destination override. + + Returns + ------- + bool + ``True`` when manifest creation completes successfully. + """ sheriff = sheriff or self.sheriff console = console or self.console manifest_path = Path(manifest_path) if manifest_path else self.manifest_path @@ -131,7 +159,22 @@ class StageCoach: console: Console | None = None, level: Severity = Severity.ERROR ) -> bool: - """Inspect a StageCoach manifest.""" + """ + Validate a manifest and report any blocking issues. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + level : Severity, default=Severity.ERROR + Minimum severity that should cause inspection to fail. + + Returns + ------- + bool + ``True`` when the manifest passes all checks at the requested + severity threshold. + """ console = console or self.console banner(console, f"📋 Inspecting manifest at level: {level.value}") @@ -148,11 +191,18 @@ class StageCoach: if manifest.get("sources", {}).get("01_gold_mine", {}).get("enabled"): info(console, "Gold Mine access requested. Checking with customs...") gold_mine_info = manifest.get("sources", {}).get("01_gold_mine", {}) - gold_mine_items = gold_mine_info.get("items", []) - for item in gold_mine_items: - item_name = item.get("name", "Unnamed Item") - item_path = item.get("path_regex", "No path provided") - # if not Path(item_path).exists() or + clearance = check_gold_mine_clearance(gold_mine_info) + + if clearance.cleared: + success(console, "Gold Mine access cleared by customs.") + else: + error( + console, + "Gold Mine access clearance failed. Please check your access and try again.", + ) + console.print(clearance) + raise ValueError("Gold Mine access clearance failed.") + # check globus if manifest.get("sources", {}).get("02_globus", {}).get("enabled"): @@ -180,7 +230,7 @@ class StageCoach: if not checker.passes(): console.print() - banner(console, "⚠️ Manifest checks failed. Please review the results above.") + failure_panel(console, "⚠️ Manifest checks failed. Please review the results above.") handbook_note(console) return False @@ -195,8 +245,19 @@ class StageCoach: self, console: Console | None = None, ) -> bool: - - """Stage data according to a StageCoach manifest.""" + """ + Stage data declared by the instance manifest. + + Parameters + ---------- + console : Console | None, default=None + Optional console override for this invocation. + + Returns + ------- + bool + ``True`` when every requested source stages successfully. + """ console = console or self.console banner(console, "🚂 Staging the data...") @@ -250,7 +311,7 @@ class StageCoach: else: error(console, "Some data sources failed to stage. Please review the messages above and address any issues in your manifest.") - banner(console, "⚠️ Staging failed. Please try again.") + failure_panel(console, "⚠️ Staging failed. Please try again.") return False ``` diff --git a/user_guide/notebooks/ui.md b/user_guide/notebooks/ui.md index 917b390..db4600f 100644 --- a/user_guide/notebooks/ui.md +++ b/user_guide/notebooks/ui.md @@ -20,6 +20,16 @@ HANDBOOK_URL = ( def banner(console: Console, message: str) -> None: + """ + Render a highlighted banner message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message to display inside the banner panel. + """ console.print( Panel.fit( message, @@ -30,22 +40,72 @@ def banner(console: Console, message: str) -> None: def success(console: Console, message: str) -> None: + """ + Render a success message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold green]✓[/bold green] {escape(message)}") def warning(console: Console, message: str) -> None: + """ + Render a warning message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold yellow]⚠[/bold yellow] {escape(message)}") def error(console: Console, message: str) -> None: + """ + Render an error message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold red]✖[/bold red] {escape(message)}") def info(console: Console, message: str) -> None: + """ + Render an informational message. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print(f"[bold cyan]•[/bold cyan] {escape(message)}") def failure_panel(console: Console, message: str) -> None: + """ + Render an error message inside a bordered panel. + + Parameters + ---------- + console : Console + Rich console used for output. + message : str + Message body to print. + """ console.print( Panel.fit( f"[bold red]✖[/bold red] {escape(message)}", @@ -55,6 +115,14 @@ def failure_panel(console: Console, message: str) -> None: def handbook_note(console: Console) -> None: + """ + Print a link to the Frontier handbook principles. + + Parameters + ---------- + console : Console + Rich console used for output. + """ console.print( "[dim]See handbook for explanation of principles:[/dim] " f"[link={HANDBOOK_URL}]{HANDBOOK_URL}[/link]" @@ -62,6 +130,19 @@ def handbook_note(console: Console) -> None: def format_principle(principle: int | list[int]) -> str: + """ + Format one or more principle identifiers for display. + + Parameters + ---------- + principle : int | list[int] + Principle number or collection of principle numbers. + + Returns + ------- + str + Comma-separated principle identifiers. + """ if isinstance(principle, list): return ", ".join(str(item) for item in principle) @@ -69,6 +150,17 @@ def format_principle(principle: int | list[int]) -> str: def check_result(console: Console, result) -> None: + """ + Render a manifest check result using severity-specific styling. + + Parameters + ---------- + console : Console + Rich console used for output. + result + Object with ``name``, ``state``, ``message``, and ``principle`` + attributes, typically a ``CheckResult`` instance. + """ principle = format_principle(result.principle) if result.state == Severity.PASS: From f9c0cf289035f54208209423db2deec4212bed63 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 14:38:09 -0400 Subject: [PATCH 04/10] feat(docs): add CI workflow for documentation build and publish * Implement CI workflow to build and publish documentation on push to main * Add preview step for pull requests * Ensure artifacts are uploaded and deployed to GitHub Pages * Enhance CLI command to accept manifest path as an option * Update customs and stage notebooks to reflect changes in globus transfer logic * Improve error handling for missing globus information in manifests --- .github/workflows/docs.yml | 102 ++++++++++++++++++++++++++++++++ notebooks/cli.qmd | 10 +++- notebooks/customs.qmd | 27 +++++++-- notebooks/stage.qmd | 6 +- src/stagecoach/cli.py | 10 +++- src/stagecoach/customs.py | 27 +++++++-- src/stagecoach/stage.py | 6 +- stagecoach_manifest.yml | 1 - user_guide/notebooks/cli.md | 10 +++- user_guide/notebooks/customs.md | 27 +++++++-- user_guide/notebooks/stage.md | 6 +- 11 files changed, 210 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..74a5007 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,102 @@ +name: CI Docs + +on: + push: + branches: + - main + pull_request: + +jobs: + build-docs: + name: "Build Docs" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # Full history for accurate page timestamps + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install package and dependencies + run: | + python -m pip install uv + uv sync + uv pip install great-docs + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + + - name: Build docs + run: uv run great-docs build + + - name: Save docs artifact + uses: actions/upload-artifact@v7 + with: + name: docs-html + path: great-docs/_site + + publish-docs: + name: "Publish Docs" + runs-on: ubuntu-latest + needs: "build-docs" + if: github.ref == 'refs/heads/main' + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/download-artifact@v7 + with: + name: docs-html + path: great-docs/_site + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v5 + with: + path: great-docs/_site + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5 + + preview-docs: + name: "Preview Docs" + runs-on: ubuntu-latest + needs: "build-docs" + if: github.event_name == 'pull_request' + permissions: + deployments: write + pull-requests: write + steps: + - uses: actions/download-artifact@v7 + with: + name: docs-html + path: great-docs/_site + + # Start deployment + - name: Configure pull release name + if: ${{ github.event_name == 'pull_request' }} + run: | + echo "RELEASE_NAME=pr-${{ github.event.number }}" >> $GITHUB_ENV + + - name: Configure branch release name + if: ${{ github.event_name != 'pull_request' }} + run: | + # use branch name, but replace slashes. E.g. feat/a -> feat-a + echo "RELEASE_NAME=${GITHUB_REF_NAME//\//-}" >> $GITHUB_ENV + + # Deploy + - name: Create Github Deployment + uses: bobheadxi/deployments@v1 + id: deployment + if: ${{ !github.event.pull_request.head.repo.fork }} + with: + step: start + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ env.RELEASE_NAME }} + ref: ${{ github.head_ref }} + logs: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/notebooks/cli.qmd b/notebooks/cli.qmd index 1bec0f0..f6abf9d 100644 --- a/notebooks/cli.qmd +++ b/notebooks/cli.qmd @@ -169,7 +169,14 @@ And, staging: ```{python} @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): """ Stage data declared by the manifest. @@ -183,6 +190,7 @@ def stage(): staged = StageCoach( sheriff=customs_sheriff, console=console, + manifest_path=manifest_path ).stage() if not staged: raise typer.Exit(code=1) diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index 061e140..eadd1fd 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -311,8 +311,10 @@ logic. This is built mostly on what we learned above: ```{python} def build_globus_transfer( - globus_info: dict, - clearance: Clearance + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", ) -> TransferData: """ Build a Globus transfer request from manifest settings. @@ -323,6 +325,10 @@ def build_globus_transfer( Manifest subsection describing Globus endpoints and staged items. clearance : Clearance Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. Returns ------- @@ -334,18 +340,29 @@ def build_globus_transfer( ValueError Raised when ``clearance`` indicates that access checks failed. """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( source_endpoint=globus_info["source_endpoint"], destination_endpoint=globus_info["destination_endpoint"], - label="Stagecoach transfer", + label=label, ) for item in globus_info["items"]: + + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + transfer.add_item( - item["source_path"], - item["destination_path"], + str(source_path), + str(destination_path), recursive=item.get("recursive", True), ) diff --git a/notebooks/stage.qmd b/notebooks/stage.qmd index 3731ac8..c3ce9a7 100644 --- a/notebooks/stage.qmd +++ b/notebooks/stage.qmd @@ -165,11 +165,15 @@ def stage_data_from_globus( ) globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + if not globus_info: error(console, "No Globus information found in the manifest.") raise ValueError("Globus information is missing from the manifest.") clearance = check_globus_clearance(globus_info) - transfer = build_globus_transfer(globus_info, clearance) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) diff --git a/src/stagecoach/cli.py b/src/stagecoach/cli.py index a6254a7..09bbb03 100644 --- a/src/stagecoach/cli.py +++ b/src/stagecoach/cli.py @@ -127,7 +127,14 @@ def inspect( @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): """ Stage data declared by the manifest. @@ -141,6 +148,7 @@ def stage(): staged = StageCoach( sheriff=customs_sheriff, console=console, + manifest_path=manifest_path ).stage() if not staged: raise typer.Exit(code=1) diff --git a/src/stagecoach/customs.py b/src/stagecoach/customs.py index 499bf6a..3305976 100644 --- a/src/stagecoach/customs.py +++ b/src/stagecoach/customs.py @@ -104,8 +104,10 @@ def check_globus_clearance( def build_globus_transfer( - globus_info: dict, - clearance: Clearance + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", ) -> TransferData: """ Build a Globus transfer request from manifest settings. @@ -116,6 +118,10 @@ def build_globus_transfer( Manifest subsection describing Globus endpoints and staged items. clearance : Clearance Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. Returns ------- @@ -127,18 +133,29 @@ def build_globus_transfer( ValueError Raised when ``clearance`` indicates that access checks failed. """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( source_endpoint=globus_info["source_endpoint"], destination_endpoint=globus_info["destination_endpoint"], - label="Stagecoach transfer", + label=label, ) for item in globus_info["items"]: + + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + transfer.add_item( - item["source_path"], - item["destination_path"], + str(source_path), + str(destination_path), recursive=item.get("recursive", True), ) diff --git a/src/stagecoach/stage.py b/src/stagecoach/stage.py index a87601c..c7c02bb 100644 --- a/src/stagecoach/stage.py +++ b/src/stagecoach/stage.py @@ -126,11 +126,15 @@ def stage_data_from_globus( ) globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + if not globus_info: error(console, "No Globus information found in the manifest.") raise ValueError("Globus information is missing from the manifest.") clearance = check_globus_clearance(globus_info) - transfer = build_globus_transfer(globus_info, clearance) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) diff --git a/stagecoach_manifest.yml b/stagecoach_manifest.yml index cc4902c..f6eb965 100644 --- a/stagecoach_manifest.yml +++ b/stagecoach_manifest.yml @@ -31,7 +31,6 @@ sources: items: - name: example_globus_item source_path: "/n/holylabs/cgolden_lab/Lab/projects/era5_database/era5_sandbox/notes" - destination_path: "/n/holylabs/LABS/cgolden_lab/Lab/frontier/stagecoach/data/inputs/inst" files_regex: ["*.ipynb"] recursive: true required: true diff --git a/user_guide/notebooks/cli.md b/user_guide/notebooks/cli.md index 1b1bdfc..27a3e27 100644 --- a/user_guide/notebooks/cli.md +++ b/user_guide/notebooks/cli.md @@ -147,7 +147,14 @@ And, staging: ``` python @app.command() -def stage(): +def stage( + manifest_path: Path = typer.Option( + Path("stagecoach_manifest.yml"), + "--manifest", + "-m", + help="Path to the manifest to inspect.", + ) + ): """ Stage data declared by the manifest. @@ -161,6 +168,7 @@ def stage(): staged = StageCoach( sheriff=customs_sheriff, console=console, + manifest_path=manifest_path ).stage() if not staged: raise typer.Exit(code=1) diff --git a/user_guide/notebooks/customs.md b/user_guide/notebooks/customs.md index da33b8e..8d6d686 100644 --- a/user_guide/notebooks/customs.md +++ b/user_guide/notebooks/customs.md @@ -219,8 +219,10 @@ is built mostly on what we learned above: ``` python def build_globus_transfer( - globus_info: dict, - clearance: Clearance + manifest: dict, + clearance: Clearance, + fix_holylabs: bool = True, + label: str = "Stagecoach transfer", ) -> TransferData: """ Build a Globus transfer request from manifest settings. @@ -231,6 +233,10 @@ def build_globus_transfer( Manifest subsection describing Globus endpoints and staged items. clearance : Clearance Successful clearance result for the same Globus configuration. + fix_holylabs : bool, default=True + Whether to apply Holylabs-specific path fix to the transfer (removes redundant LAB segment from paths). + label: str, optional + Optional label for the transfer task. Returns ------- @@ -242,18 +248,29 @@ def build_globus_transfer( ValueError Raised when ``clearance`` indicates that access checks failed. """ + globus_info = manifest.get("sources", {}).get("02_globus", {}) + + if not globus_info: + raise ValueError("Globus information is missing from the manifest.") + if not clearance.cleared: raise ValueError(f"Cannot build transfer: clearance failed with message: {clearance.message}") + transfer = TransferData( source_endpoint=globus_info["source_endpoint"], destination_endpoint=globus_info["destination_endpoint"], - label="Stagecoach transfer", + label=label, ) for item in globus_info["items"]: + + source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] + destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name + transfer.add_item( - item["source_path"], - item["destination_path"], + str(source_path), + str(destination_path), recursive=item.get("recursive", True), ) diff --git a/user_guide/notebooks/stage.md b/user_guide/notebooks/stage.md index ee0a087..dd2c9f5 100644 --- a/user_guide/notebooks/stage.md +++ b/user_guide/notebooks/stage.md @@ -147,11 +147,15 @@ def stage_data_from_globus( ) globus_info = manifest.get("sources", {}).get("02_globus", {}) + input_root = Path(manifest["project"]["input_data_dir"]) + source_root = input_root / "02_globus" + source_root.mkdir(parents=True, exist_ok=True) + if not globus_info: error(console, "No Globus information found in the manifest.") raise ValueError("Globus information is missing from the manifest.") clearance = check_globus_clearance(globus_info) - transfer = build_globus_transfer(globus_info, clearance) + transfer = build_globus_transfer(manifest, clearance, fix_holylabs=True, label = manifest.get("project", {}).get("project_name", "stagecoach_transfer")) try: with globus_transfer_client() as client: client.get_endpoint(globus_info["source_endpoint"]) From 5be69d70c613ec9eafb849a2cb072e112148cee6 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 15:59:16 -0400 Subject: [PATCH 05/10] set sheriff version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cc279f2..6952e88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,4 +28,4 @@ build-backend = "uv_build" "stagecoach.templates" = ["*.yaml"] [tool.uv.sources] -sheriff = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git", rev = "v0.1.2" } +sheriff = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git", rev = "v0.1.1" } From 18abe283798e28fed87c01957eef401ea40d8560 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 23:00:35 -0400 Subject: [PATCH 06/10] feat(outpost): add temporary mock Frontier scaffold * Update Python version to 3.14 * Introduce `outpost` module for creating a mock Frontier directory * Add `--outpost` CLI flag for manifest creation outside FASRC * Update documentation to reflect new outpost functionality --- .python-version | 2 +- notebooks/cli.qmd | 6 ++ notebooks/outpost.qmd | 101 +++++++++++++++++++++++++++++ notebooks/stagecoach.qmd | 33 ++++++++-- pyproject.toml | 1 + src/stagecoach/cli.py | 6 ++ src/stagecoach/outpost.py | 55 ++++++++++++++++ src/stagecoach/stagecoach.py | 33 ++++++++-- user_guide/notebooks/cli.md | 6 ++ user_guide/notebooks/outpost.md | 79 ++++++++++++++++++++++ user_guide/notebooks/stagecoach.md | 33 ++++++++-- uv.lock | 9 +-- 12 files changed, 338 insertions(+), 26 deletions(-) create mode 100644 notebooks/outpost.qmd create mode 100644 src/stagecoach/outpost.py create mode 100644 user_guide/notebooks/outpost.md diff --git a/.python-version b/.python-version index e4fba21..6324d40 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12 +3.14 diff --git a/notebooks/cli.qmd b/notebooks/cli.qmd index f6abf9d..35764bd 100644 --- a/notebooks/cli.qmd +++ b/notebooks/cli.qmd @@ -76,6 +76,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -106,6 +111,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: diff --git a/notebooks/outpost.qmd b/notebooks/outpost.qmd new file mode 100644 index 0000000..971240a --- /dev/null +++ b/notebooks/outpost.qmd @@ -0,0 +1,101 @@ +--- +title: "Outpost" +subtitle: "Scaffold a Fake Frontier Directory to Run `stagecoach` Outside of FASRC" +filters: + - sorting-hat + - ripper +extensions: + sorting-hat: + keep: + - python + verbose: true + ripper: + include-yaml: false + output-name: "outpost" + output-dir: "../src/stagecoach" + script-links-position: "none" +params: + export: true +format: gfm +--- + +We have a problem. The `stagecoach` tool is designed to run on the FASRC cluster, and it +relies on the `sheriff` to run a citizenship test (i.e. determine that you +are bought in to use The Frontier). This means that if you want to +run `stagecoach` outside of FASRC, you need to have those documents, +which will be a barrier for many users. + +Here, we introduce the `outpost` module, which will temporarily +scaffold a fake Frontier directory that the sheriff will +accept as valid. This +will allow users to run `stagecoach` on their local machines, +without needing to have the actual citizenship documents. + +We're going ot implement this as a CLI flag, `--outpost`, +which will trigger the creation of this fake directory. +Additionally, ChatGPT recommends using the `contextlib` +module to create a context manager that will handle the +setup and teardown of this fake directory, ensuring that it is cleaned up after use. + +The `outpost` module will have a `create_outpost` function +context. + +```{python} +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier +``` + + diff --git a/notebooks/stagecoach.qmd b/notebooks/stagecoach.qmd index 9625ff9..57c1946 100644 --- a/notebooks/stagecoach.qmd +++ b/notebooks/stagecoach.qmd @@ -68,6 +68,8 @@ the data to the user. #| sorting-hat: keep import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker from stagecoach.checks import Severity @@ -129,6 +131,7 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, + outpost: bool = False, ) -> bool: """ Create a manifest for a new Stagecoach workflow. @@ -145,6 +148,9 @@ class StageCoach: Optional console override for this invocation. manifest_path : str | Path | None, default=None Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. Returns ------- @@ -159,13 +165,26 @@ class StageCoach: banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) + + outpost = create_outpost() if outpost else nullcontext() + + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") diff --git a/pyproject.toml b/pyproject.toml index 6952e88..6405d5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "dataclasses>=0.8", + "globus-sdk>=4.5.0", "pyprojroot>=0.3.0", "pyyaml>=6.0.3", "questionary>=2.1.1", diff --git a/src/stagecoach/cli.py b/src/stagecoach/cli.py index 09bbb03..c1cbf55 100644 --- a/src/stagecoach/cli.py +++ b/src/stagecoach/cli.py @@ -40,6 +40,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -70,6 +75,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: diff --git a/src/stagecoach/outpost.py b/src/stagecoach/outpost.py new file mode 100644 index 0000000..9ff118a --- /dev/null +++ b/src/stagecoach/outpost.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier diff --git a/src/stagecoach/stagecoach.py b/src/stagecoach/stagecoach.py index 998604e..644e320 100644 --- a/src/stagecoach/stagecoach.py +++ b/src/stagecoach/stagecoach.py @@ -1,4 +1,6 @@ import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker from stagecoach.checks import Severity @@ -60,6 +62,7 @@ def hail( sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, + outpost: bool = False, ) -> bool: """ Create a manifest for a new Stagecoach workflow. @@ -76,6 +79,9 @@ def hail( Optional console override for this invocation. manifest_path : str | Path | None, default=None Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. Returns ------- @@ -90,13 +96,26 @@ def hail( banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) + + outpost = create_outpost() if outpost else nullcontext() + + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") diff --git a/user_guide/notebooks/cli.md b/user_guide/notebooks/cli.md index 27a3e27..a581427 100644 --- a/user_guide/notebooks/cli.md +++ b/user_guide/notebooks/cli.md @@ -54,6 +54,11 @@ def hail( "-o", help="Where to write the manifest.", ), + outpost: bool = typer.Option( + False, + "--outpost/--no-outpost", + help="Whether to create a temporary mock Frontier citizenship scaffold for manifest creation outside the assigned Frontier.", + ), overwrite: bool = typer.Option( False, "--overwrite/--no-overwrite", @@ -84,6 +89,7 @@ def hail( ).hail( interactive=interactive, overwrite=overwrite, + outpost=outpost ) except Exception as exc: diff --git a/user_guide/notebooks/outpost.md b/user_guide/notebooks/outpost.md new file mode 100644 index 0000000..36d3b73 --- /dev/null +++ b/user_guide/notebooks/outpost.md @@ -0,0 +1,79 @@ +# Outpost + + +We have a problem. The `stagecoach` tool is designed to run on the FASRC +cluster, and it relies on the `sheriff` to run a citizenship test +(i.e. determine that you are bought in to use The Frontier). This means +that if you want to run `stagecoach` outside of FASRC, you need to have +those documents, which will be a barrier for many users. + +Here, we introduce the `outpost` module, which will temporarily scaffold +a fake Frontier directory that the sheriff will accept as valid. This +will allow users to run `stagecoach` on their local machines, without +needing to have the actual citizenship documents. + +We’re going ot implement this as a CLI flag, `--outpost`, which will +trigger the creation of this fake directory. Additionally, ChatGPT +recommends using the `contextlib` module to create a context manager +that will handle the setup and teardown of this fake directory, ensuring +that it is cleaned up after use. + +The `outpost` module will have a `create_outpost` function context. + +``` python +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import yaml + + +@contextmanager +def create_outpost( + *, + mode: str = "outpost", + frontier_dirname: str = "frontier", + frontier_file: str = "frontier.yml", +): + """ + Create a temporary mock Frontier file and point THE_FRONTIER at it. + + This does not grant access to real Frontier resources. It only satisfies + Sheriff's file-based citizenship check for commands that need to run + outside the assigned Frontier, such as `stagecoach hail --outpost`. + """ + old_the_frontier = os.environ.get("THE_FRONTIER") + + with TemporaryDirectory(prefix="stagecoach-outpost-") as tmp: + root = Path(tmp) + frontier_root = root / frontier_dirname + frontier_root.mkdir(parents=True, exist_ok=True) + + frontier_path = frontier_root / frontier_file + + mock_frontier = { + "frontier": { + "mode": mode, + "kind": "mock", + "created_by": "stagecoach.forger", + "purpose": "temporary citizenship scaffold", + } + } + + frontier_path.write_text( + yaml.safe_dump(mock_frontier, sort_keys=False), + encoding="utf-8", + ) + + os.environ["THE_FRONTIER"] = str(frontier_path) + + try: + yield frontier_path + finally: + if old_the_frontier is None: + os.environ.pop("THE_FRONTIER", None) + else: + os.environ["THE_FRONTIER"] = old_the_frontier +``` diff --git a/user_guide/notebooks/stagecoach.md b/user_guide/notebooks/stagecoach.md index 2ce04c4..78d8625 100644 --- a/user_guide/notebooks/stagecoach.md +++ b/user_guide/notebooks/stagecoach.md @@ -51,6 +51,8 @@ defined in the `stagecoach` class: ``` python import yaml +from contextlib import nullcontext +from stagecoach.outpost import create_outpost from stagecoach.issue_manifest import issue_manifest from stagecoach.manifest_checker import ManifestChecker from stagecoach.checks import Severity @@ -112,6 +114,7 @@ class StageCoach: sheriff: Sheriff | None = None, console: Console | None = None, manifest_path: str | Path | None = None, + outpost: bool = False, ) -> bool: """ Create a manifest for a new Stagecoach workflow. @@ -128,6 +131,9 @@ class StageCoach: Optional console override for this invocation. manifest_path : str | Path | None, default=None Optional manifest destination override. + outpost : bool, default=False + Whether to create a temporary mock Frontier citizenship scaffold for + manifest creation outside the assigned Frontier. Returns ------- @@ -142,13 +148,26 @@ class StageCoach: banner(console, "The stagecoach manifest is a YAML file that describes the data sources for your project and how to access them. You can create a manifest interactively, or stagecoach will provide a template manifest and you will fill in the details. The manifest will also include information about the data dependencies for your project, and how to access those dependencies.") - issue_manifest( - customs_sheriff=sheriff, - interactive=interactive, - output_path=manifest_path, - overwrite=overwrite, - console=console, - ) + if outpost: + + banner( + console, + "Using Outpost mode for manifest creation. ⛺" + "Stagecoach will create a temporary mock Frontier file for the " + "citizenship check, allowing the Stagecoach to " + "ferry the data to a fictional Frontier." + ) + + outpost = create_outpost() if outpost else nullcontext() + + with outpost: + issue_manifest( + customs_sheriff=sheriff, + interactive=interactive, + output_path=manifest_path, + overwrite=overwrite, + console=console, + ) banner(console, f"✅ Manifest created at: {manifest_path}. Fill it out and then run `stagecoach inspect` to check it!") diff --git a/uv.lock b/uv.lock index 144320d..c4f2c77 100644 --- a/uv.lock +++ b/uv.lock @@ -1727,10 +1727,9 @@ wheels = [ [[package]] name = "sheriff" -version = "0.1.2" -source = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.2#8ce66b626ed2d23dc8e9909b0114d90d19ba6797" } +version = "0.1.1" +source = { git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.1#491af8005d20b9ea6d5c0df58226e504107aac41" } dependencies = [ - { name = "globus-sdk" }, { name = "great-docs" }, { name = "ipykernel" }, { name = "jupyter" }, @@ -1779,6 +1778,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "dataclasses" }, + { name = "globus-sdk" }, { name = "pyprojroot" }, { name = "pyyaml" }, { name = "questionary" }, @@ -1790,11 +1790,12 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "dataclasses", specifier = ">=0.8" }, + { name = "globus-sdk", specifier = ">=4.5.0" }, { name = "pyprojroot", specifier = ">=0.3.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "questionary", specifier = ">=2.1.1" }, { name = "rich", specifier = ">=15.0.0" }, - { name = "sheriff", git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.2" }, + { name = "sheriff", git = "https://github.com/GoldenPlanetaryHealthLab/sheriff.git?rev=v0.1.1" }, { name = "typer", specifier = ">=0.25.1" }, ] From 69dc5d01e88a614e1b708807e20eae919f35e557 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 23:21:11 -0400 Subject: [PATCH 07/10] feat(stagecoach): add staging result messages for gold mine and globus * Implemented default messages for cases where staging is not requested. * Enhanced user feedback during data staging process. --- notebooks/stagecoach.qmd | 12 ++++++++++++ src/stagecoach/stagecoach.py | 12 ++++++++++++ user_guide/notebooks/stagecoach.md | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/notebooks/stagecoach.qmd b/notebooks/stagecoach.qmd index 57c1946..ea52f3f 100644 --- a/notebooks/stagecoach.qmd +++ b/notebooks/stagecoach.qmd @@ -310,6 +310,12 @@ class StageCoach: success(console, "Gold Mine data staged successfully.") else: error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) # globus globus = manifest.get("sources", {}).get("02_globus", {}) @@ -319,6 +325,12 @@ class StageCoach: success(console, "Globus data staged successfully.") else: error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) # dataverse dataverse = manifest.get("sources", {}).get("03_dataverse", {}) diff --git a/src/stagecoach/stagecoach.py b/src/stagecoach/stagecoach.py index 644e320..23c7df6 100644 --- a/src/stagecoach/stagecoach.py +++ b/src/stagecoach/stagecoach.py @@ -241,6 +241,12 @@ def stage( success(console, "Gold Mine data staged successfully.") else: error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) # globus globus = manifest.get("sources", {}).get("02_globus", {}) @@ -250,6 +256,12 @@ def stage( success(console, "Globus data staged successfully.") else: error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) # dataverse dataverse = manifest.get("sources", {}).get("03_dataverse", {}) diff --git a/user_guide/notebooks/stagecoach.md b/user_guide/notebooks/stagecoach.md index 78d8625..4994f7a 100644 --- a/user_guide/notebooks/stagecoach.md +++ b/user_guide/notebooks/stagecoach.md @@ -293,6 +293,12 @@ class StageCoach: success(console, "Gold Mine data staged successfully.") else: error(console, f"Gold Mine staging failed: {fasrc_stageresult.message}") + else: + fasrc_stageresult = StageResult( + source="01_gold_mine", + staged=True, + message="Gold Mine staging not requested.", + ) # globus globus = manifest.get("sources", {}).get("02_globus", {}) @@ -302,6 +308,12 @@ class StageCoach: success(console, "Globus data staged successfully.") else: error(console, f"Globus staging failed: {globus_stageresult.message}") + else: + globus_stageresult = StageResult( + source="02_globus", + staged=True, + message="Globus staging not requested.", + ) # dataverse dataverse = manifest.get("sources", {}).get("03_dataverse", {}) From 8a6844a4b117eaf0224615ef5e9a0b2a8c5c82be Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 13 May 2026 23:31:34 -0400 Subject: [PATCH 08/10] add optional globus root path --- notebooks/customs.qmd | 7 ++++++- src/stagecoach/customs.py | 7 ++++++- user_guide/notebooks/customs.md | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/notebooks/customs.qmd b/notebooks/customs.qmd index eadd1fd..98af6a6 100644 --- a/notebooks/customs.qmd +++ b/notebooks/customs.qmd @@ -357,7 +357,12 @@ def build_globus_transfer( for item in globus_info["items"]: source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] - destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name transfer.add_item( diff --git a/src/stagecoach/customs.py b/src/stagecoach/customs.py index 3305976..d53da69 100644 --- a/src/stagecoach/customs.py +++ b/src/stagecoach/customs.py @@ -150,7 +150,12 @@ def build_globus_transfer( for item in globus_info["items"]: source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] - destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name transfer.add_item( diff --git a/user_guide/notebooks/customs.md b/user_guide/notebooks/customs.md index 8d6d686..932734e 100644 --- a/user_guide/notebooks/customs.md +++ b/user_guide/notebooks/customs.md @@ -265,7 +265,12 @@ def build_globus_transfer( for item in globus_info["items"]: source_path = item["source_path"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else item["source_path"] - destination_root = manifest["project"]["input_data_dir"].replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] + + if item.get("destination_path", None): + destination_root = item["destination_path"] + else: + destination_root = manifest["project"]["input_data_dir"] + destination_root = destination_root.replace("/n/holylabs/LABS/", "/n/holylabs/") if fix_holylabs else manifest["project"]["input_data_dir"] destination_path = Path(destination_root) / "02_globus" / item['name'] / Path(source_path).name transfer.add_item( From ceeae0475ec41c0d2041bbc746bbbc3dfeaf8b91 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Thu, 14 May 2026 12:18:51 -0400 Subject: [PATCH 09/10] Stage Data Fixes #2 --- README.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/README.md b/README.md index e69de29..be66765 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,78 @@ +# Stagecoach: A CLI tool for Staging Analysis Data + +Stage coach is a CLI tool for "staging" analysis +data to your project. It helps you organize and orchestrate input +data for your projects in a consistent, reproducible way, +making it easy to scaffold your data science project. +All of the configuration is done through a `stagecoach_manifest.yaml` +file, which you can customize to fit your needs. +Simply run `stagecoach hail` to request a manifest, edit it to source your, +data, and request the data to be staged with `stagecoach stage`. The tool +will run the necessary checks to ensure you have permission to access the +data. Additionally, `stagecoach` gently promotes best practices for data +management, such as symlinking source data, using R or Python projects, +using `git` for version control, `gitignore`-ing sensitive data, +and more! You can use the `stagecoach inspect` command to have it inspect +your manifest and report any issues before you stage your data. + +The best way to get `stagecoach` is to install it using `uv`. +First, make sure you have [`uv` installed](https://docs.astral.sh/uv/getting-started/installation/): + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Then, [create a project](https://docs.astral.sh/uv/concepts/projects/init/#creating-a-minimal-project) that `uv` will isolate for you with a +virtual environment: + +```bash +uv init my-project +uv venv +``` + +Finally, you can install `stagecoach` to your virtual environment +with the following command: + +```bash +# by using tool install, you get this tool available as a command line tool +uv tool install https://github.com/GoldenPlanetaryHealthLab/stagecoach/tree/2-Stage-Data +``` + +## Usage + +First, hail a `stagecoach`: + +```bash +stagecoach hail +``` + +If you are NOT part of a Frontier workspace, +you must use the `--outpost` flag to mock a +Frontier workspace. This flag was designed with +collaborators in mind who might not be on FASRC but still want to use `stagecoach` to stage data for their projects. + +```bash +stagecoach hail --outpost +``` + +Following this command, you will be prompted to answer a few questions +about your project and the data you want to stage. +This will help prefill a `stagecoach_manifest.yaml` file for you, which you +can then edit to customize your data staging. + +Once you've filled in the manifest, use the `stagecoach inspect` command to check for any issues with your manifest: + +```bash +stagecoach inspect +``` + +If there are no issues, you can proceed to stage your data with the `stagecoach stage` command: + +```bash +stagecoach stage +``` + +This will run the necessary checks to ensure you have permission to access +the data, and then stage the data according to your manifest configuration. + +Run `stagecoach [COMMAND] --help` to see more details about each command and its options. From 6f1fc15714dcadb03fac31761e4794ad1d56d48f Mon Sep 17 00:00:00 2001 From: Tinashe Michael Tapera Date: Tue, 19 May 2026 17:23:05 -0400 Subject: [PATCH 10/10] Fix installation command for tool in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be66765..38e35c6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ with the following command: ```bash # by using tool install, you get this tool available as a command line tool -uv tool install https://github.com/GoldenPlanetaryHealthLab/stagecoach/tree/2-Stage-Data +uv tool install git+https://github.com/GoldenPlanetaryHealthLab/stagecoach.git@2-Stage-Data ``` ## Usage