diff --git a/bagger/README.md b/bagger/README.md index 48ac9f0..f0e1f81 100644 --- a/bagger/README.md +++ b/bagger/README.md @@ -24,7 +24,7 @@ the environment path. One simple way to do this is to download the executable to environment bin directory: ```text -$ wget -P venv/bin https://s3.amazonaws.com/aptrust.public.download/dart-runner/v0.95-beta/linux-x64/dart-runner +$ wget -P venv/bin https://s3.amazonaws.com/aptrust.public.download/dart-runner/v1.0.4/linux/amd64/dart-runner $ chmod +x venv/bin/dart-runner ``` @@ -40,7 +40,7 @@ ReBACH-Bagger can be used on the command line by running the main script as a mo $ python -m bagger.scripts.main -h usage: main.py [-h] [-c config_file] [-b batch_dir] [-d | --delete | --no-delete] [-o archival_staging_storage] [-w workflow_file] [--dart_command dart_command] - [--overwrite | --no-overwrite] [--dry-run] + [--overwrite | --no-overwrite] [--skip-artifacts] [--dry-run] path positional arguments: @@ -62,6 +62,8 @@ optional arguments: Command to invoke DART Runner. --overwrite, --no-overwrite Overwrite duplicate bags. (default: False) + --skip-artifacts + Do not include the artifacts folder if true (default: True. Requires dart-runner v1.0 or higher) --dry-run, --dryrun Log execution steps without actually executing. (default: False) ``` @@ -74,7 +76,7 @@ example follows: from bagger.bag import Bagger # Instantiate Bagger with necessary arguments -B = Bagger(workflow, archival_staging_storage, delete, dart_command, config, log) +B = Bagger(workflow, archival_staging_storage, delete, dart_command, skip_artifacts, config, log) # Run DART on the package specified with path run = B.run_dart(path) diff --git a/bagger/bag.py b/bagger/bag.py index d24134c..1b012c2 100644 --- a/bagger/bag.py +++ b/bagger/bag.py @@ -2,7 +2,6 @@ from logging import Logger from os import PathLike from pathlib import Path -from shutil import rmtree from typing import Union from figshare.Utils import extract_item_id_only, extract_version_only, extract_metadata_hash_only, check_local_path, compare_hash @@ -19,7 +18,7 @@ class Bagger: def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delete: bool, - dart_command: str, config: dict, log: Logger, + dart_command: str, skip_artifacts: bool, config: dict, log: Logger, overwrite: bool, dryrun: bool = False) -> None: """ Set up environment for generating bags with DART @@ -28,6 +27,7 @@ def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delet :param archival_staging_storage: Directory for generated bag output by DART if no upload :param delete: Delete output bag if True :param dart_command: Path to DART executable + :param skip_artifacts: Do not include the artifacts folder if true :param config: Config dict :param log: Logger object :param overwrite: Overwrite duplicate bags if True @@ -36,6 +36,7 @@ def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delet self.log: Logger = log self.dart_command: str = dart_command self.delete: bool = delete + self.skip_artifacts: bool = skip_artifacts self.archival_staging_storage: PathLike = archival_staging_storage self.workflow: PathLike = workflow self.workflow_file: TemporaryFile = None @@ -173,7 +174,7 @@ def run_dart(self, package_path: PathLike) -> Status: bag_name, metadata_tags = init_status job = Job(self.workflow, bag_name, self.archival_staging_storage, self.delete, - self.dart_command, self.log) + self.dart_command, self.log, self.skip_artifacts) job.add_file(package_path) @@ -191,11 +192,6 @@ def run_dart(self, package_path: PathLike) -> Status: if data: data_json = json.loads(data) - package_artifact = data_json['packageResult']['filepath'].replace('.tar', '_artifacts') - package_artifact = Path(package_artifact) - if package_artifact.exists() and package_artifact.is_dir(): - rmtree(package_artifact) - errors = data_json['packageResult']['errors'] validation_result = data_json.get('validationResult', data_json.get('validationResults')) if isinstance(validation_result, list): diff --git a/bagger/config/__init__.py b/bagger/config/__init__.py index 7c58a6c..420d1f9 100644 --- a/bagger/config/__init__.py +++ b/bagger/config/__init__.py @@ -73,6 +73,9 @@ def get_args(path: Optional[PathLike] = None, parser.add_argument('--dry-run', '--dryrun', help='Log execution steps without actually executing. (default: False)', action='store_true') + parser.add_argument('--skip-artifacts', + help='Don\'t save artifacts (tag files and manifests) to a separate directory when creating bags. (default: True)', + action='store_false') parser.add_argument('path', help='Path to the package or batch directory.') args, remaining_argv = parser.parse_known_args(remaining_argv) diff --git a/bagger/job.py b/bagger/job.py index 9f4f427..1670f58 100644 --- a/bagger/job.py +++ b/bagger/job.py @@ -9,7 +9,7 @@ class Job: def __init__(self, workflow: PathLike, bag_name: str, archival_staging_storage: PathLike, - delete: bool, dart_command: str, log: Logger) -> None: + delete: bool, dart_command: str, log: Logger, skip_artifacts: bool) -> None: """ Init the Job class with attributes for passing to DART @@ -17,12 +17,14 @@ def __init__(self, workflow: PathLike, bag_name: str, archival_staging_storage: :param bag_name: Name of bag to generate :param archival_staging_storage: Directory for outputting temp bag prior to upload :param delete: Delete the output bag if true + :param skip_artifacts: Do not include the artifacts folder if true :param dart_command: Command to run DART executable """ self.workflow: PathLike = workflow self.bag_name: str = bag_name self.archival_staging_storage: PathLike = archival_staging_storage self.delete: bool = delete + self.skip_artifacts: bool = skip_artifacts self.dart_command: str = dart_command self.files: list[str] = [] self.tags: list[dict[str, str]] = [] @@ -30,7 +32,7 @@ def __init__(self, workflow: PathLike, bag_name: str, archival_staging_storage: def __str__(self): return f"Job( workflow='{self.workflow}', bag_name='{self.bag_name}', " \ - f"archival_staging_storage='{self.archival_staging_storage}', delete={self.delete}, " \ + f"archival_staging_storage='{self.archival_staging_storage}', delete={self.delete}, skip_artifacts={self.skip_artifacts}, " \ f"dart_command='{self.dart_command}', log='{self.log.handlers[-1].baseFilename}' " \ f"files={self.files} " \ f"tags={self.tags} )" \ @@ -81,7 +83,8 @@ def run(self) -> tuple[str, str, int]: cmd = (f"{self.dart_command} " f"--workflow={self.workflow} " f"--output-dir={self.archival_staging_storage} " - f"--delete={self.delete}") + f"--delete={self.delete} " + f"--skip-artifacts={self.skip_artifacts}") child = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True, text=True) stdout_data, stderr_data = child.communicate(job_params + "\n") diff --git a/bagger/scripts/main.py b/bagger/scripts/main.py index cc32619..dfcaf65 100755 --- a/bagger/scripts/main.py +++ b/bagger/scripts/main.py @@ -51,7 +51,7 @@ def main() -> None: os.environ['WASABI_SECRET_ACCESS_KEY'] = config['Wasabi']['secret_key'] bagger = Bagger(workflow=args.workflow, archival_staging_storage=args.archival_staging_storage, - delete=args.delete, dart_command=args.dart_command, + delete=args.delete, dart_command=args.dart_command, skip_artifacts=args.skip_artifacts, config=config, log=log, overwrite=args.overwrite, dryrun=args.dry_run) if args.batch: diff --git a/figshare/Integration.py b/figshare/Integration.py index aae46a5..97c22b9 100644 --- a/figshare/Integration.py +++ b/figshare/Integration.py @@ -93,7 +93,7 @@ def post_process_script_function(self, *args): preservation_package_name = os.path.basename(preservation_package_path) bagger = Bagger(workflow=args.workflow, archival_staging_storage=args.archival_staging_storage, - delete=args.delete, dart_command=args.dart_command, + delete=args.delete, dart_command=args.dart_command, skip_artifacts=args.skip_artifacts, config=config, log=log, overwrite=args.overwrite, dryrun=False) if args.batch: