Feat: Add --skip-artifacts flag to not create artifacts folder

bagger/README.md

@@ -24,7 +24,7 @@ the environment path. One simple way to do this is to download the executable to
 environment bin directory:
 
 ```text
-$ wget -P venv/bin https://s3.amazonaws.com/aptrust.public.download/dart-runner/v0.95-beta/linux-x64/dart-runner
+$ wget -P venv/bin https://s3.amazonaws.com/aptrust.public.download/dart-runner/v1.0.4/linux/amd64/dart-runner
 $ chmod +x venv/bin/dart-runner
 ```
 
@@ -40,7 +40,7 @@ ReBACH-Bagger can be used on the command line by running the main script as a mo
 $ python -m bagger.scripts.main -h
 usage: main.py [-h] [-c config_file] [-b batch_dir] [-d | --delete | --no-delete]
                [-o archival_staging_storage] [-w workflow_file] [--dart_command dart_command]
-               [--overwrite | --no-overwrite] [--dry-run]
+               [--overwrite | --no-overwrite] [--skip-artifacts] [--dry-run]
                path
 
 positional arguments:
@@ -62,6 +62,8 @@ optional arguments:
                         Command to invoke DART Runner.
   --overwrite, --no-overwrite
                         Overwrite duplicate bags. (default: False)
+  --skip-artifacts
+                        Do not include the artifacts folder if true (default: True. Requires dart-runner v1.0 or higher) 
   --dry-run, --dryrun   Log execution steps without actually executing. (default: False)
 ```
 
@@ -74,7 +76,7 @@ example follows:
 from bagger.bag import Bagger
 
 # Instantiate Bagger with necessary arguments
-B = Bagger(workflow, archival_staging_storage, delete, dart_command, config, log)
+B = Bagger(workflow, archival_staging_storage, delete, dart_command, skip_artifacts, config, log)
 
 # Run DART on the package specified with path
 run = B.run_dart(path)

bagger/bag.py

@@ -2,7 +2,6 @@
 from logging import Logger
 from os import PathLike
 from pathlib import Path
-from shutil import rmtree
 from typing import Union
 
 from figshare.Utils import extract_item_id_only, extract_version_only, extract_metadata_hash_only, check_local_path, compare_hash
@@ -19,7 +18,7 @@
 class Bagger:
 
     def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delete: bool,
-                 dart_command: str, config: dict, log: Logger,
+                 dart_command: str, skip_artifacts: bool, config: dict, log: Logger,
                  overwrite: bool, dryrun: bool = False) -> None:
         """
         Set up environment for generating bags with DART
@@ -28,6 +27,7 @@ def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delet
         :param archival_staging_storage: Directory for generated bag output by DART if no upload
         :param delete: Delete output bag if True
         :param dart_command: Path to DART executable
+        :param skip_artifacts: Do not include the artifacts folder if true
         :param config: Config dict
         :param log: Logger object
         :param overwrite: Overwrite duplicate bags if True
@@ -36,6 +36,7 @@ def __init__(self, workflow: PathLike, archival_staging_storage: PathLike, delet
         self.log: Logger = log
         self.dart_command: str = dart_command
         self.delete: bool = delete
+        self.skip_artifacts: bool = skip_artifacts
         self.archival_staging_storage: PathLike = archival_staging_storage
         self.workflow: PathLike = workflow
         self.workflow_file: TemporaryFile = None
@@ -173,7 +174,7 @@ def run_dart(self, package_path: PathLike) -> Status:
             bag_name, metadata_tags = init_status
 
         job = Job(self.workflow, bag_name, self.archival_staging_storage, self.delete,
-                  self.dart_command, self.log)
+                  self.dart_command, self.log, self.skip_artifacts)
 
         job.add_file(package_path)
 
@@ -191,11 +192,6 @@ def run_dart(self, package_path: PathLike) -> Status:
         if data:
             data_json = json.loads(data)
 
-            package_artifact = data_json['packageResult']['filepath'].replace('.tar', '_artifacts')
-            package_artifact = Path(package_artifact)
-            if package_artifact.exists() and package_artifact.is_dir():
-                rmtree(package_artifact)
-
             errors = data_json['packageResult']['errors']
             validation_result = data_json.get('validationResult', data_json.get('validationResults'))
             if isinstance(validation_result, list):

bagger/config/__init__.py

@@ -73,6 +73,9 @@ def get_args(path: Optional[PathLike] = None,
     parser.add_argument('--dry-run', '--dryrun',
                         help='Log execution steps without actually executing. (default: False)',
                         action='store_true')
+    parser.add_argument('--skip-artifacts',
+                        help='Don\'t save artifacts (tag files and manifests) to a separate directory when creating bags. (default: True)',
+                        action='store_false')
     parser.add_argument('path', help='Path to the package or batch directory.')
     args, remaining_argv = parser.parse_known_args(remaining_argv)
 

bagger/job.py

@@ -9,28 +9,30 @@
 class Job:
 
     def __init__(self, workflow: PathLike, bag_name: str, archival_staging_storage: PathLike,
-                 delete: bool, dart_command: str, log: Logger) -> None:
+                 delete: bool, dart_command: str, log: Logger, skip_artifacts: bool) -> None:
         """
         Init the Job class with attributes for passing to DART
 
         :param workflow: Workflow JSON file
         :param bag_name: Name of bag to generate
         :param archival_staging_storage: Directory for outputting temp bag prior to upload
         :param delete: Delete the output bag if true
+        :param skip_artifacts: Do not include the artifacts folder if true
         :param dart_command: Command to run DART executable
         """
         self.workflow: PathLike = workflow
         self.bag_name: str = bag_name
         self.archival_staging_storage: PathLike = archival_staging_storage
         self.delete: bool = delete
+        self.skip_artifacts: bool = skip_artifacts
         self.dart_command: str = dart_command
         self.files: list[str] = []
         self.tags: list[dict[str, str]] = []
         self.log: Logger = log
 
     def __str__(self):
         return f"Job( workflow='{self.workflow}', bag_name='{self.bag_name}', " \
-               f"archival_staging_storage='{self.archival_staging_storage}', delete={self.delete}, " \
+               f"archival_staging_storage='{self.archival_staging_storage}', delete={self.delete}, skip_artifacts={self.skip_artifacts}, " \
                f"dart_command='{self.dart_command}', log='{self.log.handlers[-1].baseFilename}' " \
                f"files={self.files} " \
                f"tags={self.tags} )" \
@@ -81,7 +83,8 @@ def run(self) -> tuple[str, str, int]:
         cmd = (f"{self.dart_command} "
                f"--workflow={self.workflow} "
                f"--output-dir={self.archival_staging_storage} "
-               f"--delete={self.delete}")
+               f"--delete={self.delete} "
+               f"--skip-artifacts={self.skip_artifacts}")
         child = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE,
                       close_fds=True, text=True)
         stdout_data, stderr_data = child.communicate(job_params + "\n")

bagger/scripts/main.py

@@ -51,7 +51,7 @@ def main() -> None:
     os.environ['WASABI_SECRET_ACCESS_KEY'] = config['Wasabi']['secret_key']
 
     bagger = Bagger(workflow=args.workflow, archival_staging_storage=args.archival_staging_storage,
-                    delete=args.delete, dart_command=args.dart_command,
+                    delete=args.delete, dart_command=args.dart_command, skip_artifacts=args.skip_artifacts,
                     config=config, log=log, overwrite=args.overwrite, dryrun=args.dry_run)
 
     if args.batch:

figshare/Integration.py

@@ -93,7 +93,7 @@ def post_process_script_function(self, *args):
 
             preservation_package_name = os.path.basename(preservation_package_path)
             bagger = Bagger(workflow=args.workflow, archival_staging_storage=args.archival_staging_storage,
-                            delete=args.delete, dart_command=args.dart_command,
+                            delete=args.delete, dart_command=args.dart_command, skip_artifacts=args.skip_artifacts,
                             config=config, log=log, overwrite=args.overwrite, dryrun=False)
 
             if args.batch: