From 715b87111d3f02baa93c9c465af18feea6b22294 Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:33:47 -0700 Subject: [PATCH 1/6] fix: consume --conflict-resolution in config-defaults workflow When 'deadline attachment download' runs without --profile and without an explicit --conflict-resolution (relying on config defaults), all CLI options passed to _apply_cli_options_to_config are None, so the else-branch cleanup runs. That branch did not pop 'conflict_resolution', leaving it in args and tripping the 'not standard AWS Deadline Cloud CLI options' RuntimeError guard. Add 'conflict_resolution' to the cleanup list so it is consumed. Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- src/deadline/client/cli/_common.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/deadline/client/cli/_common.py b/src/deadline/client/cli/_common.py index 07e47f9a0..808cb0ad8 100644 --- a/src/deadline/client/cli/_common.py +++ b/src/deadline/client/cli/_common.py @@ -218,7 +218,15 @@ def _apply_cli_options_to_config( ) else: # Remove the standard option names from the args list - for name in ["profile", "farm_id", "region", "queue_id", "job_id", "storage_profile_id"]: + for name in [ + "profile", + "farm_id", + "region", + "queue_id", + "job_id", + "storage_profile_id", + "conflict_resolution", + ]: args.pop(name, None) # Check that the required options have values, auto-selecting if only one exists From ce70fcdac3a25bf24d1b386a1e04df01babc30ce Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:33:54 -0700 Subject: [PATCH 2/6] fix: honor --s3-root-uri independently of --profile 'deadline attachment download'/'upload' unconditionally overwrote s3_root_uri with the queue's job-attachment settings whenever --profile was not passed, discarding any explicit --s3-root-uri. Only fall back to the queue settings when --s3-root-uri was not provided. Also adds arg-plumbing regression tests covering both the --conflict-resolution config-defaults workflow and the --s3-root-uri handling. Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../client/cli/_groups/attachment_group.py | 10 +- .../cli/test_cli_attachment_args.py | 168 ++++++++++++++++++ 2 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 test/unit/deadline_client/cli/test_cli_attachment_args.py diff --git a/src/deadline/client/cli/_groups/attachment_group.py b/src/deadline/client/cli/_groups/attachment_group.py index 458c96445..d03ef69c1 100644 --- a/src/deadline/client/cli/_groups/attachment_group.py +++ b/src/deadline/client/cli/_groups/attachment_group.py @@ -131,7 +131,10 @@ def attachment_download( if not s3_settings: raise MissingJobAttachmentSettingsError(f"Queue {queue_id} has no attachment settings") - s3_root_uri = s3_settings.to_s3_root_uri() + # Only fall back to the queue's S3 settings when the caller did not provide an + # explicit --s3-root-uri. An explicitly-supplied value must always be honored. + if not s3_root_uri: + s3_root_uri = s3_settings.to_s3_root_uri() deadline_client = get_session_client(boto3_session, "deadline", region=region) boto3_session = api.get_queue_user_boto3_session(deadline=deadline_client, config=config) @@ -243,7 +246,10 @@ def attachment_upload( if not s3_settings: raise MissingJobAttachmentSettingsError(f"Queue {queue_id} has no attachment settings") - s3_root_uri = s3_settings.to_s3_root_uri() + # Only fall back to the queue's S3 settings when the caller did not provide an + # explicit --s3-root-uri. An explicitly-supplied value must always be honored. + if not s3_root_uri: + s3_root_uri = s3_settings.to_s3_root_uri() deadline_client = get_session_client(boto3_session, "deadline", region=region) boto3_session = api.get_queue_user_boto3_session(deadline=deadline_client, config=config) diff --git a/test/unit/deadline_client/cli/test_cli_attachment_args.py b/test/unit/deadline_client/cli/test_cli_attachment_args.py new file mode 100644 index 000000000..e8f6e174d --- /dev/null +++ b/test/unit/deadline_client/cli/test_cli_attachment_args.py @@ -0,0 +1,168 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +""" +Tests for argument plumbing in the `deadline attachment` CLI group: + * `--conflict-resolution` must not trip the `_apply_cli_options_to_config` + "not standard CLI options" RuntimeError guard in the config-defaults workflow. + * `--s3-root-uri` must be honored independently of `--profile`. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest +from click.testing import CliRunner + +from deadline.client.cli import main +from deadline.client.config import config_file +from deadline.client.cli._groups import attachment_group +from deadline.job_attachments.models import JobAttachmentS3Settings +from deadline.job_attachments.progress_tracker import DownloadSummaryStatistics +from ..shared_constants import MOCK_FARM_ID, MOCK_QUEUE_ID + +MOCK_REGION = "eu-central-1" + +MOCK_S3_SETTINGS = JobAttachmentS3Settings(s3BucketName="mock-bucket", rootPrefix="MockRootPrefix") + + +def _write_manifest(tmp_path): + manifest_path = tmp_path / "abc123_manifest" + manifest_path.write_text( + json.dumps({"hashAlg": "xxh128", "manifestVersion": "2023-03-03", "paths": []}) + ) + return str(manifest_path) + + +@pytest.fixture +def configured_farm_region(fresh_deadline_config): + config_file.set_setting("defaults.farm_id", MOCK_FARM_ID) + config_file.set_setting("defaults.queue_id", MOCK_QUEUE_ID) + config_file.set_setting("defaults.farm_region", MOCK_REGION) + yield fresh_deadline_config + + +def test_attachment_download_config_defaults_no_conflict_resolution( + configured_farm_region, tmp_path +): + """ + C12: In the config-defaults workflow (no --profile, no --conflict-resolution on the + CLI), the unset --conflict-resolution option must not survive into the + `_apply_cli_options_to_config` args and trip the "not standard CLI options" RuntimeError. + """ + manifest_path = _write_manifest(tmp_path) + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = MOCK_S3_SETTINGS + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, + "_attachment_download", + return_value=DownloadSummaryStatistics(), + ), + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + ], + ) + + assert result.exit_code == 0, result.output + assert "not standard AWS Deadline Cloud CLI options" not in result.output + + +def test_attachment_download_honors_s3_root_uri_without_profile(configured_farm_region, tmp_path): + """ + Bug: --s3-root-uri must be honored even when --profile is not passed. It must not be + overwritten by the queue's job-attachment settings. + """ + manifest_path = _write_manifest(tmp_path) + + explicit_uri = "s3://my-explicit-bucket/my-explicit-prefix" + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = MOCK_S3_SETTINGS + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, + "_attachment_download", + return_value=DownloadSummaryStatistics(), + ) as mock_download, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + "--s3-root-uri", + explicit_uri, + ], + ) + + assert result.exit_code == 0, result.output + mock_download.assert_called_once() + assert mock_download.call_args.kwargs["s3_root_uri"] == explicit_uri + + +def test_attachment_upload_honors_s3_root_uri_without_profile(configured_farm_region, tmp_path): + """ + Bug: --s3-root-uri must be honored on `upload` even when --profile is not passed. It + must not be overwritten by the queue's job-attachment settings. + """ + manifest_path = _write_manifest(tmp_path) + + explicit_uri = "s3://my-explicit-bucket/my-explicit-prefix" + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = MOCK_S3_SETTINGS + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, "_attachment_upload", return_value=MagicMock() + ) as mock_upload, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "upload", + "--manifests", + manifest_path, + "--s3-root-uri", + explicit_uri, + ], + ) + + assert result.exit_code == 0, result.output + mock_upload.assert_called_once() + assert mock_upload.call_args.kwargs["s3_root_uri"] == explicit_uri From fa3bbe8a9d0d71bc6b508cd142795d369f66a098 Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:56:01 -0700 Subject: [PATCH 3/6] fix: allow explicit --s3-root-uri when queue lacks attachment settings Move the MissingJobAttachmentSettingsError guard inside the s3_root_uri fallback so an explicitly-supplied URI is honored even when the queue has no jobAttachmentSettings. Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../client/cli/_groups/attachment_group.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/deadline/client/cli/_groups/attachment_group.py b/src/deadline/client/cli/_groups/attachment_group.py index d03ef69c1..1a7fdf325 100644 --- a/src/deadline/client/cli/_groups/attachment_group.py +++ b/src/deadline/client/cli/_groups/attachment_group.py @@ -128,12 +128,15 @@ def attachment_download( queue_id=queue_id, session=boto3_session, ).jobAttachmentSettings - if not s3_settings: - raise MissingJobAttachmentSettingsError(f"Queue {queue_id} has no attachment settings") # Only fall back to the queue's S3 settings when the caller did not provide an - # explicit --s3-root-uri. An explicitly-supplied value must always be honored. + # explicit --s3-root-uri. An explicitly-supplied value must always be honored, + # even when the queue has no attachment settings of its own. if not s3_root_uri: + if not s3_settings: + raise MissingJobAttachmentSettingsError( + f"Queue {queue_id} has no attachment settings" + ) s3_root_uri = s3_settings.to_s3_root_uri() deadline_client = get_session_client(boto3_session, "deadline", region=region) @@ -243,12 +246,15 @@ def attachment_upload( queue_id=queue_id, session=boto3_session, ).jobAttachmentSettings - if not s3_settings: - raise MissingJobAttachmentSettingsError(f"Queue {queue_id} has no attachment settings") # Only fall back to the queue's S3 settings when the caller did not provide an - # explicit --s3-root-uri. An explicitly-supplied value must always be honored. + # explicit --s3-root-uri. An explicitly-supplied value must always be honored, + # even when the queue has no attachment settings of its own. if not s3_root_uri: + if not s3_settings: + raise MissingJobAttachmentSettingsError( + f"Queue {queue_id} has no attachment settings" + ) s3_root_uri = s3_settings.to_s3_root_uri() deadline_client = get_session_client(boto3_session, "deadline", region=region) From 0a2af63f374bf9f26121ea02a5904c0482b2e90b Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:55:17 -0700 Subject: [PATCH 4/6] test: cover explicit --s3-root-uri when queue lacks attachment settings Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../cli/test_cli_attachment_args.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/test/unit/deadline_client/cli/test_cli_attachment_args.py b/test/unit/deadline_client/cli/test_cli_attachment_args.py index e8f6e174d..a2064dce7 100644 --- a/test/unit/deadline_client/cli/test_cli_attachment_args.py +++ b/test/unit/deadline_client/cli/test_cli_attachment_args.py @@ -4,7 +4,8 @@ Tests for argument plumbing in the `deadline attachment` CLI group: * `--conflict-resolution` must not trip the `_apply_cli_options_to_config` "not standard CLI options" RuntimeError guard in the config-defaults workflow. - * `--s3-root-uri` must be honored independently of `--profile`. + * `--s3-root-uri` must be honored independently of `--profile`, including when the + queue has no jobAttachmentSettings of its own. """ from __future__ import annotations @@ -127,6 +128,52 @@ def test_attachment_download_honors_s3_root_uri_without_profile(configured_farm_ assert mock_download.call_args.kwargs["s3_root_uri"] == explicit_uri +def test_attachment_download_honors_s3_root_uri_when_queue_lacks_settings( + configured_farm_region, tmp_path +): + """ + Bug: an explicit --s3-root-uri must be usable even when the queue has no + jobAttachmentSettings. The MissingJobAttachmentSettingsError guard only applies + when falling back to the queue's settings. + """ + manifest_path = _write_manifest(tmp_path) + + explicit_uri = "s3://my-explicit-bucket/my-explicit-prefix" + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = None + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, + "_attachment_download", + return_value=DownloadSummaryStatistics(), + ) as mock_download, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + "--s3-root-uri", + explicit_uri, + ], + ) + + assert result.exit_code == 0, result.output + mock_download.assert_called_once() + assert mock_download.call_args.kwargs["s3_root_uri"] == explicit_uri + + def test_attachment_upload_honors_s3_root_uri_without_profile(configured_farm_region, tmp_path): """ Bug: --s3-root-uri must be honored on `upload` even when --profile is not passed. It From 1dfb445767ef1570ec44473516c655ab6f02622f Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:45:52 -0700 Subject: [PATCH 5/6] test: cover conflict-resolution threading and missing-settings failure modes Add coverage for: - --conflict-resolution SKIP reaching the _attachment_download call - explicit --s3-root-uri on upload when the queue lacks jobAttachmentSettings - MissingJobAttachmentSettingsError still raised when neither an explicit --s3-root-uri nor queue attachment settings are available Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../cli/test_cli_attachment_args.py | 138 +++++++++++++++++- 1 file changed, 136 insertions(+), 2 deletions(-) diff --git a/test/unit/deadline_client/cli/test_cli_attachment_args.py b/test/unit/deadline_client/cli/test_cli_attachment_args.py index a2064dce7..bdb68d461 100644 --- a/test/unit/deadline_client/cli/test_cli_attachment_args.py +++ b/test/unit/deadline_client/cli/test_cli_attachment_args.py @@ -5,7 +5,9 @@ * `--conflict-resolution` must not trip the `_apply_cli_options_to_config` "not standard CLI options" RuntimeError guard in the config-defaults workflow. * `--s3-root-uri` must be honored independently of `--profile`, including when the - queue has no jobAttachmentSettings of its own. + queue has no jobAttachmentSettings of its own (while the no-URI/no-settings case + must still fail with MissingJobAttachmentSettingsError). + * `--conflict-resolution` must be threaded through to the download call. """ from __future__ import annotations @@ -19,7 +21,8 @@ from deadline.client.cli import main from deadline.client.config import config_file from deadline.client.cli._groups import attachment_group -from deadline.job_attachments.models import JobAttachmentS3Settings +from deadline.job_attachments.exceptions import MissingJobAttachmentSettingsError +from deadline.job_attachments.models import FileConflictResolution, JobAttachmentS3Settings from deadline.job_attachments.progress_tracker import DownloadSummaryStatistics from ..shared_constants import MOCK_FARM_ID, MOCK_QUEUE_ID @@ -174,6 +177,137 @@ def test_attachment_download_honors_s3_root_uri_when_queue_lacks_settings( assert mock_download.call_args.kwargs["s3_root_uri"] == explicit_uri +def test_attachment_download_missing_settings_and_no_uri_still_raises( + configured_farm_region, tmp_path +): + """ + Failure mode: when the caller does NOT pass --s3-root-uri and the queue has no + jobAttachmentSettings, the command must still fail with + MissingJobAttachmentSettingsError rather than proceeding with no S3 root. + """ + manifest_path = _write_manifest(tmp_path) + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = None + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, + "_attachment_download", + return_value=DownloadSummaryStatistics(), + ) as mock_download, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + ], + ) + + assert result.exit_code != 0 + assert isinstance(result.exception, (MissingJobAttachmentSettingsError, SystemExit)) or ( + "has no attachment settings" in result.output + ) + mock_download.assert_not_called() + + +def test_attachment_download_conflict_resolution_reaches_download_call( + configured_farm_region, tmp_path +): + """ + --conflict-resolution must be applied to the config and threaded through to the + _attachment_download call, not just silently consumed. + """ + manifest_path = _write_manifest(tmp_path) + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = MOCK_S3_SETTINGS + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, + "_attachment_download", + return_value=DownloadSummaryStatistics(), + ) as mock_download, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + "--conflict-resolution", + FileConflictResolution.SKIP.name, + ], + ) + + assert result.exit_code == 0, result.output + mock_download.assert_called_once() + assert mock_download.call_args.kwargs["conflict_resolution"] == FileConflictResolution.SKIP + + +def test_attachment_upload_honors_s3_root_uri_when_queue_lacks_settings( + configured_farm_region, tmp_path +): + """ + Bug: an explicit --s3-root-uri must be usable on `upload` even when the queue has no + jobAttachmentSettings. The MissingJobAttachmentSettingsError guard only applies when + falling back to the queue's settings. + """ + manifest_path = _write_manifest(tmp_path) + + explicit_uri = "s3://my-explicit-bucket/my-explicit-prefix" + + mock_queue = MagicMock() + mock_queue.jobAttachmentSettings = None + + with ( + patch.object(attachment_group.api, "get_boto3_session", return_value=MagicMock()), + patch.object(attachment_group, "get_queue", return_value=mock_queue), + patch.object(attachment_group, "get_session_client", return_value=MagicMock()), + patch.object( + attachment_group.api, "get_queue_user_boto3_session", return_value=MagicMock() + ), + patch.object( + attachment_group, "_attachment_upload", return_value=MagicMock() + ) as mock_upload, + ): + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "upload", + "--manifests", + manifest_path, + "--s3-root-uri", + explicit_uri, + ], + ) + + assert result.exit_code == 0, result.output + mock_upload.assert_called_once() + assert mock_upload.call_args.kwargs["s3_root_uri"] == explicit_uri + + def test_attachment_upload_honors_s3_root_uri_without_profile(configured_farm_region, tmp_path): """ Bug: --s3-root-uri must be honored on `upload` even when --profile is not passed. It From e937dd05299c744b8e1e10defd48bd584a7a8600 Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:43:17 -0700 Subject: [PATCH 6/6] test: verify explicit --s3-root-uri end-to-end against moto S3 Add two end-to-end tests that run the attachment download/upload CLI with no job-attachments internals mocked: the queue-role credential path (AssumeQueueRoleForUser via the deadline mock), real manifest decoding, and the real S3 transfer code all execute against moto S3. Both the queue's configured bucket and the explicitly-named bucket exist; the tests prove the transfer touches the explicit bucket (download reads its content instead of a same-key decoy in the queue bucket, upload writes CAS data there and leaves the queue bucket empty). Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../cli/test_cli_attachment_args.py | 153 +++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/test/unit/deadline_client/cli/test_cli_attachment_args.py b/test/unit/deadline_client/cli/test_cli_attachment_args.py index bdb68d461..2435704bf 100644 --- a/test/unit/deadline_client/cli/test_cli_attachment_args.py +++ b/test/unit/deadline_client/cli/test_cli_attachment_args.py @@ -8,23 +8,29 @@ queue has no jobAttachmentSettings of its own (while the no-URI/no-settings case must still fail with MissingJobAttachmentSettingsError). * `--conflict-resolution` must be threaded through to the download call. + * End-to-end against moto S3 (real transfer code, queue-role credential path): the + explicit --s3-root-uri bucket is the one actually read from / written to, not the + queue's configured bucket. """ from __future__ import annotations import json +import os from unittest.mock import MagicMock, patch +import boto3 import pytest from click.testing import CliRunner from deadline.client.cli import main from deadline.client.config import config_file from deadline.client.cli._groups import attachment_group +from deadline.job_attachments.asset_manifests.hash_algorithms import HashAlgorithm, hash_data from deadline.job_attachments.exceptions import MissingJobAttachmentSettingsError from deadline.job_attachments.models import FileConflictResolution, JobAttachmentS3Settings from deadline.job_attachments.progress_tracker import DownloadSummaryStatistics -from ..shared_constants import MOCK_FARM_ID, MOCK_QUEUE_ID +from ..shared_constants import MOCK_BUCKET_NAME, MOCK_FARM_ID, MOCK_QUEUE_ID MOCK_REGION = "eu-central-1" @@ -308,6 +314,151 @@ def test_attachment_upload_honors_s3_root_uri_when_queue_lacks_settings( assert mock_upload.call_args.kwargs["s3_root_uri"] == explicit_uri +# ─── End-to-end tests against moto S3 ──────────────────────────────────────── +# +# These run the full CLI path with NO job-attachments internals mocked: the queue-role +# credential provider (AssumeQueueRoleForUser via the deadline_mock), the real manifest +# decode, and the real S3 transfer code all execute against moto's S3. Two buckets +# exist — the queue's configured bucket (MOCK_BUCKET_NAME, created by the deadline_mock +# fixture) and an "other" bucket named by --s3-root-uri — and the tests assert the +# transfer actually touches the OTHER bucket, not just that the URI was plumbed along. + +OTHER_BUCKET = "explicit-other-bucket" +OTHER_PREFIX = "OtherPrefix" +QUEUE_ROOT_PREFIX = "MockRootPrefix" + +FULL_GET_QUEUE_RESPONSE = { + "queueId": MOCK_QUEUE_ID, + "farmId": MOCK_FARM_ID, + "displayName": "Mock Queue", + "status": "IDLE", + "defaultBudgetAction": "NONE", + "jobAttachmentSettings": { + "rootPrefix": QUEUE_ROOT_PREFIX, + "s3BucketName": MOCK_BUCKET_NAME, + }, +} + + +@pytest.fixture +def moto_farm_config(fresh_deadline_config, deadline_mock): + """Config-defaults workflow pointed at the moto-backed farm/queue in us-west-2.""" + config_file.set_setting("defaults.farm_id", MOCK_FARM_ID) + config_file.set_setting("defaults.queue_id", MOCK_QUEUE_ID) + config_file.set_setting("defaults.farm_region", "us-west-2") + # The job_attachments get_queue helper requires more response fields than the + # conftest default provides. + deadline_mock.get_queue.return_value = FULL_GET_QUEUE_RESPONSE + s3 = boto3.client("s3", region_name="us-west-2") + s3.create_bucket( + Bucket=OTHER_BUCKET, CreateBucketConfiguration={"LocationConstraint": "us-west-2"} + ) + yield s3 + + +def _write_real_manifest(directory, file_rel_path: str, content: bytes, manifest_name: str) -> str: + """Write a valid v2023-03-03 manifest for a single file with real xxh128 hashes.""" + file_hash = hash_data(content, HashAlgorithm.XXH128) + manifest = { + "hashAlg": "xxh128", + "manifestVersion": "2023-03-03", + "totalSize": len(content), + "paths": [{"path": file_rel_path, "hash": file_hash, "size": len(content), "mtime": 1}], + } + manifest_path = os.path.join(directory, manifest_name) + with open(manifest_path, "w", encoding="utf8") as f: + json.dump(manifest, f) + return manifest_path + + +def test_attachment_download_reads_from_explicit_bucket_moto(moto_farm_config, temp_cwd, tmp_path): + """ + End-to-end: `attachment download --s3-root-uri s3://other-bucket/...` must GET the + CAS object from the OTHER bucket. A decoy object with the same CAS key exists in the + queue's configured bucket with different content; the downloaded bytes prove which + bucket was read. + """ + s3 = moto_farm_config + content = b"explicit bucket content" + decoy = b"WRONG: queue bucket content!!!" + file_hash = hash_data(content, HashAlgorithm.XXH128) + + # CAS object in the explicitly-named bucket, decoy under the queue's settings. + s3.put_object(Bucket=OTHER_BUCKET, Key=f"{OTHER_PREFIX}/Data/{file_hash}.xxh128", Body=content) + s3.put_object( + Bucket=MOCK_BUCKET_NAME, Key=f"{QUEUE_ROOT_PREFIX}/Data/{file_hash}.xxh128", Body=decoy + ) + + manifest_path = _write_real_manifest(str(tmp_path), "test_file.txt", content, "e2e_manifest") + + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "download", + "--manifests", + manifest_path, + "--s3-root-uri", + f"s3://{OTHER_BUCKET}/{OTHER_PREFIX}", + ], + ) + + assert result.exit_code == 0, result.output + # No path mapping: files land under // + downloaded = os.path.join(temp_cwd, "e2e_manifest", "test_file.txt") + assert os.path.isfile(downloaded), result.output + with open(downloaded, "rb") as f: + assert f.read() == content # not the queue-bucket decoy + + +def test_attachment_upload_writes_to_explicit_bucket_moto(moto_farm_config, tmp_path): + """ + End-to-end: `attachment upload --s3-root-uri s3://other-bucket/...` must PUT the CAS + object and manifest into the OTHER bucket and leave the queue's configured bucket + untouched. + """ + s3 = moto_farm_config + content = b"asset file uploaded to the explicit bucket" + file_hash = hash_data(content, HashAlgorithm.XXH128) + + root_dir = tmp_path / "asset_root" + root_dir.mkdir() + (root_dir / "asset_file.txt").write_bytes(content) + + # For --root-dirs, the manifest file name must contain the hash of the root path. + root_hash = hash_data(str(root_dir).encode("utf-8"), HashAlgorithm.XXH128) + manifest_path = _write_real_manifest( + str(tmp_path), "asset_file.txt", content, f"{root_hash}_input" + ) + + runner = CliRunner() + result = runner.invoke( + main, + [ + "attachment", + "upload", + "--manifests", + manifest_path, + "--root-dirs", + str(root_dir), + "--s3-root-uri", + f"s3://{OTHER_BUCKET}/{OTHER_PREFIX}", + ], + ) + + assert result.exit_code == 0, result.output + + # The CAS object was written to the explicitly-named bucket... + cas_key = f"{OTHER_PREFIX}/Data/{file_hash}.xxh128" + body = s3.get_object(Bucket=OTHER_BUCKET, Key=cas_key)["Body"].read() + assert body == content + + # ...and nothing was written to the queue's configured bucket. + queue_objects = s3.list_objects_v2(Bucket=MOCK_BUCKET_NAME) + assert queue_objects.get("KeyCount", 0) == 0, queue_objects.get("Contents") + + def test_attachment_upload_honors_s3_root_uri_without_profile(configured_farm_region, tmp_path): """ Bug: --s3-root-uri must be honored on `upload` even when --profile is not passed. It