Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 73 additions & 1 deletion src/deadline/client/cli/_groups/bundle_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,16 @@ def cli_bundle():
Use `submit` for headless/scripted submission, or `gui-submit` to
review and edit parameters in a GUI before submitting.

\b
A job bundle directory must contain template.yaml (or template.json).
It may also include parameter_values.yaml and asset_references.yaml.

\b
Scripted workflow (no prompts):
deadline bundle submit ./bundle --yes
deadline job wait --job-id <id>
deadline job download-output --job-id <id> --yes
Comment on lines +65 to +68

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: let's recommend the new flags!


\b
Learn more about [job bundles](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html)
"""
Expand Down Expand Up @@ -218,7 +228,10 @@ def _interactive_confirmation_prompt(message: str, default_response: bool) -> bo
"--known-asset-path",
multiple=True,
help="Path that should not generate warnings when outside storage profile locations. "
"Can be specified multiple times for different paths.",
"Use this when submitting from a temporary or non-standard directory to suppress "
"the 'unknown asset paths' confirmation prompt. "
"Can be specified multiple times for different paths. "
"Equivalent to adding paths to config setting 'settings.known_asset_paths'.",
)
@click.option(
"--save-debug-snapshot",
Expand All @@ -233,6 +246,20 @@ def _interactive_confirmation_prompt(message: str, default_response: bool) -> bo
"Use when S3 bucket contents may be out of sync with local caches. "
"Overrides the 'settings.force_s3_check' config setting.",
)
@click.option(
"--wait",
is_flag=True,
help="After submitting, block until the job reaches a terminal state (the "
"equivalent of running `deadline job wait` on the new job). Exits non-zero if "
"the job does not succeed.",
)
@click.option(
"--download-on-success",
is_flag=True,
help="Implies --wait: after the job SUCCEEDS, download its output to the paths "
"recorded at submission time (the equivalent of `deadline job download-output`). "
"No-op if the job does not succeed.",
)
@click.argument("job_bundle_dir")
@_handle_error
def bundle_submit(
Expand All @@ -250,6 +277,8 @@ def bundle_submit(
submitter_name,
save_debug_snapshot,
force_s3_check,
wait,
download_on_success,
**args,
):
"""
Expand All @@ -269,6 +298,24 @@ def bundle_submit(
to see its current taskRunStatus, or `deadline job wait --job-id <id>` to
block until the job reaches a terminal state (SUCCEEDED / FAILED / CANCELED).

\b
JOB_BUNDLE_DIR is the path to the directory containing template.yaml (or
template.json), and optionally parameter_values.yaml and
asset_references.yaml.

\b
If asset files reference paths outside the configured storage profile
locations (settings.storage_profile_id), submission will warn about
"unknown asset paths" and prompt for confirmation. To suppress this in
non-interactive use, either pass --known-asset-path <dir> for each additional
root, or pass --yes to auto-confirm all prompts.

\b
To do the whole workflow in one command, add --wait to block until the job
finishes (exit code 0 = success), and --download-on-success to also download
its output once it succeeds. Otherwise you can run the steps separately:
`deadline job wait --job-id <id>` then `deadline job download-output --job-id <id>`.

\b
Learn more about [job bundles](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html)
"""
Expand Down Expand Up @@ -351,6 +398,31 @@ def _check_create_job_wait_canceled() -> bool:
):
config_file.set_setting("defaults.job_id", job_id)

# --download-on-success implies --wait. Skipped when there is no real job
# (e.g. --save-debug-snapshot yields job_id=None).
if job_id and (wait or download_on_success):
farm_id = config_file.get_setting("defaults.farm_id", config=config)
queue_id = config_file.get_setting("defaults.queue_id", config=config)
click.echo(f"Waiting for job {job_id} to complete...")
result = api.wait_for_job_completion(

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new --wait / --download-on-success block runs inside the submission try, so failures that happen after the job is already created get reported as submission failures. For example, if _download_job_output raises a ClientError (it calls deadline.get_job, S3 ops, etc.), it lands in the except ClientError handler below and is re-raised as "Failed to submit the job bundle to AWS Deadline Cloud" — even though the job submitted fine and is running. Likewise the generic except Exception records this as an "on_submit" telemetry error.

For a scripted workflow this is misleading: a non-zero exit + "submission failed" message could prompt a caller to resubmit and create a duplicate job. Consider moving the wait/download after the try/except/finally (once submission has definitively succeeded), or wrapping it in its own try/except with a distinct error message.

farm_id=farm_id, queue_id=queue_id, job_id=job_id, config=config
)
click.echo(f"Job completed with status: {result.status}")
if result.status != "SUCCEEDED":
sys.exit(1)
if download_on_success:
# Imported lazily to avoid a circular import with job_group.
from .job_group import _download_job_output

_download_job_output(

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--download-on-success calls _download_job_output, which only skips the interactive "confirm root paths" prompt when settings.auto_accept is true (see job_group.py if not auto_accept: _prompt_to_confirm_roots(...)). That config value is only set to true when --yes is passed to this command. So deadline bundle submit ./bundle --download-on-success (without --yes) will submit, wait, then block on an interactive prompt during the download — defeating the "one command for scripts/agents" purpose this feature is built for.

Consider either forcing auto-accept for the implied download here, or documenting in the --download-on-success help that --yes is required for a fully non-interactive run.

config=config,
farm_id=farm_id,
queue_id=queue_id,
job_id=job_id,
step_id=None,
task_id=None,
)

except AssetSyncCancelledError as exc:
if sigint_handler.continue_operation:
raise DeadlineOperationError(f"Job submission unexpectedly canceled:\n{exc}") from exc
Expand Down
27 changes: 25 additions & 2 deletions src/deadline/client/cli/_groups/job_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,11 @@ def cli_job():
to check status, read logs, wait for completion, download output,
cancel, or requeue failed tasks.

\b
For scripted workflows, prefer `wait` over polling `get`:
deadline job wait --job-id <id> # blocks, exit 0 = success
deadline job download-output --job-id <id> --yes

Comment on lines +143 to +147

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feels like this string is too specific for the higher leveljob command. Maybe put some example commands that include wait in here.

\b
Learn more about [Deadline Cloud jobs](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/deadline-cloud-jobs.html)
"""
Expand Down Expand Up @@ -239,6 +244,11 @@ def job_get(search_term: Optional[str], **args):
If exactly one job matches, shows full details. If multiple match, shows a summary list.
If no arguments provided, shows the default job from config.

\b
Output includes lifecycleStatus, taskRunStatus, and taskRunStatusCounts.
To block until a job finishes, use `deadline job wait --job-id <id>`
instead of polling this command.

\b
Learn more about [Deadline Cloud jobs](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/deadline-cloud-jobs.html)
"""
Expand Down Expand Up @@ -1105,7 +1115,14 @@ def job_download_output(
):
"""
Download the output of a Deadline Cloud job that was saved as job
attachments.
attachments. Files are downloaded to the paths specified at submission
time (mapped via storage profiles, or unmapped with
--ignore-storage-profiles).

\b
Pass --yes to skip confirmation prompts (useful when scripting).
Pass --ignore-storage-profiles when submitting and downloading on the
same machine to skip storage profile path mapping.
Comment on lines +1122 to +1125

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aren't these flags documented when you call --help already?


Scope is controlled by which ids you pass:

Expand Down Expand Up @@ -1442,7 +1459,8 @@ def job_wait_for_completion(max_poll_interval, timeout, output, **args):

Blocks until the job reaches a terminal state (SUCCEEDED, FAILED,
CANCELED, SUSPENDED, or NOT_COMPATIBLE), then prints any failed
step-task combinations.
step-task combinations. This is the recommended way to poll for job
completion in scripts and automation (instead of looping on `job get`).

Uses exponential backoff for polling, starting at 0.5s and doubling
until reaching --max-poll-interval.
Expand All @@ -1457,6 +1475,11 @@ def job_wait_for_completion(max_poll_interval, timeout, output, **args):
4 - Job was suspended
5 - Job is not compatible

\b
Example (submit then wait):
deadline bundle submit ./bundle --yes
deadline job wait --job-id job-abc123 && deadline job download-output --job-id job-abc123 --yes

\b
Learn more about [Deadline Cloud jobs](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/deadline-cloud-jobs.html)
"""
Expand Down
21 changes: 20 additions & 1 deletion src/deadline/client/cli/_groups/queue_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,19 @@ def cli_queue():
export queue credentials for scripting, inspect queue parameter
definitions, or sync job output for all jobs in a queue.

\b
Available subcommands:
list List queues in the farm
get Get details of a queue (incl. job attachment settings)
paramdefs List parameters from queue environments (e.g. conda)
export-credentials Export temporary queue role credentials
sync-output Incrementally download output for all jobs in queue

\b
Note: There is no subcommand for listing queue environments directly.
Use `deadline queue paramdefs` to see what parameters (and therefore
which queue environments such as Conda) are configured on a queue.

Comment on lines +49 to +61

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't think this is necessary. The current help text looks like:

$ deadline queue --help
Usage: deadline queue [OPTIONS] COMMAND [ARGS]...

  Manage Deadline Cloud queues. View queue details, list available queues,
  export queue credentials for scripting, inspect queue parameter definitions,
  or sync job output for all jobs in a queue.

  Learn more about queues (https://docs.aws.amazon.com/deadline-cloud/latest/userguide/queues.html)

Options:
  -h, --help  Show this message and exit.

Commands:
  export-credentials  Export queue credentials in a format compatible...
  get                 Get the details of a Deadline Cloud queue in the farm.
  list                Lists the available Deadline Cloud queues in the farm.
  paramdefs           Lists the parameter definitions for a Deadline...
  sync-output         Downloads any new job attachment output for all...

\b
Learn more about [queues](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/queues.html)
"""
Expand Down Expand Up @@ -200,7 +213,13 @@ def queue_paramdefs(**args):
Lists the parameter definitions for a Deadline Cloud queue in the farm.

The parameter definitions include all the parameters defined by the
queue environments configured for the queue.
queue environments configured for the queue. This is the way to
discover which queue environments (e.g. Conda, service-managed fleet
software) are attached to a queue and what parameters they expose.

\b
For example, a Conda queue environment defines parameters like
CondaPackages and CondaChannels that job templates can reference.

\b
Learn more about [queue environments](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html)
Expand Down
20 changes: 16 additions & 4 deletions src/deadline/client/cli/_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,24 @@ def deadline(
Common workflows:

\b
Submit a job: deadline bundle submit <path>
Monitor a job: deadline job get --job-id <job-id>
Wait for a job: deadline job wait
Download output: deadline job download-output
Submit a job: deadline bundle submit <path> [--yes]
Wait for completion: deadline job wait --job-id <id> (exit 0=ok)
Download output: deadline job download-output --job-id <id> [--yes]
Monitor a job: deadline job get --job-id <job-id> | deadline job logs
Sync all output: deadline queue sync-output

\b
Scripted end-to-end example (no interactive prompts):
deadline bundle submit ./bundle --yes
deadline job wait --job-id job-abc123
deadline job download-output --job-id job-abc123 --yes
Comment on lines +130 to +133

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Replace with our new command


\b
Configuration:
deadline config show Show current farm/queue/profile
deadline config set <k> <v> Set a config value
deadline auth status Check authentication state

Works with any configured AWS credentials, or with Deadline Cloud monitor
for identity-provider-based login (see `deadline auth login`).

Expand Down
65 changes: 65 additions & 0 deletions test/unit/deadline_client/cli/test_cli_bundle_submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -1437,3 +1437,68 @@ def test_bundle_gui_submit_submitter_info_file_missing_submitter_name(

assert result.exit_code != 0
assert "submitter_name is required" in result.output


def test_cli_bundle_submit_wait_exits_nonzero_on_failure(
fresh_deadline_config, deadline_mock, temp_job_bundle_dir
):
"""--wait blocks on the submitted job and exits non-zero when it does not succeed."""
with open(os.path.join(temp_job_bundle_dir, "template.json"), "w", encoding="utf8") as f:
f.write(MOCK_JOB_TEMPLATE_CASES["MINIMAL_JSON"][1])
deadline_mock.create_job.return_value = MOCK_CREATE_JOB_RESPONSE
deadline_mock.get_job.return_value = MOCK_GET_JOB_RESPONSE

with patch.object(api_module, "wait_for_job_completion") as mock_wait:
mock_wait.return_value = MagicMock(status="FAILED")
runner = CliRunner()
result = runner.invoke(
main,
[
"bundle",
"submit",
temp_job_bundle_dir,
"--farm-id",
MOCK_FARM_ID,
"--queue-id",
MOCK_QUEUE_ID,
"--wait",
],
)

mock_wait.assert_called_once()
assert "Job completed with status: FAILED" in result.output
assert result.exit_code != 0


def test_cli_bundle_submit_download_on_success(
fresh_deadline_config, deadline_mock, temp_job_bundle_dir
):
"""--download-on-success waits, then downloads output when the job SUCCEEDS."""
with open(os.path.join(temp_job_bundle_dir, "template.json"), "w", encoding="utf8") as f:
f.write(MOCK_JOB_TEMPLATE_CASES["MINIMAL_JSON"][1])
deadline_mock.create_job.return_value = MOCK_CREATE_JOB_RESPONSE
deadline_mock.get_job.return_value = MOCK_GET_JOB_RESPONSE

with (
patch.object(api_module, "wait_for_job_completion") as mock_wait,
patch("deadline.client.cli._groups.job_group._download_job_output") as mock_download,
):
mock_wait.return_value = MagicMock(status="SUCCEEDED")
runner = CliRunner()
result = runner.invoke(
main,
[
"bundle",
"submit",
temp_job_bundle_dir,
"--farm-id",
MOCK_FARM_ID,
"--queue-id",
MOCK_QUEUE_ID,
"--download-on-success",
],
)

mock_wait.assert_called_once()
mock_download.assert_called_once()
assert result.exit_code == 0
Loading