Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/docs/assets/img/coverage.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions lib/src/blackfish/cli/services/speech_recognition.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ def run_speech_recognition(

if dry_run:
service = SpeechRecognition(
id=uuid4(),
name=name,
model=repo_id,
profile=profile.name,
Expand Down
1 change: 1 addition & 0 deletions lib/src/blackfish/cli/services/text_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ def run_text_generation(

if dry_run:
service = TextGeneration(
id=uuid4(),
name=name,
model=repo_id,
profile=profile.name,
Expand Down
5 changes: 5 additions & 0 deletions lib/tests/cli/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
from click.testing import CliRunner
from unittest.mock import patch

from blackfish.server.config import ContainerProvider


@pytest.fixture()
def cli_runner() -> CliRunner:
Expand All @@ -18,4 +20,7 @@ def mock_config():
mock_config.HOME_DIR = (
Path(__file__).parent.parent / "tests",
) # "/tmp/blackfish-test"
# A real provider, so job scripts rendered from this config aren't
# empty: the templates branch on `provider == "docker"`.
mock_config.CONTAINER_PROVIDER = ContainerProvider.Docker
yield mock_config
50 changes: 17 additions & 33 deletions lib/tests/cli/test_cli_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import shlex
import pytest
import requests
from unittest.mock import patch, Mock, MagicMock
from unittest.mock import patch, Mock
from blackfish.cli.__main__ import main
from blackfish.server.models.profile import LocalProfile, SlurmProfile

Expand Down Expand Up @@ -99,26 +99,22 @@ def test_dry_run_local_profile(self, cli_runner, mock_config, local_profile):
patch(
"blackfish.cli.services.text_generation.get_model_dir"
) as mock_get_model_dir,
patch(
"blackfish.cli.services.text_generation.TextGeneration"
) as mock_service_class,
):
mock_deserialize.return_value = local_profile
mock_get_models.return_value = ["openai/gpt-2"]
mock_get_revisions.return_value = ["abc123"]
mock_get_latest.return_value = "abc123"
mock_get_model_dir.return_value = "/path/to/model"

mock_service = MagicMock()
mock_service.image = "text_generation"
mock_service.render_job_script.return_value = "#!/bin/bash\necho test"
mock_service_class.return_value = mock_service

result = cli_runner.invoke(main, cmd)

assert result.exit_code == 0, result.exception
assert "Rendering job script" in result.output
assert "model: openai/gpt-2" in result.output
assert "profile: default" in result.output
# The script itself, not just the echoed metadata above it.
script = result.output.split("> image_ref:")[-1]
assert "docker run" in script

def test_dry_run_slurm_profile(self, cli_runner, mock_config, slurm_profile):
"""Test dry run with SlurmProfile renders job script."""
Expand Down Expand Up @@ -147,27 +143,23 @@ def test_dry_run_slurm_profile(self, cli_runner, mock_config, slurm_profile):
patch(
"blackfish.cli.services.text_generation.get_model_dir"
) as mock_get_model_dir,
patch(
"blackfish.cli.services.text_generation.TextGeneration"
) as mock_service_class,
):
mock_deserialize.return_value = slurm_profile
mock_get_models.return_value = ["openai/gpt-2"]
mock_get_revisions.return_value = ["abc123"]
mock_get_latest.return_value = "abc123"
mock_get_model_dir.return_value = "/path/to/model"

mock_service = MagicMock()
mock_service.scheduler = "slurm"
mock_service.render_job_script.return_value = "#!/bin/bash\n#SBATCH"
mock_service_class.return_value = mock_service

result = cli_runner.invoke(main, cmd)

assert result.exit_code == 0, result.exception
assert "Rendering job script" in result.output
assert "model: openai/gpt-2" in result.output
assert "profile: cluster" in result.output
assert "host: hpc.example.com" in result.output
# The script itself, not just the echoed metadata above it.
script = result.output.split("> image_ref:")[-1]
assert "#SBATCH" in script

def test_success_local_profile(self, cli_runner, mock_config, local_profile):
"""Test successful API call with LocalProfile."""
Expand Down Expand Up @@ -627,26 +619,22 @@ def test_dry_run_local_profile(self, cli_runner, mock_config, local_profile):
patch(
"blackfish.cli.services.speech_recognition.get_model_dir"
) as mock_get_model_dir,
patch(
"blackfish.cli.services.speech_recognition.SpeechRecognition"
) as mock_service_class,
):
mock_deserialize.return_value = local_profile
mock_get_models.return_value = ["openai/whisper-tiny"]
mock_get_revisions.return_value = ["abc123"]
mock_get_latest.return_value = "abc123"
mock_get_model_dir.return_value = "/path/to/models/whisper-tiny"

mock_service = MagicMock()
mock_service.image = "speech_recognition"
mock_service.render_job_script.return_value = "#!/bin/bash\necho test"
mock_service_class.return_value = mock_service

result = cli_runner.invoke(main, cmd)

assert result.exit_code == 0, result.exception
assert "Rendering job script" in result.output
assert "model: openai/whisper-tiny" in result.output
assert "profile: default" in result.output
# The script itself, not just the echoed metadata above it.
script = result.output.split("> image_ref:")[-1]
assert "docker run" in script

def test_dry_run_slurm_profile(self, cli_runner, mock_config, slurm_profile):
"""Test dry run with SlurmProfile renders job script."""
Expand Down Expand Up @@ -675,27 +663,23 @@ def test_dry_run_slurm_profile(self, cli_runner, mock_config, slurm_profile):
patch(
"blackfish.cli.services.speech_recognition.get_model_dir"
) as mock_get_model_dir,
patch(
"blackfish.cli.services.speech_recognition.SpeechRecognition"
) as mock_service_class,
):
mock_deserialize.return_value = slurm_profile
mock_get_models.return_value = ["openai/whisper-tiny"]
mock_get_revisions.return_value = ["abc123"]
mock_get_latest.return_value = "abc123"
mock_get_model_dir.return_value = "/path/to/models/whisper-tiny"

mock_service = MagicMock()
mock_service.scheduler = "slurm"
mock_service.render_job_script.return_value = "#!/bin/bash\n#SBATCH"
mock_service_class.return_value = mock_service

result = cli_runner.invoke(main, cmd)

assert result.exit_code == 0, result.exception
assert "Rendering job script" in result.output
assert "model: openai/whisper-tiny" in result.output
assert "profile: cluster" in result.output
assert "host: hpc.example.com" in result.output
# The script itself, not just the echoed metadata above it.
script = result.output.split("> image_ref:")[-1]
assert "#SBATCH" in script

def test_success_local_profile(self, cli_runner, mock_config, local_profile):
"""Test successful API call with LocalProfile."""
Expand Down
Loading