diff --git a/oss_crs/src/templates/renderer.py b/oss_crs/src/templates/renderer.py index 29bd0dfc..c0c63abd 100644 --- a/oss_crs/src/templates/renderer.py +++ b/oss_crs/src/templates/renderer.py @@ -410,6 +410,7 @@ def render_run_crs_compose_docker_compose( ), "litellm_image": OSS_CRS_LITELLM_TAG, "litellm_internal_url": LITELLM_INTERNAL_URL, + "offline": crs_compose.offline, "litellm_spend_report_path": litellm_spend_report_path, "postgres_image": OSS_CRS_POSTGRES_TAG, "postgres_user": POSTGRES_USER, diff --git a/oss_crs/src/templates/run-crs-compose.docker-compose.yaml.j2 b/oss_crs/src/templates/run-crs-compose.docker-compose.yaml.j2 index cc481da2..210148b7 100644 --- a/oss_crs/src/templates/run-crs-compose.docker-compose.yaml.j2 +++ b/oss_crs/src/templates/run-crs-compose.docker-compose.yaml.j2 @@ -83,6 +83,11 @@ services: image: {{ litellm_image }} attach: false restart: on-failure:3 +{%- if offline %} + environment: + # offline the model cost map cannot be fetched; skip the attempt + - LITELLM_LOCAL_MODEL_COST_MAP=True +{%- endif %} depends_on: oss-crs-postgres: condition: service_healthy @@ -114,7 +119,7 @@ services: interval: 5s timeout: 5s retries: 10 - start_period: 15s + start_period: 90s # ~140 Prisma migrations run on first boot oss-crs-postgres: image: {{ postgres_image }} attach: false diff --git a/oss_crs/tests/unit/test_renderer_run_compose.py b/oss_crs/tests/unit/test_renderer_run_compose.py index faaa5c00..4af33ece 100644 --- a/oss_crs/tests/unit/test_renderer_run_compose.py +++ b/oss_crs/tests/unit/test_renderer_run_compose.py @@ -8,11 +8,13 @@ from oss_crs.src.templates.renderer import render_run_crs_compose_docker_compose -def _patch_renderer(monkeypatch, build_env_fn=None): +def _patch_renderer(monkeypatch, build_env_fn=None, llm_context_fn=None): """Stub out the two external collaborators the renderer reaches for. Pass a custom ``build_env_fn`` to observe the kwargs the renderer passes to ``build_run_service_env``; otherwise a minimal fixed response is used. + Pass ``llm_context_fn`` to render an LLM-backed compose; otherwise no LLM + context is produced and the litellm services are omitted. """ monkeypatch.setattr( "oss_crs.src.templates.renderer.build_run_service_env", @@ -25,7 +27,7 @@ def _patch_renderer(monkeypatch, build_env_fn=None): ) monkeypatch.setattr( "oss_crs.src.templates.renderer.prepare_llm_context", - lambda *_args, **_kwargs: None, + llm_context_fn or (lambda *_args, **_kwargs: None), ) @@ -47,6 +49,7 @@ def _make_crs_compose(tmp_path: Path, crs_list: list) -> SimpleNamespace: ), crs_compose_env=SimpleNamespace(get_env=lambda: {"type": "local"}), llm=SimpleNamespace(exists=lambda: False, mode="external"), + offline=False, config=SimpleNamespace( oss_crs_infra=SimpleNamespace(cpuset="0-1", memory="16G") ), @@ -795,3 +798,61 @@ def test_target_independent_module_renders_unhashed_image_no_build( service = yaml.safe_load(rendered)["services"]["crs-shelley_lsp"] assert service["image"] == "oss-crs-runner:crs-shelley-lsp" assert "build" not in service + + +# --------------------------------------------------------------------------- +# Internal LiteLLM proxy: offline gate on the model cost map +# --------------------------------------------------------------------------- + + +def _internal_llm_context(tmp_path: Path): + """A ``llm_context_fn`` yielding an internal-mode LiteLLM stack.""" + return lambda *_args, **_kwargs: { + "mode": "internal", + "litellm_env_secret_files": {"OPENAI_API_KEY": str(tmp_path / "sec")}, + "litellm_config_path": str(tmp_path / "litellm-config.yaml"), + "key_gen_request_path": str(tmp_path / "key_gen_request.yaml"), + "secret_files": {}, + "api_keys": {}, + } + + +@pytest.mark.parametrize( + "offline,expected_env", + [ + pytest.param(True, ["LITELLM_LOCAL_MODEL_COST_MAP=True"], id="offline"), + pytest.param(False, None, id="online"), + ], +) +def test_offline_gates_litellm_local_cost_map_env( + monkeypatch, tmp_path: Path, offline: bool, expected_env: list | None +) -> None: + """``--offline`` pins litellm to the cost map bundled in its image. + + Offline the fetch of model_prices_and_context_window.json cannot succeed and + litellm falls back to that bundled copy regardless, so the attempt is + skipped. Online the fetch is left alone and live pricing is preserved. + """ + _patch_renderer(monkeypatch, llm_context_fn=_internal_llm_context(tmp_path)) + + crs_compose = _make_crs_compose(tmp_path, [_make_crs(tmp_path, "crs-libfuzzer")]) + crs_compose.offline = offline + target = _make_target(tmp_path, has_repo=False) + + rendered, _ = _render(crs_compose, target, tmp_path) + + litellm_service = yaml.safe_load(rendered)["services"]["oss-crs-litellm"] + assert litellm_service.get("environment") == expected_env, ( + f"offline={offline} must render environment {expected_env}; " + f"got: {litellm_service.get('environment')}" + ) + + # Ensure we set a start period; also needed for online - Prisma migrations are slow + assert "start_period" in litellm_service["healthcheck"] + + # The proxy is started from secret-derived exports in `command`; adding an + # `environment:` key must not displace them. + command = " ".join(litellm_service["command"]) + assert "$(cat /run/secrets/litellm_env_OPENAI_API_KEY)" in command, ( + f"secret-derived exports missing from command; got: {command}" + )