Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,226 changes: 2,226 additions & 0 deletions packages/tangle-cli/src/tangle_cli/pipeline_compiler.py

Large diffs are not rendered by default.

48 changes: 47 additions & 1 deletion packages/tangle-cli/src/tangle_cli/pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,15 @@
from collections import deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Mapping
from typing import TYPE_CHECKING, Any, Iterable, Mapping

import yaml

from .utils import dump_yaml

if TYPE_CHECKING:
from .pipeline_compiler import CompileResult

PIPELINE_GRAPH_PATH = "implementation.graph"
TASKS_PATH = f"{PIPELINE_GRAPH_PATH}.tasks"
POSITION_ANNOTATION = "editor.position"
Expand Down Expand Up @@ -419,6 +422,49 @@ def hydrate_pipeline_file(
)


# ---------------------------------------------------------------------------
# Compile
# ---------------------------------------------------------------------------


def compile_pipeline_file(
script: str | Path,
output: str | Path,
*,
overrides: Mapping[str, str] | None = None,
pipeline_name: str | None = None,
emit_components_sidecar: bool = True,
logger: Any | None = None,
) -> CompileResult:
"""Compile a Python-authored pipeline to a dehydrated YAML bundle.

Instantiates the ported :class:`~tangle_cli.pipeline_compiler.PipelineCompiler`
handler and delegates to its ``compile_file`` method, translating the
compiler's domain errors into :class:`PipelineValidationError` for a uniform
CLI failure contract. Mirrors :func:`hydrate_pipeline_file`.

The :class:`~tangle_cli.pipeline_compiler.CompileResult` is returned as-is —
unlike hydrate, the compiler already exposes its public result type, so there
is nothing to repackage.
"""

from .pipeline_compiler import PipelineCompiler
from .python_pipeline.errors import CompileError
from .schema_validation import SchemaValidationError

compiler = PipelineCompiler(logger=logger)
try:
return compiler.compile_file(
Path(script),
Path(output),
overrides=dict(overrides) if overrides else None,
pipeline_name=pipeline_name,
emit_components_sidecar=emit_components_sidecar,
)
except (CompileError, SchemaValidationError) as exc:
raise PipelineValidationError(str(exc)) from exc


# ---------------------------------------------------------------------------
# Layout
# ---------------------------------------------------------------------------
Expand Down
73 changes: 73 additions & 0 deletions packages/tangle-cli/src/tangle_cli/pipelines_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from .logger import logger_for_log_type
from .pipelines import (
PipelineValidationError,
compile_pipeline_file,
generate_mermaid,
hydrate_pipeline_file,
layout_pipeline_file,
Expand Down Expand Up @@ -125,6 +126,18 @@ def _parse_vars(values: list[str] | dict[str, object] | None) -> dict[str, str]:
return parsed


def _parse_overrides(values: list[str] | None) -> dict[str, str]:
parsed: dict[str, str] = {}
for value in values or []:
if "=" not in value:
raise SystemExit("--override entries must use KEY=VALUE syntax")
key, parsed_value = value.split("=", 1)
if not key:
raise SystemExit("--override entries must use KEY=VALUE syntax")
parsed[key] = parsed_value
return parsed


@app.command(name="hydrate")
def pipelines_hydrate(
pipeline_path: pathlib.Path,
Expand Down Expand Up @@ -223,6 +236,66 @@ def pipelines_hydrate(
)


@app.command(name="compile")
def pipelines_compile(
pipeline_path: pathlib.Path,
*,
output: Annotated[
pathlib.Path,
Parameter(
name="--output",
alias="-o",
help="Output path for the compiled dehydrated pipeline YAML.",
),
],
pipeline: Annotated[
str | None,
Parameter(
name="--pipeline",
help=(
"Select the root @pipeline function by name when the file "
"defines several."
),
),
] = None,
override: Annotated[
list[str] | None,
Parameter(
name="--override",
help="Compile-time config override as KEY=VALUE. Repeat for multiple.",
negative_iterable=(),
),
] = None,
log_type: LogTypeOption = "console",
) -> None:
"""Compile a Python-authored pipeline to a dehydrated YAML bundle."""

logger, finalize_logs = logger_for_log_type(log_type)
try:
result = compile_pipeline_file(
pipeline_path,
output,
overrides=_parse_overrides(override),
pipeline_name=pipeline,
logger=logger,
)
except PipelineValidationError as exc:
raise SystemExit(str(exc)) from exc
finally:
finalize_logs()

print(
f"Compiled {pipeline_path} -> {result.pipeline_path} "
f"({result.task_count} task(s))."
)
if result.components_path is not None:
print(f"Wrote component sidecar: {result.components_path}")
for subgraph_path in result.subgraph_paths:
print(f"Wrote subgraph: {subgraph_path}")
for warning in result.warnings:
print(f"warning: {warning}")


@app.command(name="layout")
def pipelines_layout(
pipeline_path: pathlib.Path,
Expand Down
Loading
Loading