From 3e9b08714a9d4ee31534c7b310caa2c77c0e405c Mon Sep 17 00:00:00 2001 From: Silin Gupta Date: Tue, 7 Jul 2026 13:50:49 -0400 Subject: [PATCH 1/3] feat(tangle-cli): add python_pipeline authoring DSL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendor the Python-first pipeline authoring surface into OSS as `tangle_cli.python_pipeline`, the first phase of moving `compile from-python` out of the internal `tangle-deploy` package. This is a clean additive port: the 17 DSL modules (pipeline/task/subpipeline/ registered/ref/raw + trace/graph/emit IR + cfg/types/errors/ids/ placeholders/task_env/compiler_context) carry no runtime Shopify seam — the only zone-root marker resolution lives in the compiler, not the DSL, and is deferred to a later phase. The package is fully OSS-native: all `tangle_deploy` docstring/reference mentions are genericized, and its public surface is `pipeline, task, registered, ref, raw, subpipeline, TaskEnv, In, Out, Outputs`. External deps are stdlib + PyYAML (already required). Generalize the component_from_func authoring-import strip to recognize BOTH authoring surfaces. `_AUTHORING_IMPORT_MODULE` (a single string) becomes `_AUTHORING_IMPORT_MODULES`, a tuple of `tangle_deploy.python_pipeline` (legacy, re-exports the OSS objects) and `tangle_cli.python_pipeline` (canonical), matched via a new `_is_authoring_module` helper that also covers submodules. Non-authoring `tangle_cli.*` / `tangle_deploy.*` runtime helpers and relative imports are still preserved. The authoring shim (`_ensure_tangle_deploy_ authoring_shim`) is intentionally NOT generalized: faking `tangle_cli.python_pipeline` in sys.modules would shadow the real DSL process-wide and break tracing — the real module is importable in-package so no shim is needed for that path. Tests: new tests/test_python_pipeline_dsl.py locks the public surface, the no-`tangle_deploy`-references guard, snake_to_title_case, the CompileError hierarchy, the In/Out/Outputs markers and the raw() contract; tests/test_component_from_func.py gains parallel `tangle_cli.python_pipeline` strip cases plus unit tests for the `_is_authoring_module` / `_is_authoring_import` predicates. Full suite: 671 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/tangle_cli/component_from_func.py | 53 ++- .../tangle_cli/python_pipeline/__init__.py | 43 ++ .../src/tangle_cli/python_pipeline/cfg.py | 169 ++++++++ .../python_pipeline/compiler_context.py | 277 +++++++++++++ .../src/tangle_cli/python_pipeline/emit.py | 289 ++++++++++++++ .../src/tangle_cli/python_pipeline/errors.py | 29 ++ .../src/tangle_cli/python_pipeline/graph.py | 113 ++++++ .../src/tangle_cli/python_pipeline/ids.py | 27 ++ .../tangle_cli/python_pipeline/pipeline.py | 183 +++++++++ .../python_pipeline/placeholders.py | 162 ++++++++ .../src/tangle_cli/python_pipeline/raw.py | 122 ++++++ .../src/tangle_cli/python_pipeline/ref.py | 324 +++++++++++++++ .../tangle_cli/python_pipeline/registered.py | 201 ++++++++++ .../tangle_cli/python_pipeline/subpipeline.py | 211 ++++++++++ .../src/tangle_cli/python_pipeline/task.py | 218 +++++++++++ .../tangle_cli/python_pipeline/task_env.py | 88 +++++ .../src/tangle_cli/python_pipeline/trace.py | 369 ++++++++++++++++++ .../src/tangle_cli/python_pipeline/types.py | 69 ++++ tests/test_component_from_func.py | 175 +++++++++ tests/test_python_pipeline_dsl.py | 210 ++++++++++ 20 files changed, 3311 insertions(+), 21 deletions(-) create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/__init__.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/cfg.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/ids.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/pipeline.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/placeholders.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/raw.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/task.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py create mode 100644 packages/tangle-cli/src/tangle_cli/python_pipeline/types.py create mode 100644 tests/test_python_pipeline_dsl.py diff --git a/packages/tangle-cli/src/tangle_cli/component_from_func.py b/packages/tangle-cli/src/tangle_cli/component_from_func.py index b24b52d..ca9cf02 100644 --- a/packages/tangle-cli/src/tangle_cli/component_from_func.py +++ b/packages/tangle-cli/src/tangle_cli/component_from_func.py @@ -757,13 +757,22 @@ def _is_main_str(n: ast.expr) -> bool: # too, exactly like @task. _AUTHORING_DECORATOR_NAMES = frozenset({"task", "pipeline", "subpipeline", "registered"}) -# The python-pipeline authoring module. ONLY imports of this module (and its -# submodules) are authoring-only and stripped from the baked source. We -# deliberately do NOT strip other ``tangle_deploy.*`` packages (e.g. -# ``tangle_deploy.utils``): those may be legitimate runtime helpers used inside a -# ``@task`` body, and dropping them would raise ``NameError`` in the operation -# container. -_AUTHORING_IMPORT_MODULE = "tangle_deploy.python_pipeline" +# The python-pipeline authoring modules. ONLY imports of these modules (and +# their submodules) are authoring-only and stripped from the baked source. We +# deliberately do NOT strip other ``tangle_deploy.*`` / ``tangle_cli.*`` packages +# (e.g. ``tangle_deploy.utils``): those may be legitimate runtime helpers used +# inside a ``@task`` body, and dropping them would raise ``NameError`` in the +# operation container. +# +# Two authoring surfaces are recognised: the legacy ``tangle_deploy.python_pipeline`` +# path (used by every pipeline authored before the OSS move) and the canonical +# OSS ``tangle_cli.python_pipeline`` path. Both expose the identical decorators — +# ``tangle_deploy.python_pipeline`` re-exports the OSS objects — so an author may +# use either import and codegen must strip either the same way. +_AUTHORING_IMPORT_MODULES = ( + "tangle_deploy.python_pipeline", + "tangle_cli.python_pipeline", +) # The authoring-only ``TaskEnv`` class name. A module-level ``X = TaskEnv(...)`` # (or ``X = .TaskEnv(...)``) declaration is authoring-only by contract and @@ -812,19 +821,25 @@ def _decorator_called_name(node: ast.expr) -> str | None: return None +def _is_authoring_module(name: str) -> bool: + """Return True if *name* is an authoring module or a submodule of one.""" + return any(name == mod or name.startswith(mod + ".") for mod in _AUTHORING_IMPORT_MODULES) + + def _is_authoring_import(node: ast.stmt) -> bool: """Return True if *node* imports the python-pipeline authoring surface. - Matches ONLY the ``tangle_deploy.python_pipeline`` module (and its - submodules): + Matches ONLY the ``tangle_deploy.python_pipeline`` / ``tangle_cli.python_pipeline`` + modules (and their submodules): - - ``from tangle_deploy.python_pipeline import ...`` (including the aliased - ``from tangle_deploy.python_pipeline import ref as operation_by_ref`` form - and submodules like ``from tangle_deploy.python_pipeline.x import y``); - - ``import tangle_deploy.python_pipeline`` / ``import - tangle_deploy.python_pipeline as tp``. + - ``from tangle_cli.python_pipeline import ...`` (including the aliased + ``from tangle_cli.python_pipeline import ref as operation_by_ref`` form + and submodules like ``from tangle_cli.python_pipeline.x import y``); + - ``import tangle_cli.python_pipeline`` / ``import + tangle_cli.python_pipeline as tp``; + - the legacy ``tangle_deploy.python_pipeline`` equivalents. - It does NOT match other ``tangle_deploy.*`` packages (e.g. + It does NOT match other ``tangle_deploy.*`` / ``tangle_cli.*`` packages (e.g. ``from tangle_deploy.utils import X``) — those can be genuine runtime helpers referenced inside a ``@task`` body and must survive into the baked program. Relative imports (``from . import x``) are never authoring imports. @@ -832,13 +847,9 @@ def _is_authoring_import(node: ast.stmt) -> bool: if isinstance(node, ast.ImportFrom): if node.level: # relative import — not the authoring package return False - module = node.module or "" - return module == _AUTHORING_IMPORT_MODULE or module.startswith(_AUTHORING_IMPORT_MODULE + ".") + return _is_authoring_module(node.module or "") if isinstance(node, ast.Import): - return any( - alias.name == _AUTHORING_IMPORT_MODULE or alias.name.startswith(_AUTHORING_IMPORT_MODULE + ".") - for alias in node.names - ) + return any(_is_authoring_module(alias.name) for alias in node.names) return False diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/__init__.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/__init__.py new file mode 100644 index 0000000..a4fbdca --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/__init__.py @@ -0,0 +1,43 @@ +"""Python-first authoring surface for Tangle pipelines. + +End users write:: + + from tangle_cli.python_pipeline import pipeline, task, registered, ref, raw, subpipeline, TaskEnv, In, Out + +``cfg`` is NOT a top-level export — it is a parameter the framework +injects into the user's pipeline function at trace time. Importing the +:class:`tangle_cli.python_pipeline.cfg.Cfg` class is reserved for the +compile driver. + +``import tangle_cli.python_pipeline`` is kept light: it does not +eagerly import the heavy ``tangle_cli.component_generator`` codegen +module or the tracer machinery. + +Module map: authoring entry points live in :mod:`.pipeline`, +:mod:`.task`, :mod:`.subpipeline`, :mod:`.registered`, :mod:`.ref` and +:mod:`.raw`; the trace-time IR is built in :mod:`.trace` / :mod:`.graph` +and lowered to the dehydrated dict shape by :mod:`.emit`. +""" +from __future__ import annotations + +from .pipeline import pipeline +from .raw import raw +from .ref import ref +from .registered import registered +from .subpipeline import subpipeline +from .task import task +from .task_env import TaskEnv +from .types import In, Out, Outputs + +__all__ = [ + "pipeline", + "task", + "registered", + "ref", + "raw", + "subpipeline", + "TaskEnv", + "In", + "Out", + "Outputs", +] diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/cfg.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/cfg.py new file mode 100644 index 0000000..63373fd --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/cfg.py @@ -0,0 +1,169 @@ +"""Config loader: YAML → attribute-access object. + +``Cfg`` is a pure, operation-agnostic attribute-access wrapper. It holds +NO operation-specific knowledge (no SQL, BigQuery table FQN building, or +other domain helpers): a pipeline that needs a fully-qualified BigQuery +table name (or any other domain-specific value) builds it in its own +pipeline-local helper code and passes the final string in as a constant. + +Contract: + +- ``cfg.`` returns the value (flat keys win over nested). +- ``cfg..`` returns the nested value if ``a`` maps to a dict. +- Unknown keys raise :class:`UnknownCfgKeyError`. +- ``--override key=value`` pairs overlay file values (CLI wins). +- ``--override`` *values* are typed via ``yaml.safe_load`` (parity with + file values, which already pass through ``yaml.safe_load``): so + ``--override batch_size=100`` yields ``int 100`` (not ``"100"``) and + ``--override dry_run=false`` yields ``bool False`` (not the truthy + string ``"false"``). The one special case is an EMPTY value: ``key=`` + maps to ``""`` (an empty string), NOT ``None`` — a bare ``--override + key=`` reads as "set to empty string". Authors who want an explicit + null write ``--override key=null``. Because the same loader the config + file uses is reused, the YAML 1.1 quirks are inherited and CONSISTENT + with file values (e.g. ``yes/no/on/off`` -> bool, ``3:14`` -> 194, + ``007`` -> 7, ``0x1F`` -> 31, ``1_000`` -> 1000, ``1.0`` -> float; but + ``1e3`` stays a string). Quote to force a string: ``--override + version='"1.0"'``. +- ``template_file:`` in the source is rejected — the Python authoring + layer IS the template authoring layer. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +from .errors import CompileError, UnknownCfgKeyError + + +def _coerce_override(value: str) -> Any: + """Type a single ``--override key=value`` *value* string. + + Override values arrive from the CLI as raw strings (see + ``pipeline_compiler._parse_overrides``). To give an override the SAME + Python type the equivalent ``key: value`` line in ``config.yaml`` would + (which is already typed via ``yaml.safe_load``), we run each value + through the same loader. + + The one deviation from raw YAML is the EMPTY string: ``yaml.safe_load("")`` + is ``None``, but a bare ``--override key=`` reads as "set to empty + string", so we keep ``""`` as ``""``. (An explicit null is available via + ``--override key=null``.) + """ + if value == "": + return "" + return yaml.safe_load(value) + + +class _CfgNested: + """A nested mapping inside ``Cfg`` — same attribute-access semantics.""" + + def __init__(self, data: dict[str, Any]) -> None: + object.__setattr__(self, "_data", data) + + def __getattr__(self, key: str) -> Any: + data: dict[str, Any] = object.__getattribute__(self, "_data") + if key not in data: + raise UnknownCfgKeyError(f"unknown config key: {key}") + value = data[key] + if isinstance(value, dict): + return _CfgNested(value) + return value + + +class Cfg: + """Attribute-access wrapper around a flat-ish config dict. + + Read-only at runtime; immutability is not enforced (Python attribute + semantics) but writes are not part of the public API. + """ + + def __init__(self, data: dict[str, Any]) -> None: + # Use object.__setattr__ to avoid recursing through __getattr__. + object.__setattr__(self, "_data", dict(data)) + + # ------------------------------------------------------------------ + # Public attribute-access surface + + def __getattr__(self, key: str) -> Any: + data: dict[str, Any] = object.__getattribute__(self, "_data") + if key in data: + value = data[key] + if isinstance(value, dict): + return _CfgNested(value) + return value + raise UnknownCfgKeyError(f"unknown config key: {key!r}. Available keys: " f"{sorted(data.keys())!r}") + + +def load_cfg(path: Path, overrides: Mapping[str, Any] | None = None, *, coerce: bool = True) -> Cfg: + """Load YAML at ``path`` and overlay ``overrides`` on top. + + Two overlay paths, selected by ``coerce``: + + * ``coerce=True`` (the default, the CLI ``--override`` path): override + *values* are raw strings typed via :func:`_coerce_override` (i.e. + ``yaml.safe_load``), so an override yields the SAME Python type the + equivalent line in ``config.yaml`` would: ``batch_size=100`` -> + ``int 100``, ``dry_run=false`` -> ``bool False``, ``ratio=0.1`` -> + ``float 0.1``, ``name=foo`` -> ``"foo"``. The empty value ``key=`` is + special-cased to ``""`` (not ``None``); ``key=null`` -> ``None``. The + same YAML 1.1 quirks the file loader has are inherited (and thus + consistent); quote to force a string (``--override version='"1.0"'``). + * ``coerce=False`` (the NATIVE pass-through path, used for programmatic + ``.override_config`` / broadcast values): override values are + already-typed Python natives and are overlaid AS-IS, NOT re-coerced + through ``yaml.safe_load``. Re-coercion is correct only for raw CLI + strings — it would, e.g., turn the string ``"no"`` into ``False`` or + fail on a non-string value. + + See the module docstring for the full coercion contract. + + Args: + path: Path to the user's config.yaml. + overrides: Optional mapping of override pairs. Under ``coerce=True`` + these are raw ``--override key=value`` strings; under + ``coerce=False`` they are already-typed native values. CLI / + override wins on conflict. + coerce: When ``True`` (default), values are coerced to their YAML + type before overlaying. When ``False``, values pass through + unchanged (native overlay). + + Returns: + A :class:`Cfg` ready for attribute access. + + Raises: + CompileError: if the config carries a ``template_file:`` key + (the authoring layer emits the template itself; the source + config must not point at a separate Jinja template). + FileNotFoundError: if ``path`` doesn't exist. + """ + raw = yaml.safe_load(Path(path).read_text()) or {} + if not isinstance(raw, dict): + raise CompileError(f"config at {path} must be a YAML mapping, got {type(raw).__name__}") + if "template_file" in raw: + raise CompileError( + f"config at {path} has a top-level `template_file:` key, but " + "the Python authoring layer emits the pipeline directly. Remove " + "this key from your config.yaml — the framework is your " + "templating layer." + ) + merged = dict(raw) + if overrides: + if coerce: + # Coerce each raw CLI override string to its YAML type so an + # override behaves identically to the same key written in the + # config file (which is already typed via yaml.safe_load above). + # Coercion is internal to load_cfg only: the override dict stays + # dict[str, str] everywhere upstream and the compile-cache + # fingerprint keeps hashing the raw CLI strings. + merged.update({k: _coerce_override(v) for k, v in overrides.items()}) + else: + # Native pass-through: values are already-typed Python natives + # (from .override_config / broadcast). Overlay verbatim — no + # yaml.safe_load re-coercion (it would mangle e.g. "no" -> False). + merged.update(dict(overrides)) + return Cfg(merged) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py new file mode 100644 index 0000000..6dd96cb --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py @@ -0,0 +1,277 @@ +"""In-memory compile context for recursive (nested) pipeline compilation. + +The compiler must build and validate ALL artifacts (the root pipeline plus +every child subgraph +sidecar) IN MEMORY before writing ANY file, so a late validation, cycle, +or asset-policy failure never leaves a partial bundle on disk. + +Types defined here: + +* :class:`PipelineCompileKey` — a frozen, hashable identity for a + ``(source, function, pipeline_name, config, overrides)`` tuple, + canonicalised per Decision B. Two ``subpipeline(child)`` call sites that + reach the SAME child collapse to one sidecar because they share a key + (Decision M dedup); two different children that slug to the same display + name stay distinct because their ``hash8`` differs. +* :class:`SubgraphArtifact` — one planned, in-memory artifact (root or + child): its dehydrated ``body`` dict, the ``output_path`` it will be + written to, an optional ``@task`` ``local_from_python`` components + sidecar (entries + path), and the child sub-artifacts it references. +* :class:`CompileContext` — the recursion state shared across a single + ``compile_pipeline`` call: a ``registry`` for dedup, an ``active_stack`` + for cycle detection (Decision L), a ``max_depth`` guard, and the + ``planned_files`` set consulted by asset-policy validation (Decision J). + +This module is intentionally free of compile orchestration logic — it only +holds the shared shapes and the canonicalisation helpers. The recursive +compile driver lives in the CLI's pipeline compiler. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .errors import CompileError + + +def find_git_root(start: Path) -> Path | None: + """Return the nearest ancestor directory containing ``.git``, or ``None``. + + Walks ``start`` (or its parent, if ``start`` is a file) upward. Used to + canonicalise compile-key paths to repo-relative form so in-repo fixture + sidecar hashes stay stable across machines (Decision B). + """ + cur = start if start.is_dir() else start.parent + for d in (cur, *cur.parents): + if (d / ".git").exists(): + return d + return None + + +def canonical_repo_path(p: Path) -> str: + """Canonical POSIX path for a compile key, per Decision B. + + Resolves symlinks, then returns the repo-relative POSIX path when ``p`` + lives inside a git repo, otherwise the resolved absolute POSIX path. + Sources outside a git root therefore produce machine-specific keys, + which is acceptable for v1 local compiles. + """ + resolved = p.resolve() + git_root = find_git_root(resolved) + if git_root is not None: + try: + return resolved.relative_to(git_root).as_posix() + except ValueError: # pragma: no cover — resolved not under git_root + pass + return resolved.as_posix() + + +def overrides_fingerprint(overrides: Mapping[str, Any] | None, *, context: str | None = None) -> str: + """Stable SHA-256 fingerprint of compile-time overrides. + + Overrides are sorted by key so the fingerprint is order-independent. + Empty for the default isolated compile (Decision F), but the field is + part of the key so the child-override API produces distinct sidecars for + distinct effective overrides (Decision M case 3). Override values must be + JSON-serializable scalars/containers (str/int/float/bool/None/list/dict); + ``json.dumps`` serializes them stably, so a native int and the string of + that int hash differently (a desirable distinction). + + A non-serializable override value (e.g. a ``_CfgNested`` from + ``cfg.``, or any arbitrary object) raises a :class:`CompileError` + naming the offending key — instead of leaking a bare ``TypeError`` past the + CLI's ``CompileError``-only handling. ``context`` is woven into the message + so the user can tell WHICH ``.override_config`` / override produced it. + + Args: + overrides: The effective overrides to fingerprint. + context: Optional label naming the affected pipeline/child edge, + appended to the error message for an actionable diagnostic. + """ + items = dict(sorted((overrides or {}).items())) + try: + blob = json.dumps(items, sort_keys=True, separators=(",", ":")) + except TypeError as e: + # Identify the first offending key/value for an actionable message. + offending = next( + ((k, v) for k, v in items.items() if not _is_json_scalar_or_container(v)), + None, + ) + where = f" {context}" if context else "" + if offending is not None: + key, value = offending + raise CompileError( + f"override value for key {key!r}{where} is not a compile-time " + f"constant: got {type(value).__name__}. .override_config / " + "config overrides accept only scalar values (str, int, float, " + "bool, None) or plain lists/dicts of them — e.g. pass " + f"`cfg.some_key` (a resolved string), not a nested config " + "object or an arbitrary Python object." + ) from e + raise CompileError( + f"a config override value{where} is not JSON-serializable " + f"({e}). .override_config / config overrides accept only scalar " + "values (str, int, float, bool, None) or plain lists/dicts of them." + ) from e + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def _is_json_scalar_or_container(value: Any) -> bool: + """Best-effort check that ``value`` is a JSON-serializable native. + + Used only to pinpoint the offending key in :func:`overrides_fingerprint`'s + error path; the authoritative serializability check is ``json.dumps`` + itself. Recurses into plain lists/tuples and dicts. + """ + if value is None or isinstance(value, (str, int, float, bool)): + return True + if isinstance(value, (list, tuple)): + return all(_is_json_scalar_or_container(v) for v in value) + if isinstance(value, dict): + return all(isinstance(k, str) and _is_json_scalar_or_container(v) for k, v in value.items()) + return False + + +@dataclass(frozen=True) +class PipelineCompileKey: + """Frozen, hashable identity for a compiled pipeline artifact. + + Fields are already canonicalised (see :func:`canonical_repo_path` and + :func:`overrides_fingerprint`) so equality / hashing dedups the same + child reached through different paths (Decision M) and so :meth:`hash8` + is stable for in-repo sources. + """ + + source_path: str + function_qualname: str + pipeline_name: str + config_path: str + overrides_fingerprint: str + + def canonical_fields(self) -> dict[str, str]: + """The sorted-key JSON object hashed to produce :meth:`hash8`. + + Matches Decision B's canonical field set exactly: + ``source_path``, ``function_qualname``, ``pipeline_name``, + ``config_path``, ``overrides``. + """ + return { + "config_path": self.config_path, + "function_qualname": self.function_qualname, + "overrides": self.overrides_fingerprint, + "pipeline_name": self.pipeline_name, + "source_path": self.source_path, + } + + def hash8(self) -> str: + """First 8 hex chars of SHA-256 over :meth:`canonical_fields`.""" + blob = json.dumps(self.canonical_fields(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:8] + + def display(self) -> str: + """Human-readable ``Name (source_path)`` for cycle / error chains.""" + return f"{self.pipeline_name} ({self.source_path})" + + +@dataclass(frozen=True) +class BroadcastLayer: + """One layer of config that flows down a flagged ancestor's subtree. + + Layers are pushed onto ``CompileContext.broadcast_stack`` outermost first + and resolve in TWO precedence tiers (PROPAGATE_CONFIG_DESIGN §4, lines + 128-159), lowest -> highest: + + * **Broadcast tier** (``explicit=False``) — a flagged pipeline's OWN + ``config.yaml`` (overlaid with any overrides it received), pushed by + ``_compile_pipeline_fn``. Same-name, lenient, nearest-wins WITHIN the + tier (inner scope shadows outer). + * **Explicit tier** (``explicit=True``) — a flagged caller's per-edge + ``.override_config`` values, pushed by ``_process_subpipeline_children`` + so they flow deep into the child's subtree. Same-name, lenient, + nearest-wins WITHIN the tier — but the WHOLE explicit tier is applied + AFTER the whole broadcast tier, so an explicit override set by an + ancestor ALWAYS outranks a nearer flagged descendant's own-config + broadcast of the same key, regardless of depth. + + ``config`` is the key/value map this layer contributes. + """ + + config: Mapping[str, Any] + # ``True`` only for per-edge ``.override_config`` layers (the explicit + # tier); the flagged-pipeline own-config broadcast layer leaves this + # ``False``. See the two-tier resolution above. + explicit: bool = False + + +@dataclass +class SubgraphArtifact: + """One planned, in-memory compiled artifact (root pipeline or child). + + Built by ``_compile_pipeline_fn``; validated and written by + ``compile_pipeline`` only after the WHOLE bundle has been built, so a + late failure writes nothing (Decision C / J). + """ + + key: PipelineCompileKey + output_path: Path + body: dict[str, Any] + is_root: bool = False + task_count: int = 0 + # ``@task`` ``local_from_python`` resolver sidecar for THIS artifact. + # ``components_entries`` is the sidecar's content; ``components_path`` + # is where it will be written (next to ``output_path``). Both ``None`` + # when the artifact uses no ``@task`` components. + components_entries: dict[str, Any] | None = None + components_path: Path | None = None + # Child sub-artifacts this artifact references (for structure only — + # the authoritative write/dedup list is ``CompileContext.registry``). + children: list["SubgraphArtifact"] = field(default_factory=list) + # Cached dumped YAML text, filled during validation so the write pass + # does not re-dump (and writes the exact bytes that were validated). + dumped_text: str | None = None + # JSON paths (dot-delimited, in the form ``iter_template_delimiters`` + # yields) whose template delimiters are legitimate RUNTIME placeholders + # — every argument wrapped in ``raw(...)`` in THIS artifact's body. + # The no-template-delimiter output guard skips exactly these paths while + # still failing on any other delimiter. Empty when the artifact uses no + # ``raw(...)`` value (the common case). Per-artifact because each body is + # emitted and validated independently. + exempt_paths: set[str] = field(default_factory=set) + + +@dataclass +class CompileContext: + """Shared recursion state for a single ``compile_pipeline`` call.""" + + root_output_path: Path + subgraph_dir: Path + root_overrides: dict[str, str] = field(default_factory=dict) + emit_components_sidecar: bool = True + max_depth: int = 32 + # Compiled CHILD artifacts keyed by compile key (Decision M dedup). + # The root is NOT stored here; it is returned directly. + registry: dict[PipelineCompileKey, SubgraphArtifact] = field(default_factory=dict) + # Keys on the current recursive compile chain, for cycle detection + # (Decision L). Pushed before recursing into a child, popped after. + active_stack: list[PipelineCompileKey] = field(default_factory=list) + # Active config-broadcast layers from flagged ancestors on the current + # chain. Pushed (outermost first) by a ``propagate_config=True`` pipeline + # before compiling its children, popped after. Nearest (last) layer that + # defines a key wins; lenient same-name overlay onto each descendant. + broadcast_stack: list[BroadcastLayer] = field(default_factory=list) + # Resolved paths the compiler WILL write (root output, child sidecars, + # and any components sidecars). Asset-policy validation treats these as + # already-existing so refs to not-yet-written sidecars do not fail. + planned_files: set[Path] = field(default_factory=set) + # Resolved SOURCE directories the compile imports from (root script dir + + # each child ``PipelineFn``'s own source dir). The driver purges + # newly-imported modules under these dirs from ``sys.modules`` after the + # compile so a subsequent in-process compile of a different bundle does + # not reuse a stale cached sibling module (the P2 sibling-import leak). + source_dirs: set[Path] = field(default_factory=set) + warnings: list[str] = field(default_factory=list) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py new file mode 100644 index 0000000..4bbd3b6 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py @@ -0,0 +1,289 @@ +"""IR → dict in canonical key order (the *dehydrated* pipeline shape). + +Canonical top-level key order: + name, description, metadata, inputs, outputs, implementation + +Per-task key order: + annotations?, componentRef, arguments + +Argument values are emitted in the runnable ``ArgumentValue`` shape, +dispatched purely on the VALUE's runtime type — never on the argument +*key*. The compiler is operation-agnostic: ``wait_for``, ``project``, +``sql_query`` and ``payload`` are all treated identically. Only the +value decides the shape: + +* :class:`TaskOutputProxy` → ``{taskOutput: {taskId, outputName}}`` +* :class:`GraphInputPlaceholder` → ``{graphInput: {inputName}}`` +* a ``str`` constant → the RAW string (no wrapper) +* a :class:`Raw` constant → its inner string verbatim (no wrapper), + exactly like a plain ``str`` constant — but the argument's JSON path is + also recorded in the returned exempt-paths set so the no-template-delimiter + output guard skips that one location (it is a legitimate RUNTIME + placeholder, e.g. a run-query ``{{input_1}}`` sentinel). + +Non-string constants are rejected: the runnable Tangle argument contract +only supports string constants, ``graphInput``, or ``taskOutput``. +Structured/non-string values must be stringified explicitly in pipeline +code (e.g. ``json.dumps(...)``) before they reach the compiler. + +The literal key the user wrote (``wait_for``, ``depends_on``, +``project``, ``payload``, ...) is preserved verbatim as the dict key. +""" +from __future__ import annotations + +from typing import Any + +from .errors import CompileError, InvalidArgumentTypeError +from .graph import EdgeRef, GraphBuilder, TaskNode +from .placeholders import GraphInputPlaceholder, TaskOutputProxy +from .raw import Raw + + +def emit_pipeline(g: GraphBuilder) -> tuple[dict[str, Any], set[str]]: + """Build the body dict for the compiled (dehydrated) pipeline YAML. + + Only keys with content are emitted (mirrors the source template: + ``description`` and ``metadata`` are present iff the user set them; + ``inputs`` and ``outputs`` are present iff non-empty). + + Returns: + A ``(body_dict, exempt_paths)`` tuple. ``exempt_paths`` is the set + of dot-delimited JSON paths (in the exact form the compiler's + template-delimiter scan yields) for every argument whose value was + wrapped in + :func:`tangle_cli.python_pipeline.raw` — i.e. legitimate RUNTIME + template placeholders the no-template-delimiter output guard must + skip. Empty when no ``raw(...)`` value was used (the common case), + so it is inert for callers that ignore it. + + Raises: + CompileError: if the graph has no tasks (the schema requires at + least one) or two tasks share the same id (the tasks dict + would otherwise silently drop the earlier task). + """ + # Collected as tasks are emitted; a Raw argument records its path in + # the exact dot-delimited form ``iter_template_delimiters`` produces so + # the guard can match and skip it. + exempt_paths: set[str] = set() + + out: dict[str, Any] = {"name": g.name} + + if g.description: + out["description"] = g.description + + if g.annotations: + # metadata.annotations preserves user-specified order. + out["metadata"] = {"annotations": dict(g.annotations)} + + if g.inputs: + out["inputs"] = list(g.inputs) + + if g.outputs: + out["outputs"] = list(g.outputs) + + graph: dict[str, Any] = {} + + # outputValues block is emitted BEFORE tasks within graph. Prefer the + # MULTI-output map (Decision D) when present — it preserves Outputs field + # declaration order — and fall back to the single-output compatibility + # shims for the legacy ``-> Out[T]`` path. + if g.output_values: + graph["outputValues"] = { + name: _emit_edge_value(edge) for name, edge in g.output_values.items() + } + elif g.output_name and g.output_taskref is not None: + graph["outputValues"] = { + g.output_name: _emit_edge_value(g.output_taskref), + } + + # Reject an empty graph up front with a friendly message — the + # schema also enforces this (minProperties: 1) but a raw schema error + # is harder to act on. + if not g.tasks: + raise CompileError( + "pipeline has no tasks; a compiled pipeline must declare at " + "least one task. Call at least one ref(...)(...) inside the " + "@pipeline body." + ) + + # Walk tasks in insertion order (= trace-time call order). Building + # the dict directly would silently overwrite an earlier task that + # shares an id, so detect duplicates explicitly. + tasks_dict: dict[str, dict[str, Any]] = {} + for node in g.tasks: + if node.task_id in tasks_dict: + raise CompileError( + f"duplicate task id {node.task_id!r}: two tasks resolved to " + "the same id. Give each task a unique id via " + ".named('Unique Id') or a distinct assignment variable name." + ) + # The task's argument JSON paths are rooted at + # ``implementation.graph.tasks.`` — the SAME prefix + # ``iter_template_delimiters`` walks to — so any Raw argument is + # recorded under a path the output guard can match verbatim. + task_path = f"implementation.graph.tasks.{node.task_id}" + tasks_dict[node.task_id] = _emit_task(node, task_path, exempt_paths) + graph["tasks"] = tasks_dict + + out["implementation"] = {"graph": graph} + return out, exempt_paths + + +# --------------------------------------------------------------------------- +# Internal helpers + + +def _emit_task( + node: TaskNode, task_path: str, exempt_paths: set[str] +) -> dict[str, Any]: + """Build the per-task body dict in canonical key order: + ``annotations?, componentRef, arguments``. + + ``task_path`` is this task's dot-delimited JSON path + (``implementation.graph.tasks.``); each argument's path is + derived from it so a :class:`Raw` value can record an exempt path that + the output guard matches verbatim. ``exempt_paths`` is mutated in place. + """ + body: dict[str, Any] = {} + + if node.annotations: + # Preserve annotation values as authored — Tangle annotations are + # string-valued in practice but the schema allows free-form. + body["annotations"] = {k: v for k, v in node.annotations.items()} + + body["componentRef"] = _emit_component_ref(node) + + if node.arguments: + args_path = f"{task_path}.arguments" + body["arguments"] = { + k: _emit_argument_value(k, v, f"{args_path}.{k}", exempt_paths) + for k, v in node.arguments.items() + } + + return body + + +def _emit_component_ref(node: TaskNode) -> dict[str, Any]: + """Render ``componentRef`` as a PURE ref — ``{url[, digest]}``, + ``{name[, digest]}`` or ``{digest}``. Never emits ``spec`` or ``text``. + + Locator dispatch (first matching branch wins): + + * ``ref_url`` set -> ``{"url": …}``; a ``ref_digest`` pins it + (``{"url": …, "digest": …}``). Subpipeline refs always take this + branch via their ``subpipeline://pending`` sentinel URL. + * ``ref_name`` set -> ``{"name": …}``; a ``ref_digest`` pins it + (``{"name": …, "digest": …}``). + * ``ref_digest`` alone -> ``{"digest": …}``. + + A node with ``ref_url=None`` AND no name/digest is only reachable via + ``@task`` refs (the ``ref()`` factory rejects a no-locator call). Such + a node gets a transient ``local-from-python://pending`` placeholder + URL here; the compile driver REWRITES it to a real + ``resolve://./.components.yaml#`` URL (via + ``_rewrite_task_componentref_urls``) BEFORE schema validation, so the + placeholder never reaches the written output. It is still a pure ref + — no ``spec``/``text``. + """ + if node.ref_url: + cref: dict[str, Any] = {"url": node.ref_url} + if node.ref_digest: + cref["digest"] = node.ref_digest + return cref + if node.ref_name: + cref = {"name": node.ref_name} + if node.ref_digest: + cref["digest"] = node.ref_digest + return cref + if node.ref_digest: + return {"digest": node.ref_digest} + # Transient placeholder for @task refs. The compile driver computes + # the real ``resolve://./.components.yaml#`` URL after + # tracing (it depends on the output path) and rewrites this in place + # before validation. Still a pure ref — no spec/text. + return {"url": _TASK_URL_PLACEHOLDER} + + +# Sentinel URL written by ``_emit_component_ref`` for @task refs whose +# real URL (``resolve://./.components.yaml#``) is only +# known after the driver has computed output paths. The driver always +# rewrites it before writing, so it never appears in compiled output. +_TASK_URL_PLACEHOLDER = "local-from-python://pending" + + +def _emit_argument_value( + key: str, value: Any, arg_path: str, exempt_paths: set[str] +) -> Any: + """Render an argument value in its runnable ``ArgumentValue`` form. + + Dispatch is purely on the VALUE's runtime type — the argument *key* + is never inspected. Produces a ``taskOutput`` / ``graphInput`` wrapper + for edges, or the RAW string for a constant (matching the runnable + Tangle argument contract). Non-string constants are rejected. + + A :class:`Raw` value is emitted as its inner string verbatim — exactly + like a plain ``str`` constant — and ``arg_path`` (this argument's + JSON path) is added to ``exempt_paths`` so the no-template-delimiter + output guard skips this one location: the inner string is a legitimate + RUNTIME placeholder (e.g. a run-query ``{{input_1}}`` sentinel), not a + leaked compile-time template. + """ + if isinstance(value, TaskOutputProxy): + return { + "taskOutput": { + "taskId": value._task_id, + "outputName": value._resolved_output_name(), + } + } + if isinstance(value, GraphInputPlaceholder): + return {"graphInput": {"inputName": value.input_name}} + if isinstance(value, Raw): + # Emit the inner string verbatim (identical to a str constant) and + # exempt this argument's path from the delimiter guard. Rawness is + # lost once the value is a plain str in the dict, so it must be + # recorded here, at the only point the Raw wrapper is still visible. + exempt_paths.add(arg_path) + return value.value + # Everything else is a constant. The runnable schema only accepts raw + # string constants, so validate and emit the string verbatim. + _validate_constant(value, key) + return value + + +def _validate_constant(value: Any, key: str) -> None: + """Assert ``value`` is a runnable string constant. + + Runnable Tangle pipeline arguments only support raw ``str`` constants + (alongside the ``graphInput`` / ``taskOutput`` wrappers). A non-string + constant (``int``, ``float``, ``bool``, ``None``, ``list``, ``dict``, + a tuple/set, a leftover helper object, a callable, ...) cannot be + represented under the runnable schema, so it is rejected with a + GENERIC, operation-agnostic message — the compiler has no + operation-specific knowledge (no SQL, BigQuery, or other domain + awareness). Authors must stringify structured/non-string values + explicitly in pipeline code before passing them as task arguments. + """ + if isinstance(value, str): + return + raise InvalidArgumentTypeError( + f"unsupported constant type {type(value).__name__!r} for " + f"argument {key!r}. Runnable Tangle pipeline arguments only support " + "string constants, graphInput, or taskOutput. Convert structured or " + "non-string values to a string explicitly in your pipeline code " + "(for example json.dumps(...) or str(...)) before passing them as " + "task arguments." + ) + + +def _emit_edge_value(edge: EdgeRef) -> dict[str, Any]: + """Render an :class:`EdgeRef` as a dehydrated ``ArgumentValue`` + sub-dict (``{taskOutput|graphInput: {...}}``) used in + ``outputValues``.""" + if edge.kind == "taskOutput": + return { + "taskOutput": { + "taskId": edge.task_id, + "outputName": edge.output or "wait_for_output", + } + } + return {"graphInput": {"inputName": edge.input_name}} diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py new file mode 100644 index 0000000..189d225 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py @@ -0,0 +1,29 @@ +"""Compile-time error hierarchy. + +All errors raised by the Python pipeline authoring layer before/during +``tangle-deploy pipeline compile from-python`` are subclasses of :class:`CompileError`. +They should include enough context for a user to fix the underlying issue +without reading framework source: file:line of the offending call, the +relevant primitive name, and a suggested remedy. +""" +from __future__ import annotations + + +class CompileError(Exception): + """Base class for all compile-time failures in the authoring layer.""" + + +class UnknownCfgKeyError(CompileError): + """Raised on ``cfg.`` access.""" + + +class MissingRequiredInputError(CompileError): + """Raised when a required In[T] graph input is missing at trace time.""" + + +class AmbiguousTaskIdError(CompileError): + """Raised when LHS-name inference for a task ID cannot be resolved.""" + + +class InvalidArgumentTypeError(CompileError): + """Raised on an argument value with no supported emit dispatch.""" diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py new file mode 100644 index 0000000..ad50fd2 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py @@ -0,0 +1,113 @@ +"""In-memory IR for the traced pipeline graph. + +The dataclasses here are the shared shapes used by the tracer +(``trace.py``) and the emitter (``emit.py``). Keeping them in their own +module avoids circular imports between those two. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + + +@dataclass +class EdgeRef: + """How one task's input wires to a producer. + + Either a taskOutput (depends on another task) or a graphInput (wired + from a pipeline-level input). + """ + + kind: Literal["taskOutput", "graphInput"] + task_id: str | None = None # for taskOutput + output: str | None = None # for taskOutput + input_name: str | None = None # for graphInput + + +@dataclass +class TaskNode: + """A single emitted task in the graph. + + ``arguments`` values may be plain strings, TaskOutputProxy objects + (for taskOutput edges in non-``wait_for`` argument positions — not + used in the PoC), or GraphInputPlaceholder objects. + + ``edge_kw`` is the separated bucket for ``wait_for`` / ``depends_on`` + kwargs — emitted with their special edge value shape rather than as + a normal argument. + """ + + task_id: str + ref_url: str | None = None + ref_name: str | None = None + ref_digest: str | None = None + arguments: dict[str, Any] = field(default_factory=dict) + annotations: dict[str, str] | None = None + edge_kw: dict[str, EdgeRef] = field(default_factory=dict) + + +@dataclass +class GraphBuilder: + """Holds the trace-time state for a single ``@pipeline`` body.""" + + name: str + description: str | None = None + annotations: dict[str, str] = field(default_factory=dict) + inputs: list[dict[str, Any]] = field(default_factory=list) + outputs: list[dict[str, Any]] = field(default_factory=list) + # MULTI-output map (Decision D): ``{output_name: EdgeRef}`` in field + # declaration order, filled when the pipeline returns an ``Outputs`` + # typed object. ``emit_pipeline`` PREFERS this map when non-empty and + # falls back to the single-output ``output_name``/``output_taskref`` + # compatibility shims below for the legacy ``-> Out[T]`` path. + output_values: dict[str, EdgeRef] = field(default_factory=dict) + # Single-output compatibility shims (legacy ``-> Out[T]`` path). + output_name: str | None = None + output_taskref: EdgeRef | None = None + tasks: list[TaskNode] = field(default_factory=list) + # AST pre-pass map filled in by ``trace.trace_pipeline`` so + # ``CallableRef.__call__`` can derive task IDs from LHS variable + # names. Keyed by ABSOLUTE source line number; values are the LHS + # variable name (snake_case). + lineno_to_lhs: dict[int, str] = field(default_factory=dict) + # ``(task_id, CallableRef)`` tuples captured from + # ``CallableRef.__call__`` when the ref was produced by ``@task`` + # (``_task_source_path`` is set). The compile driver uses this + # list to (a) auto-emit a sibling ``.components.yaml`` with one + # ``local_from_python:`` entry per unique source file, and (b) + # rewrite each task's ``componentRef.url`` to + # ``resolve://./.components.yaml#`` so hydrate + # uses tangle-deploy's own ``local_from_python`` resolver. + task_refs_for_local_from_python: list[tuple[str, Any]] = field( + default_factory=list + ) + # ``(task_id, CallableRef)`` tuples captured from + # ``CallableRef.__call__`` when the ref was produced by ``@registered`` + # (``_registered_source_path`` is set). The compile driver uses this + # list to rewrite each task's ``componentRef.url`` from the + # ``registered://pending`` sentinel to a pure + # ``resolve:///gen_config.yaml#`` URL pointing at the + # operation's EXISTING ``gen_config.yaml``. Unlike + # ``task_refs_for_local_from_python``, this does NOT drive sidecar + # generation -- the gen_config.yaml already exists on disk; the driver + # only computes the relative ``resolve://`` URL (per artifact, against + # that artifact's output dir) and stamps it onto the task. + task_refs_for_registered: list[tuple[str, Any]] = field( + default_factory=list + ) + # ``(task_id, SubpipelineRef)`` tuples captured from + # ``SubpipelineRef.__call__`` for every ``subpipeline(child)(...)`` call. + # Recorded separately from ``@task`` refs so the compile driver can, + # in a later phase, (a) compile each child ``PipelineFn`` to a graph + # component sidecar and (b) rewrite the parent task's ``componentRef`` + # from the ``subpipeline://pending`` sentinel to a pure ``file://`` + # URL pointing at that sidecar. This list IS the explicit subpipeline + # discriminator (Decision A) — the compiler never infers the kind + # from ``ref_url is None``. + task_refs_for_subpipelines: list[tuple[str, Any]] = field( + default_factory=list + ) + + def add_task(self, node: TaskNode) -> None: + """Append a task node in trace order.""" + self.tasks.append(node) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/ids.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/ids.py new file mode 100644 index 0000000..b2d4410 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/ids.py @@ -0,0 +1,27 @@ +"""Identifier helpers — snake_case to Title Case With Spaces. + +Used to derive Tangle's task IDs from Python variable names. The PoC +authors write ``build_quality_tables = build_dbt(...)`` and the +framework infers the task ID ``"Build Quality Tables"``. +""" +from __future__ import annotations + + +def snake_to_title_case(name: str) -> str: + """Convert a ``snake_case`` identifier to ``Title Case With Spaces``. + + Examples: + >>> snake_to_title_case("build_quality_tables") + 'Build Quality Tables' + >>> snake_to_title_case("foo") + 'Foo' + + Notes: + - Multiple consecutive underscores collapse: empty segments are + dropped (so ``a__b`` → ``A B``, not ``A B``). + - All-caps tokens are unchanged by ``str.capitalize`` semantics + (which lowercases trailing letters), so ``GPU`` becomes ``Gpu``. + Users with acronyms should call ``.named("...")`` explicitly. + """ + parts = [p for p in name.split("_") if p] + return " ".join(p.capitalize() for p in parts) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/pipeline.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/pipeline.py new file mode 100644 index 0000000..dd7e81a --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/pipeline.py @@ -0,0 +1,183 @@ +"""``@pipeline`` decorator — metadata holder. + +The decorator captures metadata (name, description, config path, +annotations, caller dir) on a :class:`PipelineFn` wrapper. It does NOT +call the user function — the tracer (``trace.trace_pipeline``) invokes it +inside a :class:`GraphBuilder` context. +""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +from . import emit +from .graph import GraphBuilder + + +@dataclass +class PipelineFn: + """Wrapper holding decorator metadata + a reference to the user fn. + + Attributes captured here are consumed by: + - ``trace.trace_pipeline`` (passes ``fn``, ``config_path``, + ``annotations`` through to GraphBuilder construction). + - the compile driver (resolves config_path relative to caller_dir + and threads it through cfg.load_cfg). + """ + + fn: Callable[..., Any] + name: str + description: str | None = None + config_path: str | None = None # path relative to caller_dir + annotations: dict[str, Any] = field(default_factory=dict) + task_annotations: dict[str, Any] = field(default_factory=dict) + caller_dir: Path | None = None + # Convention for the single Out[T] slot's name. Defaults to the PoC + # canonical sentinel ``wait_for_output``; overridable via + # ``@pipeline(output_name=...)``. + output_name: str = "wait_for_output" + # When True, this pipeline broadcasts its OWN resolved config.yaml deep + # into its subtree by matching key name (lenient same-name overlay). + # Off by default (Decision F isolation is preserved). Metadata only — no + # config is read at decoration time; the compile driver acts on it. + propagate_config: bool = False + + # ------------------------------------------------------------------ + # Calling the decorated PipelineFn directly is reserved for the + # tracer; users normally invoke via the CLI compile driver. The + # underlying function is still accessible as ``pipeline_fn.fn`` for + # tests that want to call it raw. + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + # Guard against the inline-execution footgun (Decision H): calling + # a @pipeline directly inside an active parent trace would record + # the child's internal tasks into the PARENT's GraphBuilder, + # destroying subgraph encapsulation. Block it with guidance toward + # ``subpipeline(child)(...)``. The compiler's own root trace is + # unaffected because ``trace_pipeline`` invokes ``self.fn(...)`` + # directly, never ``self(...)``. Imports are local to avoid an + # import cycle with trace.py at module load. + from .trace import current_builder + + if current_builder() is not None: + from .errors import CompileError + + raise CompileError( + f"Cannot call @pipeline {self.name!r} directly inside another " + "@pipeline. Use subpipeline(child_pipeline)(...) so the child " + "is compiled as a subgraph." + ) + return self.fn(*args, **kwargs) + + # ------------------------------------------------------------------ + # Compile path: trace + emit. Returns the body dict ready for + # ``dump_yaml``. The CLI driver calls this after loading cfg and + # merging ``--override`` pairs. + + def compile_to_dict( + self, + cfg: Any | None = None, + inputs: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Trace the pipeline and emit the body dict. + + Args: + cfg: The :class:`Cfg` object to bind to the pipeline's + ``cfg`` parameter. If ``None``, an empty Cfg is used + (useful for tests where no config is needed). + inputs: Optional runtime values for ``In[T]`` parameters. + Missing entries fall back to ``GraphInputPlaceholder`` + so emit can render them as ``graphInput:`` references. + """ + # Local imports to avoid a circular dep with trace.py at module + # load time (trace.py imports back into pipeline for typing). + from . import emit, trace + from .cfg import Cfg + + if cfg is None: + cfg = Cfg({}) + builder = trace.trace_pipeline(self, cfg=cfg, inputs=inputs or {}) + # emit_pipeline returns (body, exempt_paths); the raw(...) exempt + # paths are a compile-driver concern (the driver threads them into + # the output guard). This trace+emit helper exposes only the body. + body, _exempt_paths = emit.emit_pipeline(builder) + return body + + # ------------------------------------------------------------------ + # Smoke-test path: emit a header-only dict without tracing. Used by + # tests to verify canonical key order without needing the tracer. + + def compile_empty(self) -> dict[str, Any]: + """Return the body dict assuming a 0-task graph.""" + builder = GraphBuilder( + name=self.name, + description=self.description, + annotations=dict(self.annotations), + ) + # emit_pipeline returns (body, exempt_paths); a 0-task graph has no + # raw(...) arguments, so only the body is relevant here. + body, _exempt_paths = emit.emit_pipeline(builder) + return body + + +def pipeline( + name: str, + *, + description: str | None = None, + config: str | None = None, + annotations: dict[str, Any] | None = None, + task_annotations: dict[str, Any] | None = None, + output_name: str = "wait_for_output", + propagate_config: bool = False, +) -> Callable[[Callable[..., Any]], PipelineFn]: + """Mark a function as a Tangle pipeline definition. + + The decorator captures metadata (name, description, config path, + annotations) on a :class:`PipelineFn` wrapper. It does NOT call the + user function — the trace driver invokes it inside a + :class:`GraphBuilder` context. + + Args: + name: The pipeline's ``name:`` value in the emitted YAML. + description: Optional ``description:`` block (multi-line OK — + block-literal style is applied by ``dump_yaml``). + config: Path to ``config.yaml``, relative to the file holding + the decorated function. Loaded by the CLI driver at compile + time so ``--override key=value`` pairs can merge in. + annotations: ``metadata.annotations`` block (e.g. ``version``, + ``author``). + task_annotations: Per-task default annotations applied to every + task in the pipeline. Accepted for API completeness but not + wired through in MVP — the PoC sets per-task annotations + explicitly via ``.with_annotations``. + output_name: Name for the single ``Out[T]`` output slot. + Defaults to ``wait_for_output`` (matches the PoC). + propagate_config: Broadcasts this pipeline's own config.yaml deep + into its subtree by matching key name; off by default. + """ + + def decorator(fn: Callable[..., Any]) -> PipelineFn: + # Resolve caller_dir from the decorated function's source file. + try: + source_file = Path(inspect.getfile(fn)).resolve() + caller_dir = source_file.parent + except (TypeError, OSError): + # Tests inside generated modules may not have a real file. + caller_dir = None + + return PipelineFn( + fn=fn, + name=name, + description=description, + config_path=config, + annotations=dict(annotations or {}), + task_annotations=dict(task_annotations or {}), + caller_dir=caller_dir, + output_name=output_name, + propagate_config=propagate_config, + ) + + return decorator diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/placeholders.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/placeholders.py new file mode 100644 index 0000000..c7f68a8 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/placeholders.py @@ -0,0 +1,162 @@ +"""Trace-time placeholder objects. + +These objects are returned by the framework's primitives during trace +mode. They are NOT executed — they only carry enough metadata for +``emit.py`` to render the corresponding YAML sub-tree. + +Two kinds: + +* :class:`TaskOutputProxy` — returned by ``CallableRef.__call__``. + Acts as a handle to a task's output. Bare proxies (``_output is None``) + emit ``outputName: wait_for_output`` (the canonical done sentinel). + Attribute access (``task_a.rows_written``) returns a NEW proxy with + ``_output="rows_written"`` so emit can pick the named output. + +* :class:`SubpipelineOutputProxy` — a STRICT :class:`TaskOutputProxy` + returned by ``subpipeline(child)(...)``. It knows the child's declared + output names (derived at parent trace time from the child + ``PipelineFn``) and rejects unknown named access; a bare proxy resolves + to a default output only when unambiguous. + +* :class:`GraphInputPlaceholder` — bound to each ``In[T]`` parameter at + trace start. When passed as ``wait_for=cfg.parent_wait_token`` (or + ``depends_on=...``) it emits as ``graphInput: {inputName: }``. + When passed as a regular argument it emits as a ``graphInput`` value + too (dispatch is purely on value type in ``emit.py``). +""" +from __future__ import annotations + + +class TaskOutputProxy: + """Handle to a task's output. + + Construction: ``TaskOutputProxy(task_id="Build Quality Tables")`` is + a bare proxy (``_output is None`` → emit defaults to + ``wait_for_output``). Attribute access ``proxy.rows_written`` yields + a new proxy with the output name pinned. + + Note: ``__getattr__`` is only called when normal lookup fails. The + private attrs ``_task_id`` / ``_output`` are set on the instance so + they go through normal lookup; user-facing names (``rows_written``, + ``foo``, ...) fall through to ``__getattr__``. + """ + + __slots__ = ("_task_id", "_output") + + def __init__(self, task_id: str, output: str | None = None) -> None: + self._task_id = task_id + self._output = output + + def __getattr__(self, name: str) -> "TaskOutputProxy": + # Names starting with `_` are framework-internal — treat as + # missing rather than silently wrapping; surfaces typos cleanly. + if name.startswith("_"): + raise AttributeError(name) + return TaskOutputProxy(self._task_id, name) + + def _resolved_output_name(self) -> str: + """Effective output name for emission. + + A bare proxy (``_output is None``) defaults to the canonical + ``wait_for_output`` done sentinel — the permissive behavior used by + generic ``ref()`` / ``@task`` outputs. :class:`SubpipelineOutputProxy` + overrides this to apply child-declared-output rules. + """ + return self._output or "wait_for_output" + + def __repr__(self) -> str: # pragma: no cover — debug only + return f"TaskOutputProxy(task_id={self._task_id!r}, output={self._output!r})" + + +class SubpipelineOutputProxy(TaskOutputProxy): + """Strict output handle for a ``subpipeline(child)(...)`` boundary task. + + Unlike the permissive base proxy, this proxy KNOWS the child's declared + output names — derived at parent trace time from the child + ``PipelineFn``'s return annotation (a single ``-> Out[T]`` yields the + one ``output_name``; an ``Outputs`` subclass yields its field names) — + and enforces them: + + * attribute access for an output NOT in the declared set raises a clear + :class:`CompileError` listing the declared outputs; + * a bare proxy (``_output is None``) used as a dependency + (``wait_for=child`` / ``depends_on=child``) resolves to a default + output only when unambiguous — a single declared output, or an output + literally named ``wait_for_output``; otherwise it raises. + + ``declared_outputs is None`` means the child's outputs could not be + determined statically (an unsupported return shape); the proxy then + degrades to the permissive base behavior so authoring is never blocked + by a best-effort gap. + """ + + __slots__ = ("_declared_outputs", "_child_name") + + def __init__( + self, + task_id: str, + declared_outputs: tuple[str, ...] | None, + child_name: str, + output: str | None = None, + ) -> None: + super().__init__(task_id, output) + self._declared_outputs = declared_outputs + self._child_name = child_name + + def __getattr__(self, name: str) -> "TaskOutputProxy": + if name.startswith("_"): + raise AttributeError(name) + declared = self._declared_outputs + if declared is not None and name not in declared: + from .errors import CompileError + + raise CompileError( + f"child pipeline {self._child_name!r} has no output {name!r}. " + f"Declared outputs: {list(declared)}." + ) + return SubpipelineOutputProxy( + self._task_id, declared, self._child_name, output=name + ) + + def _resolved_output_name(self) -> str: + if self._output is not None: + return self._output + declared = self._declared_outputs + # Permissive fallback when the child interface is undeterminable. + if declared is None: + return "wait_for_output" + if len(declared) == 1: + return declared[0] + if "wait_for_output" in declared: + return "wait_for_output" + from .errors import CompileError + + raise CompileError( + f"child pipeline {self._child_name!r} declares multiple outputs " + f"{list(declared)} and none named 'wait_for_output', so a bare " + "reference is ambiguous. Access a named output explicitly, e.g. " + f"`.{declared[0]}`." + ) + + def __repr__(self) -> str: # pragma: no cover — debug only + return ( + f"SubpipelineOutputProxy(task_id={self._task_id!r}, " + f"output={self._output!r}, declared={self._declared_outputs!r})" + ) + + +class GraphInputPlaceholder: + """Trace-time stand-in for a pipeline ``In[T]`` parameter. + + Carried verbatim through trace; ``emit.py`` decides how to render it + based on the argument key (``wait_for`` / ``depends_on`` → graphInput + sub-dict; otherwise bare input name). + """ + + __slots__ = ("input_name",) + + def __init__(self, input_name: str) -> None: + self.input_name = input_name + + def __repr__(self) -> str: # pragma: no cover — debug only + return f"GraphInputPlaceholder(input_name={self.input_name!r})" diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/raw.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/raw.py new file mode 100644 index 0000000..5e1b812 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/raw.py @@ -0,0 +1,122 @@ +"""``raw()`` — mark a string argument as a legitimate RUNTIME placeholder. + +The compiler's no-template-delimiter output guard +rejects any ``{{``, ``{%`` or ``{#`` anywhere in the compiled dehydrated +pipeline, because a surviving delimiter normally means an upstream +template was not rendered at compile time. That guard is intentionally +operation-agnostic and cannot tell a genuine compile-time leak from an +intentional runtime sentinel. + +Some Tangle ops legitimately ship a literal ``{{name}}`` placeholder that +is substituted at RUN time, NOT at compile/hydrate time, and NOT by Jinja. +The canonical case is a ``run-query`` task whose ``input_1`` is the +runtime-timestamped output table of an upstream ``unique_output=True`` +task: the fully-qualified table name is unknown at compile time, so the +SQL must carry a literal ``{{input_1}}`` that the run-query op substitutes +at runtime via a plain ``query.replace("{{" + key + "}}", value)`` — there +is no Jinja involved. Hydrate passes the sentinel through untouched. + +Wrapping such a value in :func:`raw` tells the compiler "this delimiter is +intentional": the inner string is emitted verbatim (exactly as a plain +``str`` constant is) AND the argument's location is recorded in an +exempt-paths allow-list so the output guard skips it. Every OTHER +delimiter in the compiled output still fails the guard, so real +compile-time leaks are still caught. + +Usage:: + + from tangle_cli.python_pipeline import raw, ref + + add_cache_key = ref(url="resolve://./components.yaml#run-query")( + # ``input_1`` is wired to the runtime-timestamped output of an + # upstream unique_output task; the op str.replaces ``{{input_1}}`` + # at run time, so the SQL must ship the literal sentinel. + input_1=mine_options.output_table, + sql_query=raw("SELECT * FROM `{{input_1}}`"), + ) + +``raw`` only accepts a ``str``; the wrapped value is still subject to the +ordinary runnable-argument contract (it is emitted as a raw string +constant). It is NOT an escape hatch for compile-time Jinja: use it ONLY +for placeholders an op resolves at runtime. +""" +from __future__ import annotations + + +class Raw: + """A string argument value whose template delimiters are intentional. + + Carries a single ``str`` whose ``{{...}}`` is a legitimate RUNTIME + placeholder (see the module docstring). At emit time the inner + string is rendered verbatim — identical to a plain ``str`` constant — + and the argument's JSON path is added to the output guard's + exempt-paths allow-list so that one location is skipped while every + other delimiter in the output still fails the guard. + + Construct via the :func:`raw` helper rather than directly. + """ + + __slots__ = ("value",) + + def __init__(self, value: str) -> None: + if not isinstance(value, str): + raise TypeError( + f"raw() requires a str, got {type(value).__name__!r}. raw() " + "marks a string runtime placeholder (e.g. a run-query " + "'{{input_1}}' sentinel) as intentional; convert non-string " + "values to a string in your pipeline code first." + ) + # raw() exempts a value from the output guard's no-template-delimiter + # check, but ONLY for {{...}} runtime sentinels — strings an op + # str.replaces at RUN time. {%...%} (statements) and {#...#} (comments) + # are Jinja constructs rendered at COMPILE time, never substituted at + # run time, so a surviving one is a leaked compile-time template, not a + # runtime placeholder — exactly what raw() must not hide. Reject them + # at construction so the exemption can never rescue a real leak. + token = "{%" if "{%" in value else ("{#" if "{#" in value else None) + if token is not None: + raise ValueError( + "raw() values may carry only {{...}} runtime-substitution " + "sentinels (e.g. a run-query '{{input_1}}' placeholder an op " + f"str.replaces at run time). The value passed to raw() contains a Jinja {token} " + "token, which is never a valid runtime placeholder: {%...%} " + "(statements) and {#...#} (comments) are rendered by Jinja at " + "COMPILE time, not substituted at run time, so a surviving one " + "is a leaked compile-time template — exactly what raw() must " + "not hide. Render the Jinja in your pipeline code before " + "passing the result to raw(), or drop raw() if the value has " + "no runtime sentinel." + ) + self.value = value + + def __repr__(self) -> str: # pragma: no cover — debug only + return f"Raw({self.value!r})" + + def __eq__(self, other: object) -> bool: + return isinstance(other, Raw) and other.value == self.value + + def __hash__(self) -> int: + return hash((Raw, self.value)) + + +def raw(value: str) -> Raw: + """Mark ``value`` as a legitimate RUNTIME template placeholder. + + Returns a :class:`Raw` wrapper. Pass it as a task argument value when + the string legitimately ships a ``{{...}}`` placeholder that an op + substitutes at RUN time (not compile/hydrate time, and not via Jinja) + — e.g. a ``run-query`` ``sql_query`` carrying ``{{input_1}}`` for a + runtime-timestamped upstream table. ``{%...%}`` and ``{#...#}`` Jinja + tokens are rejected: they are rendered at compile time, so a surviving + one is a leaked compile-time template, never a runtime placeholder. + + The wrapped string is emitted verbatim (exactly like a plain ``str`` + constant) and its location is exempted from the compiler's + no-template-delimiter output guard, while every other delimiter in the + compiled pipeline still fails that guard. + + Raises: + TypeError: when ``value`` is not a ``str``. + ValueError: when ``value`` contains a ``{%...%}`` or ``{#...#}`` Jinja token. + """ + return Raw(value) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py new file mode 100644 index 0000000..4f540ad --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py @@ -0,0 +1,324 @@ +"""``ref(url=...)`` — callable component handle. + +Returns a :class:`CallableRef` that records the user's URL/name/digest +verbatim. Methods that capture additional metadata (``.bind``, +``.named``, ``.with_annotations``) are composable, immutable operations +the tracer can call. Calling a :class:`CallableRef` inside a live +``@pipeline`` trace context records a :class:`TaskNode` into the active +:class:`GraphBuilder`; calling it outside a trace raises ``RuntimeError``. +""" +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any + +from .errors import CompileError + + +@dataclass(frozen=True) +class CallableRef: + """Immutable handle to a Tangle component. + + Core shape: ``url``, ``ref_name``, ``ref_digest``, ``bound_kwargs``, + ``task_id_hint``, ``annotations``. A parallel ``_task_*`` metadata + bundle lets a CallableRef produced by ``@task`` be + ``materialize()``-d into a component YAML at compile time, and a + ``_registered_*`` bundle lets a CallableRef produced by ``@registered`` + be rewritten to a ``resolve://`` URL pointing at an EXISTING + ``gen_config.yaml`` (no sidecar generated). The same proxy type is + reused for ``ref()``, ``@task`` and ``@registered`` so the tracer + doesn't need to special-case anything; for ``@task`` the ``url`` field + is left ``None`` and the compile driver fills it in with a + ``resolve://`` URL pointing at the generated sidecar, while for + ``@registered`` the ``url`` carries the ``registered://pending`` + sentinel the driver rewrites to the gen_config.yaml ``resolve://`` URL. + ``@task`` and ``@registered`` are mutually exclusive: a ref is built by + one decorator or the other, so only one metadata bundle is ever set. + """ + + url: str | None + ref_name: str | None = None + ref_digest: str | None = None + bound_kwargs: dict[str, Any] = field(default_factory=dict) + task_id_hint: str | None = None + annotations: dict[str, str] | None = None + # ``@task`` metadata. ``None`` for ``ref()``-derived refs; + # populated by the ``@task`` decorator. ``materialize()`` rejects + # any ref where ``_task_source_path`` is None. + _task_source_path: Path | None = None + _task_function_name: str | None = None + _task_image: str | None = None + _task_dependencies_from: Path | None = None + _task_custom_annotations: dict[str, str] | None = None + # ``@registered`` metadata. ``None`` for ``ref()``/``@task`` refs; + # populated by the ``@registered`` decorator. Drives the compile-time + # rewrite of the ``registered://pending`` sentinel URL to a + # ``resolve:///gen_config.yaml#`` URL pointing at an + # EXISTING gen_config.yaml (NOT sidecar generation -- nothing is + # written). ``_registered_gen_config`` is the author-supplied path (or + # ``None`` to default to the nearest ancestor ``gen_config.yaml``); + # ``_registered_fragment`` is the gen_config top-level key (or ``None`` + # to default to the function name verbatim). + _registered_source_path: Path | None = None + _registered_function_name: str | None = None + _registered_fragment: str | None = None + _registered_gen_config: str | None = None + + # ------------------------------------------------------------------ + # Builders — all return a fresh CallableRef (immutable composition) + + def bind(self, **kwargs: Any) -> "CallableRef": + """Return a new CallableRef with ``kwargs`` merged into + ``bound_kwargs``. Later .bind calls win on conflict.""" + merged = {**self.bound_kwargs, **kwargs} + return replace(self, bound_kwargs=merged) + + def named(self, task_id: str) -> "CallableRef": + """Return a new CallableRef whose task ID will be ``task_id`` + instead of the LHS-derived auto ID.""" + return replace(self, task_id_hint=task_id) + + def with_annotations(self, ann: dict[str, Any]) -> "CallableRef": + """Return a new CallableRef with per-task annotations. + + emit.py renders these as an ``annotations:`` block on the task + before ``componentRef:``. + """ + merged: dict[str, str] = dict(self.annotations or {}) + # Values are coerced to str at emit time; here we accept Any but + # store as a dict to preserve user intent. + for k, v in ann.items(): + merged[k] = v # type: ignore[assignment] + return replace(self, annotations=merged) + + # ------------------------------------------------------------------ + # @task codegen — materialize() writes the component YAML. + + def materialize(self, output_path: Path | None = None) -> Path: + """Write the component YAML for a ``@task``-derived ref to disk. + + Only valid on CallableRefs produced by the ``@task`` decorator + — those carry the source path / function name / image metadata + needed by ``tangle_cli.component_generator.regenerate_yaml``. + Calling this on a plain ``ref(url=...)`` raises + :class:`RuntimeError` (the user already has a YAML on the other + end of that URL — there's nothing for us to generate). + + Args: + output_path: Where to write the YAML. Defaults to + ``/generated/.yaml`` to + match the convention used by the emitted + ``componentRef.url`` (``file://./generated/.yaml``). + + Returns: + The resolved ``output_path``. + + Lazy import: ``tangle_cli.component_generator`` is imported only + here, never at package import time. + """ + if self._task_source_path is None or self._task_function_name is None: + raise RuntimeError( + "CallableRef.materialize() is only valid for refs created " + "by the @task decorator. This ref was built via ref(url=...) " + "and points at an existing YAML; there is nothing to " + "generate." + ) + + if output_path is None: + output_path = ( + self._task_source_path.parent + / "generated" + / f"{self._task_function_name}.yaml" + ) + + # Lazy import keeps package import cheap. ``ComponentGenerator`` is + # imported only when an authoring ref materializes component YAML. + from tangle_cli.component_generator import ComponentGenerator + + generator = ComponentGenerator() + if self._task_custom_annotations: + image = ( + self._task_image + or generator.extract_image_from_yaml(output_path) + or generator.default_container_image + ) + deps = self._task_dependencies_from or generator.find_dependencies_file( + self._task_source_path + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + generator.generate_component_yaml( + file_path=self._task_source_path, + output_path=output_path, + container_image=image, + function_name=self._task_function_name, + dependencies_from=deps, + custom_annotations=self._task_custom_annotations, + ) + else: + generator.regenerate_yaml( + python_file=self._task_source_path, + output_path=output_path, + function_name=self._task_function_name, + image=self._task_image, + dependencies_from=self._task_dependencies_from, + ) + return output_path + + # ------------------------------------------------------------------ + # Trace-mode call site + + def __call__(self, **kwargs: Any) -> Any: + """Trace-mode invocation. + + Records a :class:`TaskNode` into the active :class:`GraphBuilder` + (looked up via ``trace.current_builder()``) and returns a bare + :class:`TaskOutputProxy` handle. The task ID is either the value + captured by ``.named(...)`` or the LHS variable name on the + call site (resolved via the AST pre-pass map stashed on the + builder). + + Edge kwargs (``wait_for`` / ``depends_on``) and regular kwargs + share one ``arguments`` dict in the IR; the value-vs-key + dispatch happens at emit time. ``.bind(...)`` kwargs are merged + in last so call-site kwargs win on conflict (same key) and come + first in insertion order (the bind block is appended). + """ + # Local import keeps the @ref shell importable during early + # bootstrap (and avoids the circular dep at module load). + import sys + + from . import trace + from .errors import AmbiguousTaskIdError + from .graph import TaskNode + from .ids import snake_to_title_case + from .placeholders import TaskOutputProxy + + builder = trace.current_builder() + if builder is None: + raise RuntimeError( + "CallableRef.__call__ requires an active @pipeline trace " + "context. Either call this inside a function decorated " + "with @pipeline, or use the `tangle-deploy pipeline compile " + "from-python` driver." + ) + + # Resolve the task ID. ``.named(...)`` always wins over the + # AST-derived auto ID. + if self.task_id_hint is not None: + task_id = self.task_id_hint + else: + caller_lineno = sys._getframe(1).f_lineno + lhs_name = builder.lineno_to_lhs.get(caller_lineno) + if lhs_name is None: + raise AmbiguousTaskIdError( + f"Cannot infer task ID at line {caller_lineno}: the LHS " + "is not a single bare variable name. Either bind to a " + "single name (e.g. `task_a = ref(...)(...)`) or call " + "`.named('Task Id')` on the ref." + ) + task_id = snake_to_title_case(lhs_name) + + # Merge bound and call-site kwargs: call-site keys come first in + # insertion order and win on conflict; bound keys are appended. + merged: dict[str, Any] = {} + for k, v in kwargs.items(): + merged[k] = v + for k, v in self.bound_kwargs.items(): + if k not in merged: + merged[k] = v + + node = TaskNode( + task_id=task_id, + ref_url=self.url, + ref_name=self.ref_name, + ref_digest=self.ref_digest, + arguments=merged, + annotations=dict(self.annotations) if self.annotations else None, + ) + builder.add_task(node) + + # If this ref was produced by ``@task``, record ``(task_id, + # self)`` on the builder so the compile driver can (a) auto-emit + # a sibling ``.components.yaml`` with a ``local_from_python:`` + # entry for each unique @task source file, and (b) rewrite this + # task's ``componentRef.url`` to + # ``resolve://./.components.yaml#``. Idle + # for plain ``ref(url=...)`` refs (their YAML is already on + # the other end of the URL). + if self._task_source_path is not None: + builder.task_refs_for_local_from_python.append((task_id, self)) + + # If this ref was produced by ``@registered``, record ``(task_id, + # self)`` on the builder so the compile driver can rewrite this + # task's ``componentRef.url`` from the ``registered://pending`` + # sentinel to a pure ``resolve:///gen_config.yaml#`` + # URL pointing at the operation's EXISTING gen_config.yaml. No + # sidecar is generated. ``@task`` and ``@registered`` are mutually + # exclusive, so this branch and the ``@task`` branch above never + # both fire for one ref. + if self._registered_source_path is not None: + builder.task_refs_for_registered.append((task_id, self)) + + return TaskOutputProxy(task_id=task_id) + + +def ref( + url: str | None = None, + *, + name: str | None = None, + digest: str | None = None, + tag: str | None = None, +) -> CallableRef: + """Build a CallableRef pointing at a Tangle component. + + A ref carries a *locator* — either a ``url`` or a published ``name`` — + and may optionally pin a ``digest`` alongside either (or stand on its + own). All values are stored verbatim (no normalization). The emitter + turns the locator into the matching ``componentRef`` form, and the + hydrator resolves it via ``_fetch_component_by_{url,name,digest}``. + + Supported (WORKING) locator combinations: + + - ``ref(url="resolve://#")`` — relative to the hydrating + pipeline; also ``file://./…`` / ``gs://…`` / ``https://…`` URLs — + emits ``{"url": …}``. + - ``ref(name="my-comp")`` — a published component by name — emits + ``{"name": …}``. + - ``ref(name="my-comp", digest="<64hex>")`` — name pinned to a digest — + emits ``{"name": …, "digest": …}``. + - ``ref(digest="<64hex>")`` — pin by digest alone — emits + ``{"digest": …}``. + - ``ref(url="gs://b/x.yaml", digest="<64hex>")`` — a URL pinned to a + digest — emits ``{"url": …, "digest": …}``. + + ``tag`` is accepted in the signature for forward compatibility but is + **NOT supported yet**: the hydrator has no tag fetcher, so a ``tag`` + ref would compile to a dehydrated YAML the hydrator silently leaves + unresolved. Passing ``tag=`` raises :class:`CompileError`. Pin by + ``name=`` and/or ``digest=`` instead. + + Raises: + CompileError: if ``tag`` is passed (deferred); if both ``url`` and + ``name`` are passed (conflicting primary locators); or if no + locator at all is given. + """ + # 1. tag is not resolvable end-to-end yet (hydrator has no tag fetcher). + if tag is not None: + raise CompileError( + "ref(tag=...) is not supported yet: the hydrator resolves " + "digest/name/url only. Pin by name=... and/or digest=... instead." + ) + # 2. exactly one PRIMARY locator family: url XOR name (digest is optional + # and may accompany either, or stand alone). + if url is not None and name is not None: + raise CompileError( + "ref() takes EITHER url=... OR name=..., not both (conflicting " + "locators). Use digest=... to pin a version alongside either." + ) + if url is None and name is None and digest is None: + raise CompileError( + "ref() requires a locator: pass url=..., or name=...[, digest=...], " + "or digest=..." + ) + return CallableRef(url=url, ref_name=name, ref_digest=digest) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py new file mode 100644 index 0000000..fdd58bd --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py @@ -0,0 +1,201 @@ +"""``@registered`` decorator. + +The decorator marks a Python function as an operation that is ALREADY +registered in an existing ``gen_config.yaml`` (a ``resolve://`` config), +typically published separately and pushed by a CI step. Unlike ``@task`` +— which authors the operation inline and auto-generates a sibling +``.components.yaml`` sidecar — ``@registered`` generates NOTHING. Its +only job is to stamp a *fragment* + the *gen_config.yaml* the operation is +registered in, so that at compile time the driver can emit a +``componentRef: {url: "resolve:///gen_config.yaml#"}`` +pointing at that already-existing config. + +This fits the common monorepo setup (e.g. the ``world`` zone +``areas/ml/upi/``) where operations are published once into a +hand-maintained ``gen_config.yaml`` and pipelines reference them by +``resolve://`` URL. Authors decorate the registered operation functions +with ``@registered`` and call them from a ``@pipeline`` body; the compiler +takes care of computing the right relative ``resolve://`` URL. + +At compile time the driver collects every ``@registered`` ref called +inside a ``@pipeline`` body and rewrites each task's componentRef URL to +``resolve://#``, computed PER +artifact against that artifact's own output directory (so nested +subpipeline children land on a correct relative path). No sidecar is ever +written — the gen_config.yaml the URL points at already exists on disk. + +Crucially, the decorator does NOT call the user function, and it does NOT +touch the filesystem beyond the cheap ``inspect.getfile(fn)`` probe at +decoration time. Resolving the gen_config.yaml (the marker walk, the +nearest-``gen_config.yaml`` walk, the relpath) happens LAZILY in the +compiler, so importing an operation module in isolation (e.g. for the +op's own unit tests) never requires the marker or gen_config to be +present. + +Lazy import contract: like ``task.py``, this module does NOT import +``tangle_cli.component_generator`` at module load. It DOES eagerly +install the ``cloud_pipelines`` shim (same as ``task.py``) so generated +ops that do ``from cloud_pipelines import components`` at top level can be +imported by ``@registered`` wrappers without each pipeline having to call +a private tangle_cli helper. + +Example:: + + from tangle_cli.python_pipeline import registered + + @registered(fragment="run-query", gen_config="shared/components/gen_config.yaml") + def run_query(sql_query: str = "SELECT 1") -> str: + '''Run a query. + + Metadata: + Name: Run Query + Version: 1.0.0 + ''' + ... + + # ``run_query`` is now a CallableRef -- NOT the function. Calling it + # inside a @pipeline body records a TaskNode and registers the ref so + # the compile driver rewrites its componentRef URL to + # ``resolve:///gen_config.yaml#run-query`` at compile time. +""" +from __future__ import annotations + +import inspect +from pathlib import Path +from typing import Any, Callable + +# Install the ``cloud_pipelines`` shim eagerly at module load so generated +# components that do ``from cloud_pipelines import components`` at top +# level can be imported by user @registered wrappers without each pipeline +# having to call a private tangle_cli helper. ``component_from_func`` +# is NOT ``component_generator`` (the heavy codegen module) -- it's the +# lighter introspection module, and the shim helper itself is a no-op when +# ``cloud_pipelines`` is already in ``sys.modules``. +from tangle_cli.component_from_func import _ensure_cloud_pipelines_shim + +from .ref import CallableRef + +_ensure_cloud_pipelines_shim() + +# Sentinel componentRef URL stamped onto a @registered ref at decoration +# time. The compile driver rewrites it to a real +# ``resolve:///gen_config.yaml#`` URL after tracing +# (the relpath depends on the artifact's output dir), so it never reaches +# the written output. This module is the single source of truth for the +# sentinel string; the compiler imports it for the pending-sentinel guard. +_REGISTERED_URL_PLACEHOLDER = "registered://pending" + + +def registered( + *, + fragment: str | None = None, + gen_config: str | None = None, +) -> Callable[[Callable[..., Any]], CallableRef]: + """Decorator: mark a function as an operation registered in a gen_config.yaml. + + The decorated function is NEVER executed by the framework. Instead the + decorator captures the *fragment* and *gen_config* onto a + :class:`CallableRef`. The compile driver rewrites the task's + componentRef URL to ``resolve:///gen_config.yaml#``, + computed relative to the compiled artifact's output directory. Hydrate + then uses tangle-deploy's own ``resolve://`` resolver to read the + gen_config.yaml fragment and inline the component spec. + + Unlike ``@task``, ``@registered`` references an EXISTING + gen_config.yaml (the operation is registered/published elsewhere) and + generates NO sidecar — so it takes no ``image`` / ``dependencies_from`` + (those live in the operation's own ``gen_config.yaml`` + ``local_from_python`` entry). + + Args: + fragment: The top-level key in ``gen_config.yaml`` this operation + is registered under. Used VERBATIM as the ``#fragment`` of the + emitted ``resolve://`` URL. When omitted, defaults to the + function name verbatim (no hyphenation). No compile-time + fragment validation is performed in v1 — a wrong fragment + surfaces at hydrate time. + gen_config: Path to the ``gen_config.yaml`` the operation is + registered in. Resolution (lazy, at compile time): + + - A genuinely remote value (``gs://…`` or ``http(s)://…``) is + used VERBATIM; the hydrator fetches it directly. + - A ``file://…`` URL or an absolute filesystem path is resolved + to a local file, existence-checked at compile time, and + emitted as a bare absolute ``resolve://`` path. The ``file://`` + scheme is STRIPPED (the hydrator's ``resolve://`` parser does + not understand it, so ``resolve://file:///abs/...`` would not + hydrate). + - An explicit relative path is resolved against the nearest + ancestor directory of the operation's source file that + contains the ``oasis.pipeline_component_root.yaml`` zone-root + marker. + - When omitted, defaults to the NEAREST ancestor + ``gen_config.yaml`` of the operation's source file. + + Returns: + A decorator that, given the user's function, returns a + :class:`CallableRef`. The returned ref: + + - Has ``url`` set to the ``registered://pending`` sentinel -- the + compile driver rewrites it to + ``resolve:///gen_config.yaml#`` after tracing. + - Has ``_registered_*`` metadata populated so the driver can + resolve the gen_config.yaml and compute the fragment. + - Behaves like any other ``ref()``-derived ref when called inside a + ``@pipeline`` trace context. + """ + + def decorator(fn: Callable[..., Any]) -> CallableRef: + # Capture the absolute path of the source file the user wrote the + # function in. ``inspect.getfile`` raises TypeError for builtins / + # dynamically-built functions; the @registered path requires a real + # on-disk file because the gen_config.yaml resolution walks the + # source file's ancestor directories. + try: + source_path = Path(inspect.getfile(fn)).resolve() + except (TypeError, OSError) as exc: + raise RuntimeError( + f"@registered could not resolve the source file for {fn!r}: {exc}. " + "@registered only supports functions defined in real .py files; the " + "compiler walks the source file's directory to resolve the gen_config.yaml." + ) from exc + + function_name = fn.__name__ + + # The sentinel URL is set at decoration time. The compile driver + # rewrites componentRef.url for @registered-derived refs to + # ``resolve:///gen_config.yaml#`` after tracing + # (the relpath depends on the output path), so the placeholder + # never reaches output. Still a pure ref -- no spec/text. + ref_instance = CallableRef( + url=_REGISTERED_URL_PLACEHOLDER, + _registered_source_path=source_path, + _registered_function_name=function_name, + _registered_fragment=fragment, + _registered_gen_config=gen_config, + ) + + # Expose function-like introspection so ``tangle_cli``'s + # ``extract_interface`` can read signature/docstring directly off + # the CallableRef. The SAME function is still published normally via + # its ``gen_config.yaml`` ``local_from_python`` entry, whose + # resolver does ``getattr(module, fn)`` -> ``inspect.signature``; + # forwarding these dunders lets that path see the function through + # the ref. (Identical to @task -- see task.py.) + # + # ``object.__setattr__`` bypasses the frozen-dataclass + # ``__setattr__`` guard so we can add these attrs without mutating + # any declared dataclass field. + object.__setattr__(ref_instance, "__name__", function_name) + object.__setattr__(ref_instance, "__qualname__", fn.__qualname__) + object.__setattr__(ref_instance, "__module__", fn.__module__) + object.__setattr__(ref_instance, "__doc__", fn.__doc__) + object.__setattr__(ref_instance, "__wrapped__", fn) + object.__setattr__(ref_instance, "__signature__", inspect.signature(fn)) + # Forward annotations too -- some introspection paths read + # ``__annotations__`` directly rather than through signature. + object.__setattr__(ref_instance, "__annotations__", dict(fn.__annotations__)) + + return ref_instance + + return decorator diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py new file mode 100644 index 0000000..6122a2a --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py @@ -0,0 +1,211 @@ +"""``subpipeline(child_pipeline)`` — nested-pipeline authoring handle. + +A parent ``@pipeline`` uses another Python ``@pipeline`` as a graph +operation by wrapping it:: + + from tangle_cli.python_pipeline import subpipeline + + child_result = subpipeline(child_pipeline).named("Run Child")( + required_input=upstream.some_output, + depends_on=upstream, + ) + +``subpipeline(child)`` returns a :class:`SubpipelineRef` — a task-like +handle that mirrors :class:`tangle_cli.python_pipeline.ref.CallableRef` +ergonomics (``.bind`` / ``.named`` / ``.with_annotations`` and call-site +kwargs). Calling the handle inside an active ``@pipeline`` trace records +ONE parent task (never the child's internals) and returns a +:class:`tangle_cli.python_pipeline.placeholders.TaskOutputProxy`. + +The child body is NOT executed into the parent's :class:`GraphBuilder`. +The compile driver (a later milestone) reads the recorded child +``PipelineFn`` from ``builder.task_refs_for_subpipelines`` to compile it +into a graph component sidecar and rewrite the parent task's +``componentRef`` from the :data:`_SUBPIPELINE_URL_PLACEHOLDER` sentinel +to a pure ``file://`` ref. Per Decision A the subpipeline kind is carried +explicitly (this dedicated handle + the separate builder registry); it is +never inferred from ``ref_url is None``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING, Any + +from .errors import CompileError + +if TYPE_CHECKING: # pragma: no cover + from .pipeline import PipelineFn + from .placeholders import TaskOutputProxy + + +# Transient componentRef URL stamped on a subpipeline task during trace. +# The compile driver rewrites it to a pure ``file://./.subgraphs/ +# .yaml`` URL after compiling the child. It is its OWN sentinel, +# distinct from ``@task``'s ``local-from-python://pending``, so a missed +# rewrite can fail with a subpipeline-specific error rather than being +# confused with a ``@task`` placeholder. It must never reach written output. +_SUBPIPELINE_URL_PLACEHOLDER = "subpipeline://pending" + + +@dataclass(frozen=True) +class SubpipelineRef: + """Immutable, task-like handle wrapping a child :class:`PipelineFn`. + + Mirrors :class:`CallableRef`'s combinators and trace-time ``__call__`` + but carries an explicit ``child`` ``PipelineFn`` instead of a + component URL. The wrapped child is the source of truth for the + child's declared inputs/outputs in later compile phases. + """ + + child: "PipelineFn" + bound_kwargs: dict[str, Any] = field(default_factory=dict) + task_id_hint: str | None = None + annotations: dict[str, str] | None = None + config_overrides: dict[str, Any] = field(default_factory=dict) + + # ------------------------------------------------------------------ + # Builders — all return a fresh SubpipelineRef (immutable composition) + + def bind(self, **kwargs: Any) -> "SubpipelineRef": + """Return a new handle with ``kwargs`` merged into ``bound_kwargs``. + Later ``.bind`` calls win on conflict.""" + merged = {**self.bound_kwargs, **kwargs} + return replace(self, bound_kwargs=merged) + + def named(self, task_id: str) -> "SubpipelineRef": + """Return a new handle whose parent task ID will be ``task_id`` + instead of the LHS-derived auto ID.""" + return replace(self, task_id_hint=task_id) + + def with_annotations(self, ann: dict[str, Any]) -> "SubpipelineRef": + """Return a new handle with per-task annotations applied to the + PARENT subpipeline task (not the child sidecar's metadata).""" + merged: dict[str, str] = dict(self.annotations or {}) + for k, v in ann.items(): + merged[k] = v # type: ignore[assignment] + return replace(self, annotations=merged) + + def override_config(self, **kwargs: Any) -> "SubpipelineRef": + """Return a new handle with compile-time cfg overrides for the direct + child merged in (later calls win on conflict). + + A DISTINCT namespace from :meth:`bind`: ``.bind(...)`` sets the + child's RUNTIME ``In[...]`` graph-input bindings, whereas + ``.override_config(...)`` sets the child's COMPILE-TIME ``cfg`` + values. The keyword is the CHILD's config key (so a value can be + remapped, e.g. ``standardized_options_model=cfg.pvso_output_table_name``) + and the value is any already-typed Python expression. These are + validated STRICT against the child's ``config.yaml`` in the compile + driver — an unknown key is a compile error (typo protection).""" + merged = {**self.config_overrides, **kwargs} + return replace(self, config_overrides=merged) + + # ------------------------------------------------------------------ + # Trace-mode call site + + def __call__(self, **kwargs: Any) -> "TaskOutputProxy": + """Trace-mode invocation. + + Records a :class:`TaskNode` for the nested-pipeline boundary into + the active :class:`GraphBuilder`, registers ``(task_id, self)`` in + ``builder.task_refs_for_subpipelines``, and returns a bare + :class:`TaskOutputProxy` handle. The task ID is derived exactly as + :meth:`CallableRef.__call__` does — ``.named(...)`` wins, else the + single-Name LHS at the call site. + + The child ``PipelineFn`` is reachable for later compile phases via + the builder registry (``self.child``); the returned proxy is a + strict :class:`SubpipelineOutputProxy` that knows the child's + declared outputs (derived from the child's return annotation) so + unknown named access fails early and a bare proxy resolves to a + default output only when unambiguous. + """ + import sys + + from . import trace + from .errors import AmbiguousTaskIdError + from .graph import TaskNode + from .ids import snake_to_title_case + from .placeholders import SubpipelineOutputProxy + + builder = trace.current_builder() + if builder is None: + raise RuntimeError( + "subpipeline(child)(...) requires an active @pipeline trace " + "context. Either call this inside a function decorated with " + "@pipeline, or use the `tangle-deploy pipeline compile from-python` driver." + ) + + # Resolve the parent task ID. ``.named(...)`` always wins over the + # AST-derived auto ID. + if self.task_id_hint is not None: + task_id = self.task_id_hint + else: + caller_lineno = sys._getframe(1).f_lineno + lhs_name = builder.lineno_to_lhs.get(caller_lineno) + if lhs_name is None: + raise AmbiguousTaskIdError( + f"Cannot infer task ID at line {caller_lineno}: the LHS " + "is not a single bare variable name. Either bind to a " + "single name (e.g. `child = subpipeline(...)(...)`) or " + "call `.named('Task Id')` on the subpipeline handle." + ) + task_id = snake_to_title_case(lhs_name) + + # Merge bound and call-site kwargs: call-site keys come first in + # insertion order and win on conflict; bound keys are appended. + merged: dict[str, Any] = {} + for k, v in kwargs.items(): + merged[k] = v + for k, v in self.bound_kwargs.items(): + if k not in merged: + merged[k] = v + + node = TaskNode( + task_id=task_id, + ref_url=_SUBPIPELINE_URL_PLACEHOLDER, + arguments=merged, + annotations=dict(self.annotations) if self.annotations else None, + ) + builder.add_task(node) + builder.task_refs_for_subpipelines.append((task_id, self)) + + # Derive the child's declared outputs statically so the returned + # proxy can reject unknown named access and resolve bare references + # per the default-output rule (Decision E). ``None`` => undeterminable + # => permissive base behavior. + declared = trace.declared_output_names(self.child) + return SubpipelineOutputProxy( + task_id=task_id, + declared_outputs=declared, + child_name=self.child.name, + ) + + +def subpipeline(child: "PipelineFn") -> SubpipelineRef: + """Wrap a child ``@pipeline`` as a task-like nested-pipeline handle. + + Args: + child: The child ``PipelineFn`` to compile as a subgraph. Must be + a ``@pipeline``-decorated function imported from another module. + + Returns: + A :class:`SubpipelineRef` supporting ``.bind`` / ``.named`` / + ``.with_annotations`` and trace-time ``__call__``. + + Raises: + CompileError: if ``child`` is not a :class:`PipelineFn` (e.g. a + plain function, a :class:`CallableRef`, or anything else). + """ + # Local import to avoid an import cycle at package load time. + from .pipeline import PipelineFn + + if not isinstance(child, PipelineFn): + raise CompileError( + "subpipeline(...) expects a @pipeline-decorated function " + f"(PipelineFn), got {type(child).__name__!r}. Import a child " + "@pipeline and wrap it, e.g. subpipeline(child_pipeline)(...). " + "For non-Python or precompiled components use ref(url=...)." + ) + return SubpipelineRef(child=child) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py new file mode 100644 index 0000000..632f269 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py @@ -0,0 +1,218 @@ +"""``@task`` decorator. + +The decorator captures metadata about a Python function (source path, +function name, container image, dependencies, custom name, annotations) +and returns a :class:`CallableRef` with NO componentRef URL set. + +At compile time the driver collects every ``@task`` ref that gets called +inside a ``@pipeline`` body, auto-emits a sibling +``.components.yaml`` with one ``local_from_python:`` entry per +unique source file, and rewrites each task's componentRef URL to +``resolve://./.components.yaml#``. Hydrate then uses +tangle-deploy's own ``local_from_python`` resolver to call +``regenerate_yaml`` at hydrate time -- no pre-codegen step on our side. + +Crucially, the decorator does NOT call the user function. + +Lazy import contract: this module does NOT import +``tangle_cli.component_generator`` at module load. The codegen +import lives inside ``CallableRef.materialize()`` (kept as a public +escape hatch for users who want to inspect the generated YAML offline) +so importing this package stays cheap. + +Example:: + + from tangle_cli.python_pipeline import task + + @task(image="python:3.12", dependencies_from="./pyproject.toml") + def my_task(out: OutputPath("Text"), greeting: str = "hello"): + '''Write a greeting. + + Metadata: + Name: My Task + Version: 1.0.0 + ''' + with open(out, "w") as f: + f.write(greeting) + + # ``my_task`` is now a CallableRef -- NOT the function. Calling it + # inside a @pipeline body records a TaskNode and registers the ref + # for local_from_python auto-emission at compile time. +""" +from __future__ import annotations + +import inspect +from pathlib import Path +from typing import Any, Callable + +# Install the ``cloud_pipelines`` shim eagerly at module load so +# upstream-generated components that do ``from cloud_pipelines import components`` +# at top level can be imported by user @task wrappers without each +# pipeline having to call a private tangle_cli helper. +# ``component_from_func`` is NOT ``component_generator`` (the heavy +# codegen module) -- it's the lighter introspection module, and the shim +# helper itself is a no-op when ``cloud_pipelines`` is already in +# ``sys.modules``. +from tangle_cli.component_from_func import _ensure_cloud_pipelines_shim + +from .ref import CallableRef +from .task_env import TaskEnv + +_ensure_cloud_pipelines_shim() + + +def task( + *, + env: TaskEnv | None = None, + image: str | None = None, + dependencies_from: str | Path | None = None, + annotations: dict[str, Any] | None = None, +) -> Callable[[Callable[..., Any]], CallableRef]: + """Decorator: turn a Python function into a Tangle component ref. + + The decorated function is NEVER executed by the framework. Instead + the decorator captures metadata onto a :class:`CallableRef`. The + compile driver emits a sibling ``.components.yaml`` with a + ``local_from_python:`` entry for the function and rewrites the + task's componentRef URL to point at it. Hydrate uses tangle-deploy's + own resolver to regenerate the component YAML at hydrate time. + + Args: + env: Optional :class:`TaskEnv` bundling a reusable container + ``image`` + ``dependencies_from`` pair so several tasks can + share one declared-once execution environment (the Python + equivalent of a ``local_from_python`` YAML anchor). ``env`` is + expanded at decoration time into the same ``_task_image`` / + ``_task_dependencies_from`` metadata an explicit + ``image=`` / ``dependencies_from=`` would produce — no + ``TaskEnv`` object ever reaches the compiler, hydrator, or + runner. Explicit ``image=`` / ``dependencies_from=`` override + the env PER FIELD. + image: Container image for the component (required for + ``local_from_python`` resolution). The image string is + written verbatim into the emitted + ``components.yaml#local_from_python.image`` field. Overrides + ``env.image`` when both are given. + dependencies_from: Path to a ``pyproject.toml`` (or any file + ``tangle-deploy`` understands) that declares pip + dependencies. Resolved relative to the caller's source + file when given as a string. Emitted into + ``components.yaml#local_from_python.dependencies_from``. + Overrides ``env.dependencies_from`` when both are given. + annotations: Extra annotations to merge into the emitted + component's ``metadata.annotations`` block. + + Returns: + A decorator that, given the user's function, returns a + :class:`CallableRef`. The returned ref: + + - Has ``url`` set to ``None`` -- the compile driver fills it in + with ``resolve://./.components.yaml#`` + after tracing. + - Has ``_task_*`` metadata populated so the driver can build + the local_from_python entry. + - Behaves like any other ``ref()``-derived ref when called + inside a ``@pipeline`` trace context. + """ + + # Validate ``env`` up front (decoration time) with a message that + # names the public keyword so authors get an actionable error. + if env is not None and not isinstance(env, TaskEnv): + raise TypeError( + "@task(env=...) expects a TaskEnv instance, got " + f"{type(env).__name__!r}. Build one with " + "TaskEnv(image=..., dependencies_from=...)." + ) + + # Per-field precedence: an explicit ``image=`` / ``dependencies_from=`` + # overrides the corresponding ``env`` field; otherwise the env value + # (if any) is used. This mirrors YAML anchor semantics: start from the + # declared-once defaults, override locally where needed. + effective_image = image if image is not None else (env.image if env else None) + effective_deps_raw = ( + dependencies_from + if dependencies_from is not None + else (env.dependencies_from if env else None) + ) + + # Normalise ``dependencies_from`` early so the driver doesn't have + # to think about string-vs-Path forms. The path is resolved relative + # to the user's source file (set inside ``decorator``). An + # ``env.dependencies_from`` is ALREADY an absolute resolved Path (the + # TaskEnv resolved it at its definition site), so it passes through + # the normalisation below unchanged; an explicit relative string is + # still resolved relative to the @task source file. + raw_dependencies_from = effective_deps_raw + + def decorator(fn: Callable[..., Any]) -> CallableRef: + # Capture the absolute path of the source file the user wrote + # the function in. ``inspect.getfile`` raises TypeError for + # builtins / dynamically-built functions; the @task path + # requires a real on-disk file because tangle-deploy reads + # the source via inspect.getfile too. + try: + source_path = Path(inspect.getfile(fn)).resolve() + except (TypeError, OSError) as exc: + raise RuntimeError( + f"@task could not resolve the source file for {fn!r}: {exc}. " + "@task only supports functions defined in real .py files; the " + "codegen reads the source via inspect.getfile." + ) from exc + + function_name = fn.__name__ + + # Resolve dependencies_from relative to the source file when + # the user gave a string. Absolute paths and explicit Path + # objects pass through unchanged. + deps_path: Path | None + if raw_dependencies_from is None: + deps_path = None + else: + deps_path = Path(raw_dependencies_from) + if not deps_path.is_absolute(): + deps_path = (source_path.parent / deps_path).resolve() + + # No URL set at decoration time -- the compile driver rewrites + # componentRef.url for @task-derived refs to + # ``resolve://./.components.yaml#`` after + # tracing. emit.py tolerates None for @task refs because the + # driver fills it in before writing. + ref_instance = CallableRef( + url=None, + _task_source_path=source_path, + _task_function_name=function_name, + _task_image=effective_image, + _task_dependencies_from=deps_path, + _task_custom_annotations=dict(annotations) if annotations else None, + ) + + # Expose function-like introspection so ``tangle_cli``'s + # ``extract_interface`` can read signature/docstring directly + # off the CallableRef. After the @task decorator runs, the + # symbol the user wrote (``def my_task(...): ...``) is bound + # to ``ref_instance`` in the module's namespace; when + # ``regenerate_yaml`` does ``getattr(module, "my_task")`` it + # gets the ref. Forwarding these dunders lets + # ``inspect.signature`` / ``inspect.getdoc`` / + # ``inspect.getsource`` (via ``__wrapped__``) treat the ref as + # a stand-in for ``fn`` itself -- no separate function registry + # needed. + # + # ``object.__setattr__`` bypasses the frozen-dataclass + # ``__setattr__`` guard so we can add these attrs without + # mutating any declared dataclass field. + object.__setattr__(ref_instance, "__name__", function_name) + object.__setattr__(ref_instance, "__qualname__", fn.__qualname__) + object.__setattr__(ref_instance, "__module__", fn.__module__) + object.__setattr__(ref_instance, "__doc__", fn.__doc__) + object.__setattr__(ref_instance, "__wrapped__", fn) + object.__setattr__(ref_instance, "__signature__", inspect.signature(fn)) + # Forward annotations too -- some introspection paths read + # ``__annotations__`` directly rather than through signature. + object.__setattr__( + ref_instance, "__annotations__", dict(fn.__annotations__) + ) + + return ref_instance + + return decorator diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py new file mode 100644 index 0000000..c68d54e --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py @@ -0,0 +1,88 @@ +"""``TaskEnv`` — declare image + dependencies once, reuse across ``@task``s. + +A :class:`TaskEnv` bundles the container ``image`` and an optional +``dependencies_from`` file so a pipeline author can declare the execution +environment once and reference it from many ``@task`` components via +``@task(env=...)``. It is the Python equivalent of a ``local_from_python`` +YAML anchor. + +``TaskEnv`` is **authoring-only**. ``@task(env=...)`` expands it at +decoration time into the existing ``CallableRef._task_image`` / +``CallableRef._task_dependencies_from`` metadata. The compiler, hydrator, +and Oasis runner never see a ``TaskEnv`` object — no downstream component +learns the word ``env``. + +Example:: + + from pathlib import Path + from tangle_cli.python_pipeline import TaskEnv, task + + UPI = TaskEnv( + image="us-docker.pkg.dev/.../areas-ml-upi:main", + dependencies_from=Path(__file__).parent / "pyproject.toml", + ) + + @task(env=UPI) + def publish_to_comet(...): + # docstring carries: Metadata / Name: [UPI Clustering] Publish To Comet + ... + +The component name comes from the function's docstring ``Metadata: Name:`` +block (auto-derived from the function name if absent); the pipeline block name +(task id) comes from the call-site variable name, or ``.named("Block Name")`` +for an explicit label. +""" +from __future__ import annotations + +import inspect +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class TaskEnv: + """Reusable execution environment for ``@task`` components. + + Bundles the container image and optional dependencies file so a + pipeline author can declare them once and reference the env from many + tasks. This is the Python equivalent of a ``local_from_python`` YAML + anchor. + + Attributes: + image: Container image for the component. Required — naming the + image once is the main point of ``TaskEnv``. + dependencies_from: Optional path to a ``pyproject.toml`` (or any + file ``tangle-deploy`` understands) declaring pip + dependencies. A relative path is resolved at the ``TaskEnv`` + *definition site* (the module where ``TaskEnv(...)`` is + written), so a shared ``_envs.py`` resolves intuitively. + Authors can pass an absolute ``Path`` to avoid frame-based + ambiguity. When omitted, the existing hydrator/generator + dependency discovery still applies. + """ + + image: str + dependencies_from: str | Path | None = None + + def __post_init__(self) -> None: + if not isinstance(self.image, str) or not self.image: + raise ValueError("TaskEnv.image must be a non-empty string") + if self.dependencies_from is None: + return + + p = Path(self.dependencies_from) + if not p.is_absolute(): + # Resolve a relative ``dependencies_from`` at the TaskEnv + # DEFINITION SITE. Walk frames: __post_init__ -> generated + # dataclass __init__ -> the caller that wrote ``TaskEnv(...)``. + frame = inspect.currentframe() + caller = ( + frame.f_back.f_back + if frame and frame.f_back and frame.f_back.f_back + else None + ) + filename = caller.f_globals.get("__file__") if caller else None + caller_dir = Path(filename).resolve().parent if filename else Path.cwd() + p = caller_dir / p + # Frozen dataclass: bypass __setattr__ to store the resolved Path. + object.__setattr__(self, "dependencies_from", p.resolve()) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py new file mode 100644 index 0000000..843cca4 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py @@ -0,0 +1,369 @@ +"""The tracer: runs a ``@pipeline`` body inside a fresh GraphBuilder. + +Public API: + +* :func:`current_builder` — returns the currently active GraphBuilder + (or ``None`` outside a trace). ``CallableRef.__call__`` consults this + to attach new tasks. + +* :func:`trace_pipeline` — the compile driver entry point. Builds a + fresh builder, binds ``cfg`` and ``In[T]`` parameters, sets up the + AST pre-pass for LHS-name auto-IDs, calls the user's pipeline + function, captures the ``Out[T]`` return slot, and returns the + populated builder. + +Trace mechanics for task IDs: + +* At ``trace_pipeline`` start we ``inspect.getsource(fn)`` and build a + ``{absolute_lineno: lhs_name}`` map by walking ``ast.Assign`` nodes + whose target is a single ``Name``. +* At trace time, ``CallableRef.__call__`` reads + ``sys._getframe(1).f_lineno`` and consults the map. If the line has + no clean Name-LHS, the call must have been wrapped in ``.named(...)`` + or we raise :class:`AmbiguousTaskIdError`. +""" +from __future__ import annotations + +import ast +import dataclasses +import inspect +import textwrap +import typing +from contextvars import ContextVar +from typing import TYPE_CHECKING, Any + +from .errors import CompileError +from .graph import EdgeRef, GraphBuilder +from .placeholders import GraphInputPlaceholder, TaskOutputProxy +from .types import In, Out, Outputs + +if TYPE_CHECKING: # pragma: no cover + from .pipeline import PipelineFn + + +_BUILDER: ContextVar["GraphBuilder | None"] = ContextVar("_BUILDER", default=None) + + +def current_builder() -> "GraphBuilder | None": + """Return the active trace builder, or ``None`` outside a trace.""" + return _BUILDER.get() + + +def _build_lineno_to_lhs(fn: Any) -> dict[int, str]: + """Walk ``fn``'s source and map each absolute line number containing + a single-Name assignment to that name. + + Returns ``{}`` for pipelines defined in places where source isn't + available (e.g. exec'd strings without ``inspect`` support); callers + must always handle the ``.get(lineno) -> None`` case. + """ + try: + src_lines, start_lineno = inspect.getsourcelines(fn) + except (OSError, TypeError): + return {} + src = textwrap.dedent("".join(src_lines)) + try: + tree = ast.parse(src) + except SyntaxError: # pragma: no cover — defensive + return {} + + out: dict[int, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + # node.lineno is 1-based in the dedented source. We + # want absolute file line numbers to match + # sys._getframe.f_lineno in the trace, which reports the + # line of the inner call (often the LAST line of the + # RHS for multi-line expressions). Map EVERY line + # spanned by the assignment to the LHS name so the call + # can be on any of them. + end_lineno = getattr(node, "end_lineno", node.lineno) or node.lineno + for ln in range(node.lineno, end_lineno + 1): + out[ln + start_lineno - 1] = target.id + return out + + +def _is_in_annotation(annotation: Any) -> bool: + """True when ``annotation`` is ``In[T]`` (PEP 484/585 generic alias).""" + return getattr(annotation, "__origin__", None) is In + + +def _is_out_annotation(annotation: Any) -> bool: + return getattr(annotation, "__origin__", None) is Out + + +def _is_outputs_class(annotation: Any) -> bool: + """True when ``annotation`` is a STRICT subclass of :class:`Outputs` + (the multi-output typed-object marker). The bare ``Outputs`` base is + not itself a valid return annotation.""" + return ( + isinstance(annotation, type) + and issubclass(annotation, Outputs) + and annotation is not Outputs + ) + + +def _outputs_fields(outputs_cls: type) -> dict[str, Any]: + """Return ordered ``{field_name: Out[T] inner type | None}`` for an + :class:`Outputs` subclass. + + Field order follows dataclass declaration order (or ``__annotations__`` + order for a non-dataclass). The inner type is the ``T`` of an ``Out[T]`` + field annotation, or ``None`` when the field is not annotated ``Out[T]`` + (the caller validates and rejects that case at trace time). + """ + try: + hints = typing.get_type_hints(outputs_cls, include_extras=True) + except Exception: + hints = dict(getattr(outputs_cls, "__annotations__", {})) + if dataclasses.is_dataclass(outputs_cls): + names = [f.name for f in dataclasses.fields(outputs_cls)] + else: + names = list(getattr(outputs_cls, "__annotations__", {}).keys()) + result: dict[str, Any] = {} + for name in names: + anno = hints.get(name) + result[name] = _annotation_inner_type(anno) if _is_out_annotation(anno) else None + return result + + +def _annotation_inner_type(annotation: Any) -> Any: + """Extract ``T`` from ``In[T]`` / ``Out[T]``.""" + args = getattr(annotation, "__args__", ()) + return args[0] if args else None + + +def declared_output_names(pipeline_fn: "PipelineFn") -> tuple[str, ...] | None: + """Best-effort static list of a child pipeline's declared output names. + + Derived from the child ``PipelineFn``'s return annotation at PARENT + trace time so ``subpipeline(child)(...)`` can return a strict + :class:`SubpipelineOutputProxy` BEFORE the child is compiled: + + * single ``-> Out[T]`` -> ``(pipeline_fn.output_name,)``; + * an ``Outputs`` subclass -> its ``Out[T]`` field names, in field order; + * anything else / unresolvable -> ``None`` (permissive fallback). + + The compiled child sidecar's ``outputs`` block remains the authoritative + source for the compiler's cross-file validation; this is the trace-time + view used purely for proxy ergonomics. + """ + try: + hints = typing.get_type_hints(pipeline_fn.fn, include_extras=True) + return_anno = hints.get( + "return", inspect.signature(pipeline_fn.fn).return_annotation + ) + except Exception: + return None + if return_anno is inspect.Signature.empty: + return None + if _is_out_annotation(return_anno): + return (pipeline_fn.output_name,) + if _is_outputs_class(return_anno): + return tuple(_outputs_fields(return_anno).keys()) + return None + + +def _python_type_to_tangle_type(t: Any) -> str: + """Map ``str``/``int``/etc. to Tangle type strings.""" + if t is str: + return "String" + if t is int: + return "Integer" + if t is float: + return "Float" + if t is bool: + return "Boolean" + # Fallback: stringify whatever it is. Tangle tolerates arbitrary + # type strings so long as the consumer recognises them. + return getattr(t, "__name__", str(t)) + + +def trace_pipeline( + pipeline_fn: "PipelineFn", + cfg: Any, + inputs: dict[str, Any] | None = None, +) -> GraphBuilder: + """Trace ``pipeline_fn`` against ``cfg`` (+ optional runtime ``inputs``). + + Returns the populated :class:`GraphBuilder` ready for emit. + + The function's parameters are inspected: + - ``cfg`` (no ``In`` annotation, named exactly ``cfg``) → bound to + the loaded :class:`Cfg` object. + - ``In[T]`` parameters → :class:`GraphInputPlaceholder` (no default) + or use the user-provided ``inputs`` dict / the parameter default. + Each becomes an entry in ``builder.inputs`` with type info. + - Any other parameter → :class:`CompileError`. + + The return annotation is inspected: + - ``Out[T]`` → ``builder.outputs`` gets a single entry, and the + return value (which must be a :class:`TaskOutputProxy`) is wired + into ``builder.output_taskref`` via an :class:`EdgeRef`. + - No annotation / ``None`` → no outputs block. + """ + inputs = inputs or {} + + builder = GraphBuilder( + name=pipeline_fn.name, + description=pipeline_fn.description, + annotations=dict(pipeline_fn.annotations), + ) + # AST pre-pass: needed by CallableRef.__call__ to derive task IDs + # from LHS variable names. Stashed on the builder so the contextvar + # carries everything in one bag. + builder.lineno_to_lhs = _build_lineno_to_lhs(pipeline_fn.fn) + + sig = inspect.signature(pipeline_fn.fn) + # Resolve PEP 563 string annotations to runtime values. This must + # use the user fn's own globals so In/Out/etc. resolve correctly + # when the user has ``from __future__ import annotations``. + try: + resolved_hints = typing.get_type_hints(pipeline_fn.fn, include_extras=True) + except Exception: + # Best effort: if name resolution fails (e.g. forward refs that + # can't be resolved at trace time), fall back to the raw + # annotation strings on each parameter. + resolved_hints = {} + call_kwargs: dict[str, Any] = {} + + for param_name, param in sig.parameters.items(): + annotation = resolved_hints.get(param_name, param.annotation) + + if param_name == "cfg" and not _is_in_annotation(annotation): + call_kwargs[param_name] = cfg + continue + + if _is_in_annotation(annotation): + inner = _annotation_inner_type(annotation) + type_str = _python_type_to_tangle_type(inner) + entry: dict[str, Any] = {"name": param_name, "type": type_str} + + has_default = param.default is not inspect.Parameter.empty + if has_default: + entry["default"] = param.default + entry["optional"] = True + builder.inputs.append(entry) + + # Bind a placeholder for the user fn body. If the caller + # supplied a runtime input, prefer that; otherwise the + # placeholder carries the input name so emit can render it. + if param_name in inputs: + call_kwargs[param_name] = inputs[param_name] + else: + call_kwargs[param_name] = GraphInputPlaceholder(input_name=param_name) + continue + + raise CompileError( + f"@pipeline parameter {param_name!r} is not annotated In[T] and is not " + f"named 'cfg'. Use In[T] for graph inputs, or name the parameter 'cfg' " + "for the loaded config object." + ) + + # Run the user's pipeline body inside a context where + # current_builder() returns this builder. + token = _BUILDER.set(builder) + try: + result = pipeline_fn.fn(**call_kwargs) + finally: + _BUILDER.reset(token) + + # Capture the declared output(s). ``get_type_hints`` returns the + # resolved return annotation under the ``return`` key. Two shapes: + # * single ``-> Out[T]`` -> one output (legacy path, unchanged); + # * ``-> `` -> one output per Out[T] field. + return_anno = resolved_hints.get("return", sig.return_annotation) + if return_anno is not inspect.Signature.empty and _is_out_annotation(return_anno): + if not isinstance(result, TaskOutputProxy): + raise CompileError( + "@pipeline declared -> Out[T] but the function did not return a " + "TaskOutputProxy. Return the result of the final task call (e.g. " + "`return publish_to_comet`)." + ) + inner = _annotation_inner_type(return_anno) + type_str = _python_type_to_tangle_type(inner) + output_name = pipeline_fn.output_name + builder.outputs.append({"name": output_name, "type": type_str}) + builder.output_name = output_name + builder.output_taskref = EdgeRef( + kind="taskOutput", + task_id=result._task_id, + output=result._resolved_output_name(), + ) + elif return_anno is not inspect.Signature.empty and _is_outputs_class(return_anno): + _trace_multi_output(builder, pipeline_fn, return_anno, result) + + return builder + + +def _trace_multi_output( + builder: GraphBuilder, + pipeline_fn: "PipelineFn", + outputs_cls: type, + result: Any, +) -> None: + """Populate ``builder`` for a pipeline returning an :class:`Outputs` + typed object. + + Emits one ``builder.outputs`` entry and one ``builder.output_values`` + edge per ``Out[T]`` field, in field declaration order (deterministic). + Each field value must be a :class:`TaskOutputProxy` or + :class:`GraphInputPlaceholder` (v1 forbids constant graph outputs). + """ + # ``@pipeline(output_name=...)`` applies only to the single-output path. + if pipeline_fn.output_name != "wait_for_output": + raise CompileError( + "@pipeline(output_name=...) cannot be combined with an Outputs " + f"return annotation ({outputs_cls.__name__}); output names come " + f"from the Outputs fields. Remove output_name=" + f"{pipeline_fn.output_name!r}." + ) + if not isinstance(result, outputs_cls): + raise CompileError( + f"@pipeline declared -> {outputs_cls.__name__} but the function " + f"returned a {type(result).__name__}. Return an instance, e.g. " + f"`return {outputs_cls.__name__}(...)`." + ) + fields = _outputs_fields(outputs_cls) + if not fields: + raise CompileError( + f"Outputs subclass {outputs_cls.__name__} declares no fields; add " + "at least one Out[T]-annotated field." + ) + for field_name, inner in fields.items(): + if inner is None: + raise CompileError( + f"Outputs field {field_name!r} on {outputs_cls.__name__} must " + "be annotated Out[T] (e.g. `rows_written: Out[str]`)." + ) + value = getattr(result, field_name) + if isinstance(value, TaskOutputProxy): + edge = EdgeRef( + kind="taskOutput", + task_id=value._task_id, + output=value._resolved_output_name(), + ) + elif isinstance(value, GraphInputPlaceholder): + edge = EdgeRef(kind="graphInput", input_name=value.input_name) + else: + raise CompileError( + f"Outputs field {field_name!r} must be wired to a task output " + "or a graph input (a TaskOutputProxy or In[...] value), got " + f"{type(value).__name__}. v1 does not support constant graph " + "outputs." + ) + type_str = _python_type_to_tangle_type(inner) + builder.outputs.append({"name": field_name, "type": type_str}) + builder.output_values[field_name] = edge + + +# Re-export commonly-used names. +__all__ = ["current_builder", "trace_pipeline"] + + +# Defensive: silence unused-import warnings for ``typing`` in some +# linters when we don't actually call typing.* — kept for symmetry with +# the other modules and to make adding generic helpers cheap. +_ = typing diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py new file mode 100644 index 0000000..889fe8c --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py @@ -0,0 +1,69 @@ +"""Annotation markers for graph-level inputs and outputs. + +``In[T]`` marks a parameter of a ``@pipeline`` function as a graphInput +(the pipeline's runtime parameter). ``Out[T]`` marks the return type +slot, which the framework binds to ``outputValues.``. + +These are pure annotation markers — no runtime behavior. The +``@pipeline`` decorator inspects ``fn.__annotations__`` to build the +``inputs:`` / ``outputs:`` blocks. +""" +from __future__ import annotations + +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class In(Generic[T]): + """Mark a pipeline parameter as a graph input. + + Examples: + def my_pipeline(parent_wait_token: In[str], cfg): ... + def my_pipeline(threshold: In[int] = 5, cfg): ... + """ + + +class Out(Generic[T]): + """Mark a pipeline return slot as a graph output. + + Example: + def my_pipeline(cfg) -> Out[str]: ... + """ + + +class Outputs: + """Base class for a typed MULTIPLE-output pipeline return object. + + A pipeline that exposes more than one named output declares a frozen + dataclass subclass of :class:`Outputs` whose fields are annotated + ``Out[T]``, and returns an instance of it:: + + from dataclasses import dataclass + from tangle_cli.python_pipeline import Out, Outputs, pipeline, ref + + @dataclass(frozen=True) + class JudgeOutputs(Outputs): + rows_written: Out[str] + run_id: Out[str] + + @pipeline(name="Judge Options") + def judge_options(input_table: In[str]) -> JudgeOutputs: + judge = ref(url="file://./judge.yaml")(input_table=input_table) + return JudgeOutputs( + rows_written=judge.rows_written, run_id=judge.run_id + ) + + The tracer recognises an :class:`Outputs` subclass return annotation, + reads its ``Out[T]`` fields (in declaration order), and emits one + top-level ``outputs`` entry plus one ``implementation.graph.outputValues`` + entry per field. Each field value must be a task-output handle or an + ``In[...]`` graph input (v1 does not support constant graph outputs). + + The existing single ``-> Out[T]`` path and ``@pipeline(output_name=...)`` + are unchanged; ``output_name`` may NOT be combined with an ``Outputs`` + return annotation (output names come from the fields). + + This is a pure marker base — it carries no runtime behavior; subclasses + are ordinary (frozen) dataclasses. + """ diff --git a/tests/test_component_from_func.py b/tests/test_component_from_func.py index a4141fe..4d53862 100644 --- a/tests/test_component_from_func.py +++ b/tests/test_component_from_func.py @@ -1,5 +1,6 @@ """Tests for the native component YAML generator (component_from_func).""" +import ast import inspect import json import subprocess @@ -16,10 +17,13 @@ InputPath, OutputPath, ParamInfo, + _AUTHORING_IMPORT_MODULES, _build_argparse_code, _build_args_section, _build_pip_install_command, _build_python_source, + _is_authoring_import, + _is_authoring_module, _python_name_to_component_name, _resolve_annotation, _resolve_return_type, @@ -1547,6 +1551,177 @@ def plain(x): assert _strip_authoring_constructs(source) == source +# ============================================================================ +# _strip_authoring_constructs — tangle_cli.python_pipeline surface +# ============================================================================ + + +class TestStripAuthoringConstructsTangleCli: + """The strip must recognise the OSS ``tangle_cli.python_pipeline`` path. + + ``tangle_deploy.python_pipeline`` re-exports the OSS objects, so an + author may import from EITHER module. Every case here mirrors a + ``tangle_deploy`` case above but through the OSS import path, so a + component baked on a thin image (without the authoring DSL) still + imports cleanly. + """ + + def test_strips_from_import_and_simple_decorator(self): + source = textwrap.dedent("""\ + from tangle_cli.python_pipeline import task + + @task(image="python:3.12") + def hello(out, who="world"): + with open(out, "w") as f: + f.write(who) + """) + result = _strip_authoring_constructs(source) + assert "from tangle_cli.python_pipeline" not in result + assert "@task" not in result + assert 'def hello(out, who="world"):' in result + assert "f.write(who)" in result + + def test_strips_dotted_decorator_form(self): + source = textwrap.dedent("""\ + import tangle_cli.python_pipeline as tp + + @tp.task(image="python:3.12") + def hello(out): + pass + """) + result = _strip_authoring_constructs(source) + assert "import tangle_cli.python_pipeline" not in result + assert "@tp.task" not in result + assert "def hello(out):" in result + + def test_strips_aliased_import(self): + source = textwrap.dedent("""\ + from tangle_cli.python_pipeline import ref as operation_by_ref + + x = 1 + """) + result = _strip_authoring_constructs(source) + assert "operation_by_ref" not in result + assert "from tangle_cli.python_pipeline" not in result + assert "x = 1" in result + + def test_strips_plain_import_of_python_pipeline(self): + source = textwrap.dedent("""\ + import tangle_cli.python_pipeline + + x = 1 + """) + result = _strip_authoring_constructs(source) + assert "import tangle_cli.python_pipeline" not in result + assert "x = 1" in result + + def test_preserves_non_authoring_tangle_cli_import(self): + # Only ``tangle_cli.python_pipeline`` is authoring. A genuine runtime + # helper from another ``tangle_cli.*`` package must survive. + source = textwrap.dedent("""\ + from tangle_cli.python_pipeline import task + from tangle_cli.utils import something + + @task(image="python:3.12") + def hello(out): + return something(out) + """) + result = _strip_authoring_constructs(source) + assert "from tangle_cli.python_pipeline import task" not in result + assert "@task" not in result + assert "from tangle_cli.utils import something" in result + assert "return something(out)" in result + + def test_multi_name_authoring_import_line_dropped(self): + source = textwrap.dedent("""\ + from tangle_cli.python_pipeline import In, Out, task + + @task(image="python:3.12") + def hello(out): + return out + """) + result = _strip_authoring_constructs(source) + assert "In" not in result + assert "Out" not in result + assert "from tangle_cli.python_pipeline" not in result + assert "@task" not in result + assert "def hello(out):" in result + + def test_strips_submodule_import(self): + # ``from tangle_cli.python_pipeline.x import y`` is a submodule of the + # authoring package and must also be dropped. + source = textwrap.dedent("""\ + from tangle_cli.python_pipeline.types import In, Out + + x = 1 + """) + result = _strip_authoring_constructs(source) + assert "from tangle_cli.python_pipeline.types" not in result + assert "x = 1" in result + + +# ============================================================================ +# _is_authoring_module / _is_authoring_import predicates +# ============================================================================ + + +class TestAuthoringImportPredicate: + """Unit tests for the authoring-module recognition helpers.""" + + def test_authoring_modules_constant_contents(self): + assert _AUTHORING_IMPORT_MODULES == ( + "tangle_deploy.python_pipeline", + "tangle_cli.python_pipeline", + ) + + @pytest.mark.parametrize( + "name", + [ + "tangle_deploy.python_pipeline", + "tangle_cli.python_pipeline", + "tangle_deploy.python_pipeline.types", + "tangle_cli.python_pipeline.task", + ], + ) + def test_is_authoring_module_true(self, name): + assert _is_authoring_module(name) is True + + @pytest.mark.parametrize( + "name", + [ + "tangle_cli.utils", + "tangle_deploy.utils", + "tangle_cli", + "tangle_deploy", + "tangle_cli.python_pipelinex", # not a submodule boundary + "os", + "", + ], + ) + def test_is_authoring_module_false(self, name): + assert _is_authoring_module(name) is False + + def test_is_authoring_import_from_both_paths(self): + for mod in ("tangle_deploy.python_pipeline", "tangle_cli.python_pipeline"): + node = ast.parse(f"from {mod} import task").body[0] + assert _is_authoring_import(node) is True + + def test_is_authoring_import_plain_import_both_paths(self): + for mod in ("tangle_deploy.python_pipeline", "tangle_cli.python_pipeline"): + node = ast.parse(f"import {mod}").body[0] + assert _is_authoring_import(node) is True + + def test_is_authoring_import_rejects_relative_import(self): + # A relative ``from . import x`` is never the authoring package. + node = ast.parse("from . import sibling").body[0] + assert _is_authoring_import(node) is False + + def test_is_authoring_import_rejects_non_authoring(self): + for src in ("import os", "from tangle_cli.utils import x", "x = 1"): + node = ast.parse(src).body[0] + assert _is_authoring_import(node) is False + + # ============================================================================ # _strip_authoring_constructs — TaskEnv env-only hardening (Phase 3, §3.5) # ============================================================================ diff --git a/tests/test_python_pipeline_dsl.py b/tests/test_python_pipeline_dsl.py new file mode 100644 index 0000000..49d28f7 --- /dev/null +++ b/tests/test_python_pipeline_dsl.py @@ -0,0 +1,210 @@ +"""Tests for the ``tangle_cli.python_pipeline`` authoring DSL surface. + +These lock the public import surface, the small pure helpers, the error +hierarchy, and the ``raw()`` contract so the DSL stays a stable, fully +OSS-native authoring layer (no ``tangle_deploy`` references) as the +compiler is migrated on top of it. +""" + +from __future__ import annotations + +import importlib +import sys +from dataclasses import dataclass +from pathlib import Path + +import pytest + +import tangle_cli.python_pipeline as pp +from tangle_cli.python_pipeline import ( + In, + Out, + Outputs, + TaskEnv, + pipeline, + raw, + ref, + registered, + subpipeline, + task, +) +from tangle_cli.python_pipeline.errors import ( + AmbiguousTaskIdError, + CompileError, + InvalidArgumentTypeError, + MissingRequiredInputError, + UnknownCfgKeyError, +) +from tangle_cli.python_pipeline.ids import snake_to_title_case +from tangle_cli.python_pipeline.raw import Raw + +_DSL_DIR = Path(pp.__file__).parent + +# ============================================================================ +# Public import surface +# ============================================================================ + + +class TestPublicSurface: + """The documented authoring entry points and ``__all__``.""" + + def test_all_names_are_exported(self): + assert set(pp.__all__) == { + "pipeline", + "task", + "registered", + "ref", + "raw", + "subpipeline", + "TaskEnv", + "In", + "Out", + "Outputs", + } + + def test_every_all_name_is_present_on_module(self): + for name in pp.__all__: + assert hasattr(pp, name), f"missing export: {name}" + + def test_from_import_yields_identical_objects(self): + # Identity matters: the compiler dispatches on ``isinstance`` against + # the classes these decorators produce, so ``from ... import`` and + # attribute access must resolve to the SAME objects (the linchpin for + # the tangle_deploy re-export shim in a later phase). + assert pipeline is pp.pipeline + assert task is pp.task + assert registered is pp.registered + assert ref is pp.ref + assert raw is pp.raw + assert subpipeline is pp.subpipeline + assert TaskEnv is pp.TaskEnv + assert In is pp.In + assert Out is pp.Out + assert Outputs is pp.Outputs + + def test_cfg_is_not_a_top_level_export(self): + # ``cfg`` is injected by the framework at trace time, never imported. + assert "cfg" not in pp.__all__ + assert not hasattr(pp, "cfg") or "cfg" not in set(pp.__all__) + + def test_import_is_light(self): + # The package docstring promises importing it does not eagerly pull in + # the heavy codegen module. Re-import in a clean module state and check + # the codegen module was not dragged in transitively. + for mod in list(sys.modules): + if mod == "tangle_cli.component_generator": + del sys.modules[mod] + importlib.reload(importlib.import_module("tangle_cli.python_pipeline")) + assert "tangle_cli.component_generator" not in sys.modules + + +class TestNoInternalReferences: + """The vendored DSL must be free of ``tangle_deploy`` references.""" + + def test_no_tangle_deploy_references_in_source(self): + offenders = [] + for path in _DSL_DIR.glob("*.py"): + text = path.read_text(encoding="utf-8") + if "tangle_deploy" in text: + offenders.append(path.name) + assert offenders == [], f"tangle_deploy referenced in: {offenders}" + + +# ============================================================================ +# ids.snake_to_title_case +# ============================================================================ + + +class TestSnakeToTitleCase: + @pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("build_quality_tables", "Build Quality Tables"), + ("foo", "Foo"), + ("a__b", "A B"), # empty segments collapse + ("", ""), + ("GPU", "Gpu"), # str.capitalize lowercases trailing letters + ("already_Title", "Already Title"), + ], + ) + def test_conversion(self, raw_name, expected): + assert snake_to_title_case(raw_name) == expected + + +# ============================================================================ +# errors hierarchy +# ============================================================================ + + +class TestErrorHierarchy: + @pytest.mark.parametrize( + "exc_cls", + [ + UnknownCfgKeyError, + MissingRequiredInputError, + AmbiguousTaskIdError, + InvalidArgumentTypeError, + ], + ) + def test_all_authoring_errors_subclass_compile_error(self, exc_cls): + assert issubclass(exc_cls, CompileError) + assert issubclass(exc_cls, Exception) + + def test_compile_error_is_catchable(self): + with pytest.raises(CompileError): + raise UnknownCfgKeyError("nope") + + +# ============================================================================ +# types markers (In / Out / Outputs) +# ============================================================================ + + +class TestTypeMarkers: + def test_in_and_out_are_subscriptable(self): + # Pure annotation markers — subscripting must not raise. + assert In[str] is not None + assert Out[int] is not None + + def test_outputs_is_usable_as_dataclass_base(self): + @dataclass(frozen=True) + class JudgeOutputs(Outputs): + rows_written: Out[str] + run_id: Out[str] + + inst = JudgeOutputs(rows_written="a", run_id="b") + assert inst.rows_written == "a" + assert inst.run_id == "b" + assert isinstance(inst, Outputs) + + +# ============================================================================ +# raw() contract +# ============================================================================ + + +class TestRaw: + def test_wraps_string_as_raw(self): + r = raw("SELECT * FROM `{{input_1}}`") + assert isinstance(r, Raw) + assert r.value == "SELECT * FROM `{{input_1}}`" + + def test_equality_and_hash(self): + assert raw("{{x}}") == raw("{{x}}") + assert hash(raw("{{x}}")) == hash(raw("{{x}}")) + assert raw("{{x}}") != raw("{{y}}") + + def test_rejects_non_string(self): + with pytest.raises(TypeError): + raw(123) # type: ignore[arg-type] + + @pytest.mark.parametrize("value", ["{% if x %}", "{# comment #}"]) + def test_rejects_jinja_tokens(self, value): + # Only ``{{...}}`` runtime sentinels are legitimate; Jinja + # statement/comment tokens are compile-time and must be rejected. + with pytest.raises(ValueError): + raw(value) + + def test_allows_plain_runtime_sentinel(self): + # A bare ``{{...}}`` with no Jinja statement/comment token is fine. + assert raw("{{input_1}}").value == "{{input_1}}" From aff19eb40a665cf7fc776b89be73a0ab73e6ef4e Mon Sep 17 00:00:00 2001 From: Silin Gupta Date: Thu, 9 Jul 2026 15:22:42 -0400 Subject: [PATCH 2/3] Address review: drop dead edge_kw, stale typing shim, fix In[] example, genericize tangle-deploy refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - graph.py: remove unused TaskNode.edge_kw field + its docstring paragraph; wait_for/depends_on travel through `arguments` and the emitter never read it - trace.py: remove stale `_ = typing` guard — typing.get_type_hints is called above, so the import is used and the "we don't call typing.*" comment was wrong - types.py: fix invalid In[] docstring example (a non-default param followed a defaulted one -> SyntaxError); reorder to `def my_pipeline(cfg, threshold=5)` - python_pipeline/*: genericize internal `tangle-deploy` references to the distribution-agnostic OSS surface ("the hydrator", `tangle sdk pipelines compile`); tighten the no-internal-references guard to catch both the `tangle_deploy` (import) and hyphenated `tangle-deploy` (product) spellings Addresses Volv's review comments on PR #20. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tangle-cli/src/tangle_cli/python_pipeline/errors.py | 2 +- .../tangle-cli/src/tangle_cli/python_pipeline/graph.py | 7 +------ packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py | 4 ++-- .../src/tangle_cli/python_pipeline/registered.py | 2 +- .../src/tangle_cli/python_pipeline/subpipeline.py | 2 +- .../tangle-cli/src/tangle_cli/python_pipeline/task.py | 8 ++++---- .../tangle-cli/src/tangle_cli/python_pipeline/task_env.py | 2 +- .../tangle-cli/src/tangle_cli/python_pipeline/trace.py | 6 ------ .../tangle-cli/src/tangle_cli/python_pipeline/types.py | 2 +- tests/test_python_pipeline_dsl.py | 8 +++++--- 10 files changed, 17 insertions(+), 26 deletions(-) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py index 189d225..19b684f 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/errors.py @@ -1,7 +1,7 @@ """Compile-time error hierarchy. All errors raised by the Python pipeline authoring layer before/during -``tangle-deploy pipeline compile from-python`` are subclasses of :class:`CompileError`. +``tangle sdk pipelines compile`` are subclasses of :class:`CompileError`. They should include enough context for a user to fix the underlying issue without reading framework source: file:line of the offending call, the relevant primitive name, and a suggested remedy. diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py index ad50fd2..4c36162 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/graph.py @@ -31,10 +31,6 @@ class TaskNode: ``arguments`` values may be plain strings, TaskOutputProxy objects (for taskOutput edges in non-``wait_for`` argument positions — not used in the PoC), or GraphInputPlaceholder objects. - - ``edge_kw`` is the separated bucket for ``wait_for`` / ``depends_on`` - kwargs — emitted with their special edge value shape rather than as - a normal argument. """ task_id: str @@ -43,7 +39,6 @@ class TaskNode: ref_digest: str | None = None arguments: dict[str, Any] = field(default_factory=dict) annotations: dict[str, str] | None = None - edge_kw: dict[str, EdgeRef] = field(default_factory=dict) @dataclass @@ -77,7 +72,7 @@ class GraphBuilder: # ``local_from_python:`` entry per unique source file, and (b) # rewrite each task's ``componentRef.url`` to # ``resolve://./.components.yaml#`` so hydrate - # uses tangle-deploy's own ``local_from_python`` resolver. + # uses the hydrator's own ``local_from_python`` resolver. task_refs_for_local_from_python: list[tuple[str, Any]] = field( default_factory=list ) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py index 4f540ad..3e1d3c4 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py @@ -199,8 +199,8 @@ def __call__(self, **kwargs: Any) -> Any: raise RuntimeError( "CallableRef.__call__ requires an active @pipeline trace " "context. Either call this inside a function decorated " - "with @pipeline, or use the `tangle-deploy pipeline compile " - "from-python` driver." + "with @pipeline, or compile the script with " + "`tangle sdk pipelines compile`." ) # Resolve the task ID. ``.named(...)`` always wins over the diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py index fdd58bd..7522396 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/registered.py @@ -98,7 +98,7 @@ def registered( :class:`CallableRef`. The compile driver rewrites the task's componentRef URL to ``resolve:///gen_config.yaml#``, computed relative to the compiled artifact's output directory. Hydrate - then uses tangle-deploy's own ``resolve://`` resolver to read the + then uses the hydrator's own ``resolve://`` resolver to read the gen_config.yaml fragment and inline the component spec. Unlike ``@task``, ``@registered`` references an EXISTING diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py index 6122a2a..549082a 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/subpipeline.py @@ -134,7 +134,7 @@ def __call__(self, **kwargs: Any) -> "TaskOutputProxy": raise RuntimeError( "subpipeline(child)(...) requires an active @pipeline trace " "context. Either call this inside a function decorated with " - "@pipeline, or use the `tangle-deploy pipeline compile from-python` driver." + "@pipeline, or compile the script with `tangle sdk pipelines compile`." ) # Resolve the parent task ID. ``.named(...)`` always wins over the diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py index 632f269..00594f3 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py @@ -9,7 +9,7 @@ ``.components.yaml`` with one ``local_from_python:`` entry per unique source file, and rewrites each task's componentRef URL to ``resolve://./.components.yaml#``. Hydrate then uses -tangle-deploy's own ``local_from_python`` resolver to call +the hydrator's own ``local_from_python`` resolver to call ``regenerate_yaml`` at hydrate time -- no pre-codegen step on our side. Crucially, the decorator does NOT call the user function. @@ -74,7 +74,7 @@ def task( the decorator captures metadata onto a :class:`CallableRef`. The compile driver emits a sibling ``.components.yaml`` with a ``local_from_python:`` entry for the function and rewrites the - task's componentRef URL to point at it. Hydrate uses tangle-deploy's + task's componentRef URL to point at it. Hydrate uses the hydrator's own resolver to regenerate the component YAML at hydrate time. Args: @@ -94,7 +94,7 @@ def task( ``components.yaml#local_from_python.image`` field. Overrides ``env.image`` when both are given. dependencies_from: Path to a ``pyproject.toml`` (or any file - ``tangle-deploy`` understands) that declares pip + the hydrator understands) that declares pip dependencies. Resolved relative to the caller's source file when given as a string. Emitted into ``components.yaml#local_from_python.dependencies_from``. @@ -148,7 +148,7 @@ def decorator(fn: Callable[..., Any]) -> CallableRef: # Capture the absolute path of the source file the user wrote # the function in. ``inspect.getfile`` raises TypeError for # builtins / dynamically-built functions; the @task path - # requires a real on-disk file because tangle-deploy reads + # requires a real on-disk file because the hydrator reads # the source via inspect.getfile too. try: source_path = Path(inspect.getfile(fn)).resolve() diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py index c68d54e..ce7d3df 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py @@ -52,7 +52,7 @@ class TaskEnv: image: Container image for the component. Required — naming the image once is the main point of ``TaskEnv``. dependencies_from: Optional path to a ``pyproject.toml`` (or any - file ``tangle-deploy`` understands) declaring pip + file the hydrator understands) declaring pip dependencies. A relative path is resolved at the ``TaskEnv`` *definition site* (the module where ``TaskEnv(...)`` is written), so a shared ``_envs.py`` resolves intuitively. diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py index 843cca4..5e871bd 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py @@ -361,9 +361,3 @@ def _trace_multi_output( # Re-export commonly-used names. __all__ = ["current_builder", "trace_pipeline"] - - -# Defensive: silence unused-import warnings for ``typing`` in some -# linters when we don't actually call typing.* — kept for symmetry with -# the other modules and to make adding generic helpers cheap. -_ = typing diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py index 889fe8c..52378dc 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/types.py @@ -20,7 +20,7 @@ class In(Generic[T]): Examples: def my_pipeline(parent_wait_token: In[str], cfg): ... - def my_pipeline(threshold: In[int] = 5, cfg): ... + def my_pipeline(cfg, threshold: In[int] = 5): ... """ diff --git a/tests/test_python_pipeline_dsl.py b/tests/test_python_pipeline_dsl.py index 49d28f7..0b8f959 100644 --- a/tests/test_python_pipeline_dsl.py +++ b/tests/test_python_pipeline_dsl.py @@ -99,15 +99,17 @@ def test_import_is_light(self): class TestNoInternalReferences: - """The vendored DSL must be free of ``tangle_deploy`` references.""" + """The vendored DSL must be free of internal ``tangle-deploy`` + references, in either the ``tangle_deploy`` (module/import) or the + ``tangle-deploy`` (product/command) spelling.""" def test_no_tangle_deploy_references_in_source(self): offenders = [] for path in _DSL_DIR.glob("*.py"): text = path.read_text(encoding="utf-8") - if "tangle_deploy" in text: + if "tangle_deploy" in text or "tangle-deploy" in text: offenders.append(path.name) - assert offenders == [], f"tangle_deploy referenced in: {offenders}" + assert offenders == [], f"tangle-deploy referenced in: {offenders}" # ============================================================================ From 5ab880e8c4eec174983994ad00ed53c52b8249cd Mon Sep 17 00:00:00 2001 From: Silin Gupta Date: Thu, 9 Jul 2026 17:04:51 -0400 Subject: [PATCH 3/3] Invert authoring-module coupling: OSS registers, downstream contributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Volv's review: OSS codegen must not hardcode the downstream package name. ``_AUTHORING_IMPORT_MODULES`` was a frozen tuple that baked in ``tangle_deploy.python_pipeline`` alongside the canonical ``tangle_cli.python_pipeline`` path, so OSS knew about its own consumer — the dependency pointed outward. Replace it with an inward-pointing registry seam mirroring the resolver/reader registries in ``pipeline_hydrator``: - ``_AUTHORING_IMPORT_MODULES`` is now a mutable list seeded with ONLY the canonical ``tangle_cli.python_pipeline`` surface. - ``register_authoring_import_module(module)`` (idempotent) lets a downstream package contribute its own re-export path at import time. - ``authoring_import_modules()`` returns the current tuple snapshot. Delete the OSS-side ``_ensure_tangle_deploy_authoring_shim`` (and its now orphaned ``_identity_decorator`` / ``_AuthoringGeneric`` helpers): OSS no longer fabricates a fake downstream authoring module. A downstream package makes its module importable itself and registers the path. All remaining ``tangle_deploy`` / ``tangle-deploy`` references in ``component_from_func.py`` are genericized, and ``TestNoInternalReferences`` now scans that module too. Tests: the downstream authoring surface (registration + a stand-in ``sys.modules`` module) moves to a suite-wide autouse fixture in ``tests/conftest.py``, standing in for the downstream package. The old constant-contents assertion becomes proper API tests (OSS seed, register, idempotency, immutable snapshot). 674 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/tangle_cli/component_from_func.py | 125 +++++++++--------- tests/conftest.py | 97 ++++++++++++++ tests/test_component_from_func.py | 44 +++++- tests/test_python_pipeline_dsl.py | 15 ++- 4 files changed, 205 insertions(+), 76 deletions(-) create mode 100644 tests/conftest.py diff --git a/packages/tangle-cli/src/tangle_cli/component_from_func.py b/packages/tangle-cli/src/tangle_cli/component_from_func.py index ca9cf02..ee8f638 100644 --- a/packages/tangle-cli/src/tangle_cli/component_from_func.py +++ b/packages/tangle-cli/src/tangle_cli/component_from_func.py @@ -181,11 +181,18 @@ def all_outputs(self) -> list[ParamInfo]: def _ensure_cloud_pipelines_shim() -> None: - """Register import-time shims used while introspecting authoring files. - - This allows loading Python files that use `from cloud_pipelines import components` - and/or TD authoring decorators without requiring those authoring packages. - The TD authoring constructs are stripped from generated runtime code later. + """Register the import-time ``cloud_pipelines`` shim used while introspecting + authoring files. + + This lets us load Python files that use ``from cloud_pipelines import + components`` without requiring that authoring package to be installed; the + authoring constructs are stripped from the generated runtime code later. + + OSS deliberately does NOT fabricate a shim for any *downstream* authoring + surface (e.g. a module a downstream package exposes to re-export the + authoring objects under its own import path). A downstream package that + wants its own authoring path recognised both makes that module importable + itself and registers it via :func:`register_authoring_import_module`. """ if "cloud_pipelines" not in sys.modules: components_mod = types.ModuleType("cloud_pipelines.components") @@ -198,41 +205,6 @@ def _ensure_cloud_pipelines_shim() -> None: sys.modules["cloud_pipelines"] = cloud_pipelines_mod sys.modules["cloud_pipelines.components"] = components_mod - _ensure_tangle_deploy_authoring_shim() - - -def _identity_decorator(*args, **kwargs): - def decorate(func): - return func - - return decorate - - -class _AuthoringGeneric: - def __class_getitem__(cls, item): - return cls - - def __init__(self, *args, **kwargs): - pass - - -def _ensure_tangle_deploy_authoring_shim() -> None: - """Register a tiny shim for TD pipeline authoring imports if absent.""" - if "tangle_deploy.python_pipeline" in sys.modules: - return - - tangle_deploy_mod = sys.modules.get("tangle_deploy") or types.ModuleType("tangle_deploy") - python_pipeline_mod = types.ModuleType("tangle_deploy.python_pipeline") - for name in ("task", "pipeline", "subpipeline", "registered"): - setattr(python_pipeline_mod, name, _identity_decorator) - for name in ("In", "Out", "Outputs", "TaskEnv"): - setattr(python_pipeline_mod, name, _AuthoringGeneric) - setattr(python_pipeline_mod, "ref", lambda *args, **kwargs: None) - - setattr(tangle_deploy_mod, "python_pipeline", python_pipeline_mod) - sys.modules.setdefault("tangle_deploy", tangle_deploy_mod) - sys.modules["tangle_deploy.python_pipeline"] = python_pipeline_mod - def load_python_module(file_path: Path, extra_sys_path: list[Path] | None = None) -> Any: """Dynamically import a Python module from a file path. @@ -759,27 +731,45 @@ def _is_main_str(n: ast.expr) -> bool: # The python-pipeline authoring modules. ONLY imports of these modules (and # their submodules) are authoring-only and stripped from the baked source. We -# deliberately do NOT strip other ``tangle_deploy.*`` / ``tangle_cli.*`` packages -# (e.g. ``tangle_deploy.utils``): those may be legitimate runtime helpers used +# deliberately do NOT strip other packages that merely share a top-level name +# (e.g. a downstream ``*.utils``): those may be legitimate runtime helpers used # inside a ``@task`` body, and dropping them would raise ``NameError`` in the # operation container. # -# Two authoring surfaces are recognised: the legacy ``tangle_deploy.python_pipeline`` -# path (used by every pipeline authored before the OSS move) and the canonical -# OSS ``tangle_cli.python_pipeline`` path. Both expose the identical decorators — -# ``tangle_deploy.python_pipeline`` re-exports the OSS objects — so an author may -# use either import and codegen must strip either the same way. -_AUTHORING_IMPORT_MODULES = ( - "tangle_deploy.python_pipeline", - "tangle_cli.python_pipeline", -) +# OSS recognises exactly one authoring surface out of the box: the canonical +# ``tangle_cli.python_pipeline`` path. A downstream package that re-exports the +# authoring objects under its own module path — so authors may write ``from +# .python_pipeline import task`` — registers that path via +# :func:`register_authoring_import_module`; codegen then strips either import +# the same way. OSS never hardcodes a downstream module name (the dependency +# points inward), mirroring the resolver/reader registries in the hydrator. +_AUTHORING_IMPORT_MODULES: list[str] = ["tangle_cli.python_pipeline"] + + +def register_authoring_import_module(module: str) -> None: + """Register *module* as an additional python-pipeline authoring surface. + + A downstream package that re-exports the ``tangle_cli.python_pipeline`` + authoring objects under its own module path calls this (typically at import + time) so codegen strips ``from import ...`` / ``import `` + lines — and their submodules — from baked runtime source exactly like the + canonical OSS surface. Idempotent: registering an already-known module is a + no-op, so repeated import-time registration is safe. + """ + if module not in _AUTHORING_IMPORT_MODULES: + _AUTHORING_IMPORT_MODULES.append(module) + + +def authoring_import_modules() -> tuple[str, ...]: + """Return the python-pipeline authoring modules recognised by codegen.""" + return tuple(_AUTHORING_IMPORT_MODULES) # The authoring-only ``TaskEnv`` class name. A module-level ``X = TaskEnv(...)`` # (or ``X = .TaskEnv(...)``) declaration is authoring-only by contract and # is stripped from the baked source by ``_strip_authoring_constructs``. # Matched by trailing NAME only (like the authoring decorators), because in -# python-pipeline authoring files ``TaskEnv`` always -# resolves to ``tangle_deploy.python_pipeline.TaskEnv``. +# python-pipeline authoring files ``TaskEnv`` always resolves to the +# python-pipeline authoring surface's ``TaskEnv``. _AUTHORING_ENV_CLASS_NAME = "TaskEnv" @@ -802,7 +792,7 @@ def _decorator_called_name(node: ast.expr) -> str | None: Handles ``@name`` / ``@name(...)`` and ``@mod.name`` / ``@mod.name(...)`` forms, returning the trailing attribute/name (e.g. ``task`` for both - ``@task(...)`` and ``@tangle_deploy.python_pipeline.task(...)``). Returns + ``@task(...)`` and ``@tangle_cli.python_pipeline.task(...)``). Returns ``None`` for shapes we do not recognise so callers leave them untouched. Limitation (v1, intentional): matching is by trailing NAME only, not by @@ -829,18 +819,19 @@ def _is_authoring_module(name: str) -> bool: def _is_authoring_import(node: ast.stmt) -> bool: """Return True if *node* imports the python-pipeline authoring surface. - Matches ONLY the ``tangle_deploy.python_pipeline`` / ``tangle_cli.python_pipeline`` - modules (and their submodules): + Matches ONLY the registered authoring modules (and their submodules) — the + canonical ``tangle_cli.python_pipeline`` plus any registered via + :func:`register_authoring_import_module`: - ``from tangle_cli.python_pipeline import ...`` (including the aliased ``from tangle_cli.python_pipeline import ref as operation_by_ref`` form and submodules like ``from tangle_cli.python_pipeline.x import y``); - ``import tangle_cli.python_pipeline`` / ``import tangle_cli.python_pipeline as tp``; - - the legacy ``tangle_deploy.python_pipeline`` equivalents. + - the equivalents for any registered downstream authoring path. - It does NOT match other ``tangle_deploy.*`` / ``tangle_cli.*`` packages (e.g. - ``from tangle_deploy.utils import X``) — those can be genuine runtime helpers + It does NOT match other packages that merely share a top-level name (e.g. a + downstream ``*.utils`` module) — those can be genuine runtime helpers referenced inside a ``@task`` body and must survive into the baked program. Relative imports (``from . import x``) are never authoring imports. """ @@ -982,7 +973,7 @@ def _strip_authoring_constructs(source_code: str) -> str: - re-running an ``@task`` / ``@pipeline`` / ``@subpipeline`` decorator replaces the function with a ``CallableRef`` recorder, which raises at call time because there is no active ``@pipeline`` trace context; - - on a thin image the ``from tangle_deploy.python_pipeline import ...`` + - on a thin image the ``from tangle_cli.python_pipeline import ...`` import itself can fail with ``ImportError``. This removes them via surgical AST line-range deletion (mirroring @@ -997,9 +988,9 @@ def _strip_authoring_constructs(source_code: str) -> str: Scope of the strip (intentional v1 boundaries): - - imports: only ``tangle_deploy.python_pipeline`` (and submodules) are - dropped — see ``_is_authoring_import``. Other ``tangle_deploy.*`` runtime - helpers are preserved. + - imports: only the registered authoring modules (and submodules) are + dropped — see ``_is_authoring_import``. Other runtime helpers that merely + share a top-level name are preserved. - decorators: matched by trailing NAME (``task`` / ``pipeline`` / ``subpipeline``), not by import resolution — see ``_decorator_called_name`` for the limitation. Unrelated decorators (``@functools.cache``, @@ -1697,7 +1688,8 @@ def generate_component_yaml( path_annotation_mode: ``"oss"`` always records source/YAML paths relative to their common ancestor. ``"td_legacy"`` only uses that relative common-root behavior inside a git checkout; outside git it records - ``file_path.name`` / ``output_path.name`` like legacy tangle-deploy. + ``file_path.name`` / ``output_path.name`` to preserve the legacy + downstream driver's historical basename-only snapshots. Returns: True on success, False on failure. @@ -1775,8 +1767,9 @@ def generate_component_yaml( # Use the common ancestor of source and output so both paths are clean # forward references (no ".."). This lets later local maintenance # commands find the source even when YAML is generated into a separate - # output directory. TD legacy compatibility keeps basename-only paths - # outside a git checkout to preserve historical snapshots. + # output directory. Legacy (``td_legacy``) compatibility keeps + # basename-only paths outside a git checkout to preserve historical + # snapshots. resolved_source = file_path.resolve() resolved_output = output_path.resolve() common_dir = Path(os.path.commonpath([resolved_source, resolved_output])) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..969c353 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,97 @@ +"""Shared test fixtures for the tangle-cli suite. + +The codegen/strip tests exercise the case where a pipeline author imported the +authoring surface from a *downstream* package's module path (e.g. +``from tangle_deploy.python_pipeline import task``). OSS recognises only its own +``tangle_cli.python_pipeline`` path out of the box; a downstream package +contributes its path through the ``register_authoring_import_module`` seam and +makes that module importable itself. This conftest stands in for that downstream +package for the whole suite — registering the path and installing a stand-in +module in ``sys.modules`` — so those tests run without OSS ever hardcoding the +downstream name, and with the process-global registry + ``sys.modules`` kept +isolated between tests. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + +import tangle_cli.component_from_func as cff +from tangle_cli.component_from_func import register_authoring_import_module + +# The downstream authoring module path this suite simulates. Kept out of the OSS +# source itself (that would defeat the dependency inversion) — it lives only in +# the test harness that plays the downstream package. +DOWNSTREAM_AUTHORING_MODULE = "tangle_deploy.python_pipeline" + + +def _install_fake_authoring_module(module: str) -> None: + """Fabricate a stand-in downstream authoring module in ``sys.modules``. + + Mirrors what a downstream package makes importable at runtime: a module + exposing the authoring decorators/markers so a file doing ``from + import task`` can be introspected by ``load_python_module``. OSS no longer + fabricates this itself — that coupling was inverted into the registry seam — + so the test harness owns it, standing in for the downstream package. + """ + if module in sys.modules: + return + + def _identity_decorator(*args, **kwargs): + def decorate(func): + return func + + return decorate + + class _AuthoringGeneric: + def __class_getitem__(cls, item): + return cls + + def __init__(self, *args, **kwargs): + pass + + parent_name, _, leaf_name = module.rpartition(".") + leaf = types.ModuleType(module) + for name in ("task", "pipeline", "subpipeline", "registered"): + setattr(leaf, name, _identity_decorator) + for name in ("In", "Out", "Outputs", "TaskEnv"): + setattr(leaf, name, _AuthoringGeneric) + leaf.ref = lambda *args, **kwargs: None + + parent = sys.modules.get(parent_name) or types.ModuleType(parent_name) + setattr(parent, leaf_name, leaf) + sys.modules.setdefault(parent_name, parent) + sys.modules[module] = leaf + + +@pytest.fixture(autouse=True) +def downstream_authoring_surface(): + """Register + provide a downstream authoring surface for each test. + + Codegen recognises only ``tangle_cli.python_pipeline`` out of the box; the + legacy ``tangle_deploy.python_pipeline`` path is contributed by the + downstream package via ``register_authoring_import_module``. The strip and + codegen tests rely on that path being both registered (so it is stripped + from baked source) and importable (so ``load_python_module`` can load + fixtures that import it), so we set it up mimicking the downstream package + and tear it down to keep the process-global registry + ``sys.modules`` + isolated between tests. + """ + before = list(cff._AUTHORING_IMPORT_MODULES) + parent_name = DOWNSTREAM_AUTHORING_MODULE.rpartition(".")[0] + had_parent = parent_name in sys.modules + had_leaf = DOWNSTREAM_AUTHORING_MODULE in sys.modules + + register_authoring_import_module(DOWNSTREAM_AUTHORING_MODULE) + _install_fake_authoring_module(DOWNSTREAM_AUTHORING_MODULE) + try: + yield + finally: + cff._AUTHORING_IMPORT_MODULES[:] = before + if not had_leaf: + sys.modules.pop(DOWNSTREAM_AUTHORING_MODULE, None) + if not had_parent: + sys.modules.pop(parent_name, None) diff --git a/tests/test_component_from_func.py b/tests/test_component_from_func.py index 4d53862..ad03b0d 100644 --- a/tests/test_component_from_func.py +++ b/tests/test_component_from_func.py @@ -11,13 +11,13 @@ import pytest import yaml +import tangle_cli.component_from_func as cff from tangle_cli.component_from_func import ( AuthoringStripError, FunctionSpec, InputPath, OutputPath, ParamInfo, - _AUTHORING_IMPORT_MODULES, _build_argparse_code, _build_args_section, _build_pip_install_command, @@ -31,18 +31,25 @@ _strip_authoring_constructs, _strip_main_guard, _strip_type_hints, + authoring_import_modules, build_component_dict, extract_interface, generate_component_yaml, get_function_from_module, load_python_module, read_dependencies, + register_authoring_import_module, ) from tangle_cli.module_bundler import ModuleBundler # Real on-disk python-pipeline fixtures (``task_env_strip_*`` for Phase 3). _PIPELINE_FIXTURES = Path(__file__).parent / "fixtures" / "python_pipeline" +# The downstream authoring surface these strip/codegen tests rely on (a +# registered ``tangle_deploy.python_pipeline`` path + a stand-in module in +# ``sys.modules``) is provided suite-wide by the autouse +# ``downstream_authoring_surface`` fixture in ``tests/conftest.py``. + # ============================================================================ # Type resolution tests # ============================================================================ @@ -1668,11 +1675,36 @@ def test_strips_submodule_import(self): class TestAuthoringImportPredicate: """Unit tests for the authoring-module recognition helpers.""" - def test_authoring_modules_constant_contents(self): - assert _AUTHORING_IMPORT_MODULES == ( - "tangle_deploy.python_pipeline", - "tangle_cli.python_pipeline", - ) + def test_oss_seeds_only_the_canonical_surface(self): + # Under a pristine registry (no downstream registered), OSS recognises + # ONLY its own authoring path — it never hardcodes a downstream module. + # The autouse fixture registers the downstream path, so reset to the + # OSS seed for this assertion, then restore. + before = list(cff._AUTHORING_IMPORT_MODULES) + cff._AUTHORING_IMPORT_MODULES[:] = ["tangle_cli.python_pipeline"] + try: + assert authoring_import_modules() == ("tangle_cli.python_pipeline",) + finally: + cff._AUTHORING_IMPORT_MODULES[:] = before + + def test_register_authoring_import_module_adds_downstream_surface(self): + register_authoring_import_module("acme_pipelines.python_pipeline") + assert "acme_pipelines.python_pipeline" in authoring_import_modules() + # A registered path is treated exactly like the canonical one. + assert _is_authoring_module("acme_pipelines.python_pipeline") is True + assert _is_authoring_module("acme_pipelines.python_pipeline.types") is True + + def test_register_authoring_import_module_is_idempotent(self): + register_authoring_import_module("acme_pipelines.python_pipeline") + register_authoring_import_module("acme_pipelines.python_pipeline") + assert authoring_import_modules().count("acme_pipelines.python_pipeline") == 1 + + def test_authoring_import_modules_returns_immutable_snapshot(self): + # Callers get a tuple copy; mutating the return value must not leak into + # the registry. + snapshot = authoring_import_modules() + assert isinstance(snapshot, tuple) + assert authoring_import_modules() == snapshot @pytest.mark.parametrize( "name", diff --git a/tests/test_python_pipeline_dsl.py b/tests/test_python_pipeline_dsl.py index 0b8f959..5867f80 100644 --- a/tests/test_python_pipeline_dsl.py +++ b/tests/test_python_pipeline_dsl.py @@ -99,13 +99,20 @@ def test_import_is_light(self): class TestNoInternalReferences: - """The vendored DSL must be free of internal ``tangle-deploy`` - references, in either the ``tangle_deploy`` (module/import) or the - ``tangle-deploy`` (product/command) spelling.""" + """The OSS authoring + compile surface must be free of internal + ``tangle-deploy`` references, in either the ``tangle_deploy`` (module/import) + or the ``tangle-deploy`` (product/command) spelling. The dependency points + inward: a downstream package registers its own authoring path via + ``register_authoring_import_module`` — OSS never names it.""" def test_no_tangle_deploy_references_in_source(self): + # Scan the whole vendored DSL package plus the sibling compiler module + # (``component_from_func.py``), which drives the authoring-import strip + # and must stay decoupled from the downstream package name. + scanned = list(_DSL_DIR.glob("*.py")) + scanned.append(_DSL_DIR.parent / "component_from_func.py") offenders = [] - for path in _DSL_DIR.glob("*.py"): + for path in scanned: text = path.read_text(encoding="utf-8") if "tangle_deploy" in text or "tangle-deploy" in text: offenders.append(path.name)