Skip to content
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ dependencies = [
"httpx==0.28.1",
"pyyaml==6.0.3",
"pydantic==2.13.3",
"ruamel-yaml==0.19.1",
]

[project.optional-dependencies]
Expand Down
23 changes: 12 additions & 11 deletions src/tycoon/commands/explore.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import click
import typer

from tycoon.config import config
from tycoon.config import TycoonConfig, load_config
from tycoon.utils.console import error, header, info, success, warn


Expand Down Expand Up @@ -75,20 +75,21 @@ def analyze_cmd(
from tycoon.utils.duckdb_utils import get_tables

# 1. Verify tycoon.yml exists
if not config.has_project_file:
cfg = load_config()
if not cfg.has_project_file:
error("No tycoon.yml found. Run 'tycoon init' first.")
raise typer.Exit(1)

# 2. Resolve which source(s) we're analyzing.
sources = config.sources
sources = cfg.sources
if all_sources:
if source_name:
error("Pass either a source name or --all, not both.")
raise typer.Exit(1)
if not sources:
error("No sources registered in tycoon.yml. Run 'tycoon data sources add' first.")
raise typer.Exit(1)
_analyze_all(force=force, no_dbt=no_dbt, rill=rill, build=build)
_analyze_all(cfg=cfg, force=force, no_dbt=no_dbt, rill=rill, build=build)
return

if not source_name:
Expand All @@ -112,7 +113,7 @@ def analyze_cmd(
info(f"Schema: {schema_name}")

# 3. Verify raw database exists and has data for this schema
raw_db = config.raw_db
raw_db = cfg.raw_db
if not raw_db.exists():
error(f"Raw database not found at {raw_db}. Run 'tycoon data sources run {source_name}' first.")
raise typer.Exit(1)
Expand All @@ -133,7 +134,7 @@ def analyze_cmd(
# 4. Generate dbt staging models
if not no_dbt:
info("Generating dbt staging models...")
staging_dir = config.dbt_project_dir / "models" / "staging" / source_name
staging_dir = cfg.dbt_project_dir / "models" / "staging" / source_name
try:
result = generate_staging_models(
raw_db_path=raw_db,
Expand Down Expand Up @@ -167,7 +168,7 @@ def analyze_cmd(
from tycoon.scaffolding.rill_generator import generate_rill_config
from tycoon.scaffolding.templates import scaffold_rill_dir

rill_dir = config.rill_dir
rill_dir = cfg.rill_dir
if not rill_dir.exists():
info(f"Rill project not found; scaffolding at {rill_dir}")
scaffold_rill_dir(rill_dir)
Expand Down Expand Up @@ -222,15 +223,15 @@ def analyze_cmd(
success("dbt build completed successfully.")


def _analyze_all(*, force: bool, no_dbt: bool, rill: bool, build: bool) -> None:
def _analyze_all(*, cfg: TycoonConfig, force: bool, no_dbt: bool, rill: bool, build: bool) -> None:
"""Iterate every registered source and analyze each.

Soft-skips sources whose raw DB doesn't exist yet. ``--rill`` and
``--build`` apply per-source. Single summary at the end.
"""
from tycoon.scaffolding.dbt_generator import generate_staging_models

sources = config.sources
sources = cfg.sources
header(f"Analyzing all sources ({len(sources)})")

total_generated: list[str] = []
Expand All @@ -239,7 +240,7 @@ def _analyze_all(*, force: bool, no_dbt: bool, rill: bool, build: bool) -> None:

for src_name, src_cfg in sources.items():
info(f" → {src_name} (schema: {src_cfg.schema_name})")
raw_db = config.raw_db
raw_db = cfg.raw_db
if not raw_db.exists():
warn(
f" Skipping {src_name} — raw DB not found at {raw_db}. "
Expand All @@ -251,7 +252,7 @@ def _analyze_all(*, force: bool, no_dbt: bool, rill: bool, build: bool) -> None:
if no_dbt:
continue

staging_dir = config.dbt_project_dir / "models" / "staging" / src_name
staging_dir = cfg.dbt_project_dir / "models" / "staging" / src_name
try:
result = generate_staging_models(
raw_db_path=raw_db,
Expand Down
17 changes: 9 additions & 8 deletions src/tycoon/commands/run_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import typer

from tycoon.config import config
from tycoon.config import load_config
from tycoon.utils.console import console, error, header, info, next_steps, success, warn


Expand Down Expand Up @@ -46,7 +46,8 @@ def run_all_cmd(
] = False,
) -> None:
"""Ingest all registered sources then run dbt build."""
if not config.has_project_file:
cfg = load_config()
if not cfg.has_project_file:
error("No tycoon.yml found. Run [bold]tycoon init[/bold] first.")
raise typer.Exit(1)

Expand All @@ -59,7 +60,7 @@ def _emit(severity: str, message: str, **fields: str) -> None:
return
from tycoon import notify as notify_mod

project = config.project
project = cfg.project
prefs = project.notify if project is not None else None
allowed = prefs.severities if prefs is not None else ["success", "error"]
if severity not in allowed:
Expand All @@ -75,12 +76,12 @@ def _emit(severity: str, message: str, **fields: str) -> None:
if not skip_ingest:
from tycoon.ingestion.runner import run_source as _run_source

sources = config.sources
sources = cfg.sources
if not sources:
error("No sources registered. Run [bold]tycoon data sources add[/bold] first.")
raise typer.Exit(1)

config.ensure_data_dir()
cfg.ensure_data_dir()
total = len(sources)
info(f"Ingesting {total} source{'s' if total != 1 else ''}...")
if max_records is not None:
Expand All @@ -92,7 +93,7 @@ def _emit(severity: str, message: str, **fields: str) -> None:
_pipeline, load_info = _run_source(
name=name,
source_config=source_config,
raw_db_path=config.raw_db,
raw_db_path=cfg.raw_db,
max_records=max_records,
)
success(f"{name}: {load_info}")
Expand All @@ -111,7 +112,7 @@ def _emit(severity: str, message: str, **fields: str) -> None:
_emit("error", "run-all failed: dbt not found on PATH", stage="transform")
raise typer.Exit(1)

project_dir = config.dbt_project_dir
project_dir = cfg.dbt_project_dir
if not project_dir.exists():
warn(f"dbt project not found at {project_dir} — skipping transform.")
else:
Expand All @@ -132,7 +133,7 @@ def _emit(severity: str, message: str, **fields: str) -> None:
elapsed = time.time() - start
console.rule("[bold green]Done")
success(f"Finished in {elapsed:.1f}s")
_emit("success", "run-all complete", elapsed=f"{elapsed:.1f}s", sources=str(len(config.sources)))
_emit("success", "run-all complete", elapsed=f"{elapsed:.1f}s", sources=str(len(cfg.sources)))
next_steps(
("tycoon data status", "check source freshness and row counts"),
("tycoon start --only rill", "explore results in Rill"),
Expand Down
60 changes: 30 additions & 30 deletions src/tycoon/commands/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import typer
from rich.table import Table

from tycoon.config import config
from tycoon.config import TycoonConfig, load_config
from tycoon.ingestion.catalog import CATALOG, CatalogEntry
from tycoon.project import SourceConfig, load_project, save_project
from tycoon.utils.console import console, error, header, info, next_steps, success, warn
Expand All @@ -32,11 +32,13 @@
app.add_typer(list_app, name="list")


def _require_project() -> None:
"""Abort if no tycoon.yml exists."""
if not config.has_project_file:
def _require_project() -> TycoonConfig:
"""Return a fresh TycoonConfig, aborting if no tycoon.yml exists."""
cfg = load_config()
if not cfg.has_project_file:
error("No tycoon.yml found. Run [bold]tycoon init[/bold] first.")
raise typer.Exit(1)
return cfg


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -79,9 +81,9 @@ def catalog_default(ctx: typer.Context) -> None:

def _list_sources() -> None:
"""Print registered sources table."""
_require_project()
cfg = _require_project()

sources = config.sources
sources = cfg.sources
if not sources:
info("No sources registered yet.")
info("Browse available sources with [bold]tycoon data sources catalog[/bold]")
Expand Down Expand Up @@ -110,9 +112,9 @@ def show_source(
name: str = typer.Argument(help="Name of the source to show"),
) -> None:
"""Show detailed configuration for a specific source."""
_require_project()
cfg = _require_project()

sources = config.sources
sources = cfg.sources
if name not in sources:
error(f"Source [bold]{name}[/bold] not found.")
info(f"Available sources: {', '.join(sources.keys()) if sources else '(none)'}")
Expand Down Expand Up @@ -381,7 +383,7 @@ def add_source(
credentials default to ``${ENV_VAR}`` references in both modes —
set the env var separately.
"""
_require_project()
cfg = _require_project()

if not source_type:
if no_prompt:
Expand Down Expand Up @@ -449,7 +451,7 @@ def add_source(
config=source_config,
)

project = load_project(config.root)
project = load_project(cfg.root)
assert project is not None # guarded by _require_project

if source_name in project.sources:
Expand All @@ -464,8 +466,7 @@ def add_source(
raise typer.Exit(0)

project.sources[source_name] = new_source
save_project(project, config.root)
config.reload()
save_project(project, cfg.root)

success(f"Source [bold]{source_name}[/bold] added to tycoon.yml")

Expand Down Expand Up @@ -534,9 +535,9 @@ def remove_source(
name: str = typer.Argument(help="Name of the source to remove"),
) -> None:
"""Remove a registered data source."""
_require_project()
cfg = _require_project()

project = load_project(config.root)
project = load_project(cfg.root)
assert project is not None

if name not in project.sources:
Expand All @@ -547,8 +548,7 @@ def remove_source(
typer.confirm(f"Remove source '{name}'?", abort=True)

del project.sources[name]
save_project(project, config.root)
config.reload()
save_project(project, cfg.root)

success(f"Source [bold]{name}[/bold] removed from tycoon.yml")

Expand Down Expand Up @@ -585,7 +585,7 @@ def _source_already_referenced(dbt_dir: Path, source_name: str) -> bool:
return False


def _maybe_auto_scaffold(source_name: str, source_config: SourceConfig, *, scaffold: bool) -> None:
def _maybe_auto_scaffold(source_name: str, source_config: SourceConfig, *, cfg: TycoonConfig, scaffold: bool) -> None:
"""Auto-run the analyze flow if a dbt project exists and no staging
models are present for this source yet.

Expand All @@ -595,10 +595,10 @@ def _maybe_auto_scaffold(source_name: str, source_config: SourceConfig, *, scaff
"""
if not scaffold:
return
project = config.project
project = cfg.project
if project is not None and not project.transform.auto_scaffold:
return
dbt_dir = config.dbt_project_dir
dbt_dir = cfg.dbt_project_dir
if not dbt_dir.exists():
return # No dbt project to scaffold into.

Expand All @@ -611,7 +611,7 @@ def _maybe_auto_scaffold(source_name: str, source_config: SourceConfig, *, scaff
from tycoon.scaffolding.dbt_generator import generate_staging_models

result = generate_staging_models(
raw_db_path=config.raw_db,
raw_db_path=cfg.raw_db,
schema_name=source_config.schema_name,
source_name=source_name,
output_dir=staging_dir,
Expand Down Expand Up @@ -644,9 +644,9 @@ def run_source(
"""Ingest data from a registered source by name."""
from tycoon.ingestion.runner import run_source as _run_source

_require_project()
cfg = _require_project()

sources = config.sources
sources = cfg.sources
if not source_name:
if not sources:
error("No sources registered. Run 'tycoon data sources add' first.")
Expand All @@ -667,17 +667,17 @@ def run_source(
if max_records is not None:
info(f"Record cap: {max_records:,}")

config.ensure_data_dir()
cfg.ensure_data_dir()

try:
_pipeline, load_info = _run_source(
name=source_name,
source_config=source_config,
raw_db_path=config.raw_db,
raw_db_path=cfg.raw_db,
max_records=max_records,
)
success(f"{source_name} load complete. {load_info}")
_maybe_auto_scaffold(source_name, source_config, scaffold=not no_scaffold)
_maybe_auto_scaffold(source_name, source_config, cfg=cfg, scaffold=not no_scaffold)
next_steps(
("tycoon data transform run", "run dbt models on the ingested data"),
("tycoon start --only rill", "open the Rill dashboard"),
Expand All @@ -704,9 +704,9 @@ def run_all(
"""Run all registered source pipelines sequentially."""
from tycoon.ingestion.runner import run_source as _run_source

_require_project()
cfg = _require_project()

sources = config.sources
sources = cfg.sources
if not sources:
error("No sources registered. Run 'tycoon data sources add' first.")
raise typer.Exit(1)
Expand All @@ -716,19 +716,19 @@ def run_all(
if max_records is not None:
info(f"Record cap per resource: {max_records:,}")

config.ensure_data_dir()
cfg.ensure_data_dir()

for i, (name, source_config) in enumerate(sources.items(), 1):
info(f"Step {i}/{total} — {name} ({source_config.type})...")
try:
_pipeline, load_info = _run_source(
name=name,
source_config=source_config,
raw_db_path=config.raw_db,
raw_db_path=cfg.raw_db,
max_records=max_records,
)
success(f"{name} complete. {load_info}")
_maybe_auto_scaffold(name, source_config, scaffold=not no_scaffold)
_maybe_auto_scaffold(name, source_config, cfg=cfg, scaffold=not no_scaffold)
except Exception as exc:
error(f"{name} pipeline failed: {exc}")
raise typer.Exit(1) from exc
Expand Down
Loading