diff --git a/.gitignore b/.gitignore index fc5309f..5cf2b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,20 @@ debug.jsonl metadata.json spans.jsonl designs/ + +# Eval run artifacts (transcripts, per-eval results, summaries). +# Keep evals/baselines/ tracked -- that's the canonical pass/fail reference. +evals/results/ + +# Python +*.pyc +.pytest_cache/ +*.egg-info/ + +# Virtualenvs +.venv/ +venv/ + +# Local secrets / env overrides +.env +.envrc diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..29529a4 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,78 @@ +# Skill Evals + +Evaluation of skills under `skills//`. For each eval, the runner spawns a headless `claude --print` agent, captures its transcript, then asks a separate LLM-as-judge call to score each expectation pass/fail. + +## Running + +```bash +# Single skill +python evals/run.py --skill rai-predictive-training + +# Single eval id (faster iteration) +python evals/run.py --skill rai-predictive-training --eval-id 1 + +# Multiple skills in one run +python evals/run.py --skill rai-ontology-design --skill rai-predictive-modeling + +# Run from a different working directory (where `raiconfig.yaml` lives, etc.) +python evals/run.py --skill --cwd /path/to/project + +# Freeze current pass/fail map as the baseline +python evals/run.py --skill --update-baseline + +# Verbose: per-expectation judge progress and banners around each eval +python evals/run.py --skill --eval-id 1 -v +``` + +`run.py` exits non-zero if any expectation regressed against the stored baseline (passing in baseline, now failing). + +**About `-v`.** It prints banners between phases (`running agent...`, `agent done`, `judging i/N: ...`, `PASS/FAIL -- `) so you can see judge-by-judge progress live. It does **not** stream the agent's stdout in real time — `claude --print` with text output buffers the entire reply and emits it at the end of the run, so the agent phase is silent regardless of `-v`. Use `-v` mainly to watch the judge phase or to confirm the runner is alive between evals. + +## Adding evals + +Edit `skills//evals/evals.json`: + +```json +{ + "id": 1, + "prompt": "", + "expected_output": "", + "expectations": [ + "", + "" + ] +} +``` + +- **Each `expectation` is judged in isolation** — keep them atomic. Vague or compound claims produce inconsistent verdicts. +- **`expected_output` is documentation only**; the judge never sees it. It's a note for whoever maintains the eval. +- **Anchor on artifacts**, not process: specific code patterns, argument names, design choices — not "explained well." + +After adding, run once, spot-check verdicts in `results///eval_.json`, then re-run with `--update-baseline` to freeze the reference. + +## Layout + +``` +evals/ + run.py # CLI orchestrator + runner.py # claude --print subprocess wrapper (one per eval) + judge.py # LLM-as-judge (one call per expectation) + diff.py # baseline comparison + baselines/.json + results/// + eval__transcript.txt # raw agent transcript + eval_.json # per-eval result + per-expectation judgments + skill_summary.json # aggregate + diff vs baseline +``` + +Eval prompts live next to each skill: `skills//evals/evals.json`. + +## Prerequisites + +- `claude` CLI authenticated (`claude --version` works). +- The skill must resolve from the agent's `--cwd` (default: `Path.cwd()`). +- Evals that touch Snowflake need a working `raiconfig.yaml` and the `relationalai` SDK in that cwd. + +## Permissions + +The runner passes `--dangerously-skip-permissions` so the agent can read files, run Bash, and introspect Snowflake without per-call approval. Same blast radius as an unsupervised agent — review prompts and `cwd` before running. diff --git a/evals/baselines/rai-predictive-training.json b/evals/baselines/rai-predictive-training.json new file mode 100644 index 0000000..8d811d7 --- /dev/null +++ b/evals/baselines/rai-predictive-training.json @@ -0,0 +1,28 @@ +{ + "1": [ + { + "expectation": "The produced script contains all phases in order: Concept declarations (Customer, Article, Transaction, task table concepts without identify_by), Table-backed define() calls, Train/Val/Test Relationships with 'at {Any:timestamp}' (Train/Val also 'has {Any:label}', Test omits the label), Graph with Edges, PropertyTransformer, GNN constructor, gnn.fit(), gnn.predictions(domain=Test) bound to Customer.predictions, and a final select(...).where(Customer.predictions).inspect()", + "passed": true + }, + { + "expectation": "Script executes without raising any Python exception or PyRel compile error end-to-end", + "passed": true + }, + { + "expectation": "Training job reaches JOB_COMPLETED status (visible in streamed logs); the agent does not declare success on JOB_START or partial logs", + "passed": true + }, + { + "expectation": "Prediction job reaches JOB_COMPLETED status and predictions are loaded back into the logic engine", + "passed": true + }, + { + "expectation": "Final select(...).inspect() (or .to_df()) returns a non-empty result with columns (c_customer_id, probs, predicted_labels), probs in [0, 1], predicted_labels in {0, 1}", + "passed": true + }, + { + "expectation": "The script does not introduce manual job polling, sleep loops, or status-check API calls around fit() or predictions() -- both are blocking and stream logs by default", + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/diff.py b/evals/diff.py new file mode 100644 index 0000000..9474c72 --- /dev/null +++ b/evals/diff.py @@ -0,0 +1,107 @@ +"""Compare a current run's pass/fail map against a stored baseline. + +Baseline format (per skill): + { + "": [ + {"expectation": "...", "passed": true|false}, + ... + ], + ... + } +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def to_passmap(results: dict[str, list[dict]]) -> dict[str, list[dict[str, Any]]]: + """Project a per-skill results dict to {eval_id: [{expectation, passed}, ...]}.""" + out: dict[str, list[dict[str, Any]]] = {} + for eid, exp_list in results.items(): + out[str(eid)] = [ + {"expectation": e["expectation"], "passed": bool(e["passed"])} + for e in exp_list + ] + return out + + +def compare( + current: dict[str, list[dict]], + baseline: dict[str, list[dict]] | None, +) -> dict: + if baseline is None: + return { + "first_run": True, + "regressions": [], + "improvements": [], + "unchanged": [], + "added": [], + "removed": [], + } + + regressions: list[dict] = [] + improvements: list[dict] = [] + unchanged: list[dict] = [] + added: list[dict] = [] + removed: list[dict] = [] + + cur = to_passmap(current) + + for eid, exp_list in cur.items(): + base_list = baseline.get(eid, []) + # Index baseline by expectation text (resilient to reordering). + base_by_text = {e["expectation"]: bool(e["passed"]) for e in base_list} + for exp in exp_list: + text = exp["expectation"] + now_pass = bool(exp["passed"]) + if text not in base_by_text: + added.append({ + "eval_id": eid, + "expectation": text, + "passed": now_pass, + }) + continue + was_pass = base_by_text[text] + row = {"eval_id": eid, "expectation": text} + if was_pass and not now_pass: + regressions.append(row) + elif not was_pass and now_pass: + improvements.append(row) + else: + unchanged.append({**row, "passed": now_pass}) + + # Detect removed expectations / removed evals. + for eid, base_list in baseline.items(): + cur_texts = {e["expectation"] for e in cur.get(eid, [])} + for be in base_list: + if be["expectation"] not in cur_texts: + removed.append({ + "eval_id": eid, + "expectation": be["expectation"], + "was_passed": bool(be["passed"]), + }) + + return { + "first_run": False, + "regressions": regressions, + "improvements": improvements, + "unchanged": unchanged, + "added": added, + "removed": removed, + } + + +def load_baseline(path: Path) -> dict[str, list[dict]] | None: + if not path.exists(): + return None + with open(path) as f: + return json.load(f) + + +def write_baseline(path: Path, current: dict[str, list[dict]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(to_passmap(current), f, indent=2, sort_keys=True) diff --git a/evals/judge.py b/evals/judge.py new file mode 100644 index 0000000..2a4c940 --- /dev/null +++ b/evals/judge.py @@ -0,0 +1,102 @@ +"""LLM-as-judge for skill evals. + +One `claude --print` call per (transcript, expectation) pair. The judge +prompt forces a single-line JSON response: {"passed": bool, "justification": str}. +""" + +from __future__ import annotations + +import json +import re +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_TIMEOUT = 180 +DEFAULT_CLAUDE_BIN = "claude" + +JUDGE_PROMPT = """\ +You are evaluating whether an AI agent's response satisfies a specific expectation. +Respond with only a single JSON object on one line. No markdown. No prose. + +EXPECTATION: +{expectation} + +AGENT_RESPONSE: +<<>> +{transcript} +<<>> + +Output schema (single line, no code fence): +{{"passed": true|false, "justification": "<1-2 sentences citing specific evidence from AGENT_RESPONSE>"}} +""" + + +@dataclass +class JudgeResult: + passed: bool + justification: str + raw: str + + def to_dict(self) -> dict: + return { + "passed": self.passed, + "justification": self.justification, + "raw": self.raw, + } + + +def _extract_json(s: str) -> dict | None: + s = s.strip() + # Strip code fences if present. + s = re.sub(r"^```(?:json)?\s*", "", s) + s = re.sub(r"\s*```$", "", s) + # Find the first {...} block. + match = re.search(r"\{.*\}", s, re.DOTALL) + if not match: + return None + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + return None + + +def judge( + transcript: str, + expectation: str, + timeout: int = DEFAULT_TIMEOUT, + claude_bin: str = DEFAULT_CLAUDE_BIN, + cwd: Path | None = None, +) -> JudgeResult: + prompt = JUDGE_PROMPT.format(expectation=expectation, transcript=transcript) + cmd = [claude_bin, "--print", prompt] + try: + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return JudgeResult( + passed=False, + justification="Judge call timed out.", + raw="", + ) + + raw = proc.stdout.strip() + parsed = _extract_json(raw) + if parsed is None: + return JudgeResult( + passed=False, + justification=f"Could not parse JSON from judge: {raw[:300]!r}", + raw=raw, + ) + + return JudgeResult( + passed=bool(parsed.get("passed", False)), + justification=str(parsed.get("justification", "")), + raw=raw, + ) diff --git a/evals/run.py b/evals/run.py new file mode 100644 index 0000000..61fbb01 --- /dev/null +++ b/evals/run.py @@ -0,0 +1,303 @@ +"""End-to-end skill eval orchestrator. + +For each eval in a skill's evals.json: + 1. Run the agent headless via `claude --print` (runner.py) + 2. Judge each expectation via a separate `claude --print` call (judge.py) + 3. Save per-eval transcript + judgments + 4. Diff against baselines/.json (diff.py) + 5. Print summary; exit non-zero if regressions + +Usage: + python evals/run.py --skill rai-predictive-modeling + python evals/run.py --all + python evals/run.py --all --update-baseline # promote current results + python evals/run.py --skill rai-predictive-training --eval-id 3 + python evals/run.py --all --cwd /data/haythem/PyRel +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) + +from runner import DEFAULT_CWD, run_eval # noqa: E402 +from judge import judge # noqa: E402 +from diff import compare, load_baseline, write_baseline, to_passmap # noqa: E402 + +REPO = ROOT.parent +SKILLS_DIR = REPO / "skills" +BASELINES_DIR = ROOT / "baselines" +RESULTS_DIR = ROOT / "results" + +PREDICTIVE_SKILLS = ["rai-predictive-training"] + + +def load_skill_evals(skill_name: str) -> dict: + path = SKILLS_DIR / skill_name / "evals" / "evals.json" + if not path.exists(): + raise FileNotFoundError(f"No evals.json for skill {skill_name}: {path}") + with open(path) as f: + return json.load(f) + + +def run_one_eval( + skill_name: str, + eval_obj: dict, + out_dir: Path, + cwd: Path, + run_timeout: int, + judge_timeout: int, + verbose: bool = False, +) -> dict: + eid = eval_obj["id"] + if verbose: + print(f"\n--- [{skill_name}] eval {eid}: running agent (timeout={run_timeout}s) ---", flush=True) + else: + print(f" [{skill_name}] eval {eid} ... ", end="", flush=True) + + run = run_eval( + skill_name=skill_name, + eval_prompt=eval_obj["prompt"], + cwd=cwd, + timeout=run_timeout, + stream_to_stdout=verbose, + ) + + if verbose: + print(f"\n--- [{skill_name}] eval {eid}: agent done (exit={run.exit_code}, timeout={run.timed_out}) ---", flush=True) + + transcript_path = out_dir / f"eval_{eid}_transcript.txt" + transcript_path.write_text(run.transcript) + + if run.timed_out or run.exit_code != 0: + print(f"runner failed (exit={run.exit_code}, timeout={run.timed_out})") + # Still save a result row so diff sees it as failing. + exp_results = [ + { + "expectation": e, + "passed": False, + "justification": f"Runner failed: exit={run.exit_code}, timeout={run.timed_out}", + "raw": "", + } + for e in eval_obj["expectations"] + ] + result = { + "id": eid, + "all_passed": False, + "expectations": exp_results, + "run_meta": run.to_dict(), + } + (out_dir / f"eval_{eid}.json").write_text(json.dumps(result, indent=2)) + return result + + exp_results: list[dict] = [] + total = len(eval_obj["expectations"]) + for i, exp in enumerate(eval_obj["expectations"], 1): + if verbose: + print(f" judging {i}/{total}: {exp[:90]}", flush=True) + j = judge(run.transcript, exp, timeout=judge_timeout) + if verbose: + mark = "PASS" if j.passed else "FAIL" + print(f" {mark} -- {j.justification[:160]}", flush=True) + exp_results.append({ + "expectation": exp, + "passed": j.passed, + "justification": j.justification, + "raw": j.raw, + }) + + passed = sum(1 for e in exp_results if e["passed"]) + if verbose: + print(f" [{skill_name}] eval {eid}: {passed}/{total} passed", flush=True) + else: + print(f"{passed}/{total} passed") + + result = { + "id": eid, + "all_passed": passed == total, + "expectations": exp_results, + "run_meta": { + "exit_code": run.exit_code, + "timed_out": run.timed_out, + "cmd": run.cmd, + }, + } + (out_dir / f"eval_{eid}.json").write_text(json.dumps(result, indent=2)) + return result + + +def run_skill( + skill_name: str, + out_root: Path, + cwd: Path, + eval_ids: list[int] | None, + run_timeout: int, + judge_timeout: int, + verbose: bool = False, +) -> dict: + print(f"\n=== {skill_name} ===") + skill_evals = load_skill_evals(skill_name) + out_dir = out_root / skill_name + out_dir.mkdir(parents=True, exist_ok=True) + + selected = skill_evals["evals"] + if eval_ids: + selected = [e for e in selected if e["id"] in eval_ids] + + results_by_id: dict[str, list[dict]] = {} + for eval_obj in selected: + res = run_one_eval( + skill_name=skill_name, + eval_obj=eval_obj, + out_dir=out_dir, + cwd=cwd, + run_timeout=run_timeout, + judge_timeout=judge_timeout, + verbose=verbose, + ) + results_by_id[str(res["id"])] = res["expectations"] + + baseline_path = BASELINES_DIR / f"{skill_name}.json" + baseline = load_baseline(baseline_path) + diff = compare(results_by_id, baseline) + + skill_summary = { + "skill_name": skill_name, + "results": results_by_id, + "diff": diff, + } + (out_dir / "skill_summary.json").write_text(json.dumps(skill_summary, indent=2)) + return skill_summary + + +def print_summary(summary: list[dict]) -> bool: + """Print aggregate; return True if any regressions were found.""" + print("\n=== Summary ===") + has_regression = False + for s in summary: + results = s["results"] + total = sum(len(v) for v in results.values()) + passed = sum(1 for v in results.values() for e in v if e["passed"]) + diff = s["diff"] + regs = len(diff.get("regressions", [])) + imps = len(diff.get("improvements", [])) + added = len(diff.get("added", [])) + removed = len(diff.get("removed", [])) + flag = " (first run, no baseline)" if diff.get("first_run") else "" + + print( + f" {s['skill_name']}: {passed}/{total} passed | " + f"regressions: {regs} | improvements: {imps} | " + f"added: {added} | removed: {removed}{flag}" + ) + + if regs > 0: + has_regression = True + for r in diff["regressions"]: + short = r["expectation"][:90] + print(f" REGRESSION eval={r['eval_id']}: {short}") + + return has_regression + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser() + p.add_argument( + "--skill", + action="append", + help="Skill name (repeatable). Defaults to predictive skills.", + ) + p.add_argument("--all", action="store_true", help="Run all predictive skills.") + p.add_argument( + "--eval-id", + type=int, + action="append", + help="Restrict to specific eval id(s).", + ) + p.add_argument( + "--cwd", + default=str(DEFAULT_CWD), + help="Working directory for the agent subprocess (default: %(default)s).", + ) + p.add_argument( + "--out", + default=None, + help="Output directory for this run (default: evals/results/).", + ) + p.add_argument( + "--run-timeout", + type=int, + default=1200, + help="Per-eval agent timeout in seconds (default: %(default)s).", + ) + p.add_argument( + "--judge-timeout", + type=int, + default=180, + help="Per-expectation judge timeout in seconds (default: %(default)s).", + ) + p.add_argument( + "--update-baseline", + action="store_true", + help="After the run, write current pass/fail map to baselines/.json.", + ) + p.add_argument( + "-v", "--verbose", + action="store_true", + help="Stream agent stdout in real time and print per-expectation judge progress.", + ) + return p.parse_args() + + +def main() -> int: + args = parse_args() + + if args.all: + skills = PREDICTIVE_SKILLS + elif args.skill: + skills = args.skill + else: + skills = PREDICTIVE_SKILLS # default + + run_id = _dt.datetime.now().strftime("%Y%m%d_%H%M%S") + out_root = Path(args.out) if args.out else (RESULTS_DIR / run_id) + out_root.mkdir(parents=True, exist_ok=True) + print(f"Run id: {run_id}") + print(f"Results -> {out_root}") + print(f"Working dir for agent: {args.cwd}") + + cwd = Path(args.cwd) + summary: list[dict] = [] + for skill_name in skills: + s = run_skill( + skill_name=skill_name, + out_root=out_root, + cwd=cwd, + eval_ids=args.eval_id, + run_timeout=args.run_timeout, + judge_timeout=args.judge_timeout, + verbose=args.verbose, + ) + summary.append(s) + + (out_root / "summary.json").write_text(json.dumps(summary, indent=2)) + + has_regression = print_summary(summary) + + if args.update_baseline: + for s in summary: + baseline_path = BASELINES_DIR / f"{s['skill_name']}.json" + write_baseline(baseline_path, s["results"]) + print(f"Baseline updated: {baseline_path}") + + return 1 if has_regression else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/runner.py b/evals/runner.py new file mode 100644 index 0000000..b8f39e9 --- /dev/null +++ b/evals/runner.py @@ -0,0 +1,107 @@ +"""Subprocess wrapper around `claude --print` for skill evals. + +One subprocess per eval. Captures stdout (the agent transcript), stderr, +exit code, and timeout state. The runner does NOT judge; it only collects. +""" + +from __future__ import annotations + +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_CWD = Path.cwd() +DEFAULT_TIMEOUT = 600 # seconds per agent run +DEFAULT_CLAUDE_BIN = "claude" + + +@dataclass +class RunResult: + transcript: str + stderr: str + exit_code: int + timed_out: bool + cmd: str + + def to_dict(self) -> dict: + return { + "transcript": self.transcript, + "stderr": self.stderr, + "exit_code": self.exit_code, + "timed_out": self.timed_out, + "cmd": self.cmd, + } + + +def build_prompt(skill_name: str, eval_prompt: str) -> str: + return ( + f"You must use the {skill_name} skill to answer the following request. " + f"Reply with the requested artifact only — no preamble, no questions back, " + f"no follow-up offers.\n\n" + f"REQUEST:\n{eval_prompt}" + ) + + +def run_eval( + skill_name: str, + eval_prompt: str, + cwd: Path = DEFAULT_CWD, + timeout: int = DEFAULT_TIMEOUT, + claude_bin: str = DEFAULT_CLAUDE_BIN, + skip_permissions: bool = True, + stream_to_stdout: bool = False, +) -> RunResult: + prompt = build_prompt(skill_name, eval_prompt) + cmd = [claude_bin, "--print"] + if skip_permissions: + cmd.append("--dangerously-skip-permissions") + cmd.append(prompt) + + cmd_str = " ".join(shlex.quote(c) for c in cmd[:-1]) + " " + + proc = subprocess.Popen( + cmd, + cwd=str(cwd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + transcript_chunks: list[str] = [] + deadline = time.monotonic() + timeout + timed_out = False + + assert proc.stdout is not None + for line in proc.stdout: + transcript_chunks.append(line) + if stream_to_stdout: + sys.stdout.write(line) + sys.stdout.flush() + if time.monotonic() > deadline: + timed_out = True + proc.kill() + break + + try: + proc.wait(timeout=max(0.1, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + timed_out = True + proc.kill() + proc.wait() + + stderr = proc.stderr.read() if proc.stderr else "" + transcript = "".join(transcript_chunks) + if timed_out: + stderr += "\n[TIMEOUT]" + + return RunResult( + transcript=transcript, + stderr=stderr, + exit_code=proc.returncode if proc.returncode is not None else -1, + timed_out=timed_out, + cmd=cmd_str, + ) diff --git a/skills/rai-discovery/SKILL.md b/skills/rai-discovery/SKILL.md index afe92b0..54cac97 100644 --- a/skills/rai-discovery/SKILL.md +++ b/skills/rai-discovery/SKILL.md @@ -429,7 +429,7 @@ Each suggestion includes a `reasoners` field — an ordered list specifying the **After discovery, load these skills before writing code:** -1. **Formulation skill** for the chosen reasoner type (e.g., `rai-prescriptive-problem-formulation`, `rai-graph-analysis`) +1. **Formulation skill** for the chosen reasoner type (e.g., `rai-prescriptive-problem-formulation`, `rai-graph-analysis`, `rai-predictive-modeling`) 2. **`rai-querying`** + **`rai-pyrel-coding`** for v1 syntax, imports, and query patterns Discovery covers *what* to ask. Coding skills cover *how* to write it. Skipping step 2 leads to hallucinated APIs and wrong imports. diff --git a/skills/rai-discovery/references/predictive.md b/skills/rai-discovery/references/predictive.md index f42212a..d857a05 100644 --- a/skills/rai-discovery/references/predictive.md +++ b/skills/rai-discovery/references/predictive.md @@ -11,7 +11,7 @@ Predictive reasoning uses historical data patterns to forecast outcomes, classify entities, or detect anomalies. -**Current platform status:** The RAI predictive reasoner is not yet integrated into the platform. Today, predictive capabilities are delivered via **pre-computed prediction tables** — external ML outputs loaded into Snowflake and mapped as ontology concepts. Discovery should identify both pre-computed predictions already in the data and predictive questions the data could support. +**Two modes:** Predictive capabilities can be delivered via **pre-computed prediction tables** (external ML outputs loaded into Snowflake) or via the **RAI predictive pipeline** (GNN-based models trained directly on the knowledge graph — see `rai-predictive-modeling` and `rai-predictive-training`). Discovery should identify both pre-computed predictions already in the data and predictive questions the data could support via GNN training. | Type | Question Pattern | Ontology Signal | |------|-----------------|-----------------| @@ -39,7 +39,7 @@ Problem type: `classification`, `regression`, `forecasting`, `anomaly_detection` ### mode How prediction is delivered: - **`pre_computed`**: A prediction/forecast table already exists in the schema. Discovery identifies it and suggests downstream use by other reasoners. -- **`rai_predictive`**: Future — when the RAI predictive reasoner is platform-integrated. +- **`rai_predictive`**: Build and train a graph neural network (GNN) using the RAI predictive pipeline (**early access** — APIs and behavior may change). See `rai-predictive-modeling` for data modeling and `rai-predictive-training` for training and evaluation. ### target_concept / target_property What to predict. E.g., `Supplier` / `delay_days`, or `Customer` / `churn_flag`. @@ -106,7 +106,7 @@ A `DelayPrediction` table with `predicted_delay_prob` and `risk_tier` per suppli ## Output Concepts -Predictive reasoning (whether pre-computed or future RAI-native) adds concepts to the ontology that downstream reasoners consume: +Predictive reasoning (whether pre-computed or via the RAI predictive pipeline) adds concepts to the ontology that downstream reasoners consume: | Prediction Type | Output Concept | Downstream Use | |----------------|----------------|----------------| @@ -128,11 +128,11 @@ What ontology patterns indicate prediction potential: - Look for columns named `predicted_*`, `probability`, `risk_*`, `forecast_*`, `confidence` - Check if the prediction table links to other ontology concepts via FK (e.g., supplier_id linking predictions to Supplier concept) -### For rai_predictive mode (future) +### For rai_predictive mode (GNN training) - **Feature availability**: Target property with sufficient non-null values; 3+ candidate features with variance - **Temporal span**: For forecasting, at least 2 full cycles of the target period (quarterly prediction needs 6+ months of history) - **Label quality**: For classification, labels exist and are reasonably balanced (flag extreme imbalance like 99%/1%) - **Row count**: Rough minimums (regression 50+, classification 30+ per class, forecasting 2+ full periods) - **Feature-target relationship**: At least some features plausibly related to target (domain signal) -**Minimum viable ontology for prediction:** For pre-computed: a prediction table exists and links to other concepts. For future rai_predictive: at least one concept with a target property (what to predict) and 2+ feature properties (what to predict from), backed by sufficient historical data. +**Minimum viable ontology for prediction:** For pre-computed: a prediction table exists and links to other concepts. For rai_predictive (GNN): at least one concept with a target property (what to predict) and 2+ feature properties (what to predict from), backed by sufficient historical data. See `rai-predictive-modeling` for the full data modeling workflow. diff --git a/skills/rai-graph-analysis/SKILL.md b/skills/rai-graph-analysis/SKILL.md index a344986..8f708a6 100644 --- a/skills/rai-graph-analysis/SKILL.md +++ b/skills/rai-graph-analysis/SKILL.md @@ -38,6 +38,7 @@ description: Graph algorithm selection and execution on PyRel v1 models. Covers - Ontology design decisions (concept modeling, data mapping) — see `rai-ontology-design` - Optimization formulation (variables, constraints, objectives) — see `rai-prescriptive-problem-formulation` - Business rule authoring (validation, classification, alerting) — see `rai-rules-authoring` +- GNN graph construction for predictive pipelines — see `rai-predictive-modeling` **Overview (process steps):** 1. Study the existing model — understand base definitions, coding conventions, and what's already wired diff --git a/skills/rai-predictive-modeling/SKILL.md b/skills/rai-predictive-modeling/SKILL.md new file mode 100644 index 0000000..e435025 --- /dev/null +++ b/skills/rai-predictive-modeling/SKILL.md @@ -0,0 +1,296 @@ +--- +name: rai-predictive-modeling +description: Build GNN data models -- concepts, Snowflake data loading, task relationships, graph edges, and PropertyTransformer features. Use when defining entity types, loading data, or configuring graph structure for a predictive GNN pipeline. +--- + +# Predictive Modeling + + +> **Early access.** The RAI predictive reasoner (GNN) is in early access — APIs, engine requirements, and behavior may change. Confirm the latest surface with the RelationalAI team before production use. + +## Summary + +**What:** Data modeling workflow for GNN pipelines -- from imports through graph construction and feature configuration. + +**When to use:** +- Defining concepts and loading data from Snowflake +- Building graph structure (edges, self-references) +- Configuring task relationships (train/val/test splits) +- Setting up PropertyTransformer features + +**When NOT to use:** +- Training, predictions, evaluation, model management -- see `rai-predictive-training` +- Graph algorithms (centrality, community detection) -- see `rai-graph-analysis` + +**Overview:** 6 steps: imports -> concepts -> populate -> task relationships -> graph -> features + +--- + +## Quick Reference + +```python +# Imports +from relationalai.semantics import Model, select, define, Integer, String, Any +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +model = Model("") +Concept, Table, Relationship = model.Concept, model.Table, model.Relationship +``` + +| Pattern | Code | +|---------|------| +| Single PK | `User = Concept("User", identify_by={"user_id": Integer})` | +| Composite PK | `Class = Concept("Class", identify_by={"courseid": Integer, "year": Integer})` | +| No PK (e.g. task table) | `TrainTable = Concept("TrainTable")` | + +```python +# Graph init +gnn_graph = Graph(model, directed=True, weighted=False) +Edge = gnn_graph.Edge + +# PropertyTransformer +pt = PropertyTransformer( + category=[User.locale, User.gender], + continuous=[User.birthyear], + datetime=[User.joinedAt, Event.start_time], + time_col=[Event.start_time], +) +``` + +--- + +## Imports and Model Setup + +```python +from relationalai.semantics import Model, select, define, Integer, String, Any +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +model = Model("") +Concept, Table, Relationship = model.Concept, model.Table, model.Relationship +``` + +Additional type imports as needed: `Date`, `DateTime`, `Float`. + +--- + +## Define and Populate Concepts + +> **User-input boundary:** the only things you need from the user are the 3 inputs in [`references/auto-discovery.md`](references/auto-discovery.md) -- source table FQNs, task table FQNs, and the experiment tracking database and schema. Auto-derive PKs, FKs, columns, types, edges, task type, and timestamp candidates from Snowflake schema introspection. Use the in-skill `get_table_schema(table_name, database, schema)` helper in `references/auto-discovery.md` as the default schema source before any manual SQL fallback. Don't ask the user for column-level details. + +Two concept categories show up in a GNN pipeline, distinguished by their role in the graph: + +| Category | Role | +|----------|------| +| **Graph (node)** | Source, target, or other node entities the GNN reasons over -- can carry features and `time_col` | +| **Task table** | Holds train/val/test split rows, joined to a graph concept by FK -- not used in edges; not a feature source | + +`identify_by` is not required by the GNN pipeline. Pass it when you want to declare an explicit primary key for a graph concept (matches a Snowflake column); omit it for task tables and for graph concepts where you don't need an explicit PK. + +> If you have an existing ontology from `rai-build-starter-ontology`, create a new `Model` for the GNN pipeline. + +### Graph (node) Concepts + +The `identify_by` key names must exist as columns in the Snowflake table. Column-name matching is **case-insensitive** in both `identify_by` keys and property accesses -- a Snowflake column `FOO_BAR` can be referenced as `Concept.foo_bar`, `Concept.FOO_BAR`, or any other casing. Spelling still has to match exactly. Check `INFORMATION_SCHEMA.COLUMNS` or run `DESCRIBE TABLE` to confirm the columns before writing `identify_by` or property accesses. + +```python +User = Concept("User", identify_by={"user_id": Integer}) +Event = Concept("Event", identify_by={"event_id": Integer}) +``` + +### Task Table Concepts + +Task table concepts have no `identify_by`: + +```python +train_table_concept = Concept("TrainTable") +val_table_concept = Concept("ValidationTable") +test_table_concept = Concept("TestTable") +``` + +### Populate from Snowflake + +```python +define(Customer.new(Table("DB.SCHEMA.CUSTOMERS").to_schema())) +define(train_table_concept.new(Table("DB.TASKS.TRAIN").to_schema())) +``` + +The GNN pipeline expects pre-existing train/val/test split tables in Snowflake. Each split table must contain: a join key column matching a source concept PK, a label/target column (train/val only), and optionally a timestamp column. + +`PropertyTransformer` and the task-table pattern also work with concepts populated from local data via `model.data(df)` -- not just `Table(...).to_schema()`. Useful when some concept data lives in local CSVs (e.g. optimizer parameters) while the graph comes from Snowflake. + +--- + +## Task Relationships + +Relationships encode the task structure using a template string with three parts: +- **Head** = source concept (the concept being predicted on) +- **"at" clause** = optional timestamp field +- **"has" clause** = label (classification/regression) or target concept (link prediction) + +### Relationship Arity Rules + +| Task Type | Train/Val template | Test template | +|-----------|-------------------|---------------| +| classification (no time) | `f"{Source} has {Any:label}"` | `f"{Source}"` | +| classification (with time) | `f"{Source} at {Any:ts} has {Any:label}"` | `f"{Source} at {Any:ts}"` | +| regression (no time) | `f"{Source} has {Any:value}"` | `f"{Source}"` | +| regression (with time) | `f"{Source} at {Any:ts} has {Any:value}"` | `f"{Source} at {Any:ts}"` | +| link_prediction | `f"{Source} has {Target}"` | `f"{Source}"` | +| repeated_link_prediction | `f"{Source} at {Any:ts} has {Target}"` | `f"{Source} at {Any:ts}"` | + +For full code examples of all task type patterns, see [references/task-relationships.md](references/task-relationships.md). + +--- + +## Graph and Edges + +```python +gnn_graph = Graph(model, directed=True, weighted=False) +Edge = gnn_graph.Edge +``` + +### Standard Edges (FK field equality) + +```python +define(Edge.new(src=Interaction, dst=User)).where( + Interaction.user_id == User.user_id) +``` + +### Self-Referential Edges (use `.ref()`) + +```python +PostRef = Post.ref() +define(Edge.new(src=Post, dst=PostRef)).where( + PostRef.parent_id == Post.id) +``` + +### Mediated Self-Reference + +```python +PeopleRef = People.ref() +define(Edge.new(src=People, dst=PeopleRef)).where( + People.Id == Related.person1, + PeopleRef.Id == Related.person2, +) +``` + +### Multiple Typed Edges Between Same Pair + +```python +BB1Edge = Concept("BB1Edge", extends=[Edge]) +BB2Edge = Concept("BB2Edge", extends=[Edge]) + +Bref = B.ref() +define(BB1Edge.new(src=B, dst=Bref)).where(B.field1 == Bref.id) +define(BB2Edge.new(src=B, dst=Bref)).where(B.field2 == Bref.id) +``` + +--- + +## Feature Configuration + +The `PropertyTransformer` annotates concept fields with their semantic types for the GNN. + +```python +pt = PropertyTransformer( + category=[User.locale, User.gender, Event.city, Event.state, Event.country], + datetime=[User.joinedAt, Event.start_time], + continuous=[User.birthyear], + time_col=[Event.start_time], +) +``` + +### Feature Type Guidelines + +| Data type | Annotation | +|-----------|-----------| +| Boolean flags, enum/status codes | `category` | +| Ages, prices, ratings | `continuous` | +| Free-form text, names, descriptions | `text` | +| Dates, timestamps | `datetime` | +| Explicit integer values (not IDs) | `integer` | + +The `integer` parameter is a distinct type from `continuous` -- use it for whole-number counts or ordinal values where float precision is not meaningful (e.g. review counts, position ranks): + +```python +pt = PropertyTransformer( + integer=[Review.num_votes, Standing.position], + continuous=[Review.rating, Result.points], + ... +) +``` + +### Feature Selection Strategy + +- **Drop all PKs and FKs.** Graph structure already captures relationships; IDs add noise. Example: `drop=[Study.nct_id, Outcome.id, Outcome.nct_id, ...]` +- **Start with minimal `text` fields.** Text embedding is expensive and too many text fields dilute signal. Begin with 3-5 key text fields, add more only if metrics improve. +- **Use `category` for discrete location/status fields.** Fields like city, state, country have limited cardinality. +- **Use `continuous` for numeric measurements.** Counts, scores, percentages. +- **Lean feature sets beat everything-in.** In practice, reducing ~30 text fields to 5 improved AUROC from 57% to 68%. + +### Graph metrics as features + +Centrality, community labels, and other graph-algorithm outputs from `rai-graph-analysis` can feed the GNN as features once they're materialized as concept properties. Compute the metric on a separate Graph instance (the algorithm graph -- often a different topology from the GNN graph), bind the result, then include in the PropertyTransformer: + +```python +# Algorithm graph (often a different topology from the GNN graph) +algo_graph = Graph(model, directed=False) +define(algo_graph.Edge.new(src=Source, dst=SourceRef)).where(...) + +# Bind metric output as a Concept property +Source.pagerank = model.Property(f"{Source} has {Float:pagerank}") +model.define(Source.pagerank(graph_algo_result)) + +# Include as a continuous (or category) feature +pt = PropertyTransformer( + continuous=[Source.pagerank, ...], + ... +) +``` + +Two-graph setups are common (the GNN graph and the algorithm graph have different shapes); name them distinctly to avoid confusion. + +PropertyTransformer is optional -- omitting it auto-infers all field types. For production, explicit annotation is recommended. Use `drop` to exclude fields or entire concepts: `drop=[Interaction, Item.internal_code]`. + +For the full feature type reference including drop patterns, see [references/property-transformer-types.md](references/property-transformer-types.md). + +--- + +## Common Pitfalls + +| Mistake | Cause | Fix | +|---------|-------|-----| +| Concept name is plural (e.g. "Customers") | Naming convention | Use singular names: `Concept("Customer")` | +| Task table concept has `identify_by` | Task tables don't need primary keys | Use plain `Concept("TrainTable")` with no `identify_by` | +| Snowflake table name not fully qualified | Missing database or schema prefix | Use `"DATABASE.SCHEMA.TABLE"` format | +| Test Relationship includes label/target | Test data should not contain the answer | Omit the "has" clause: `f"{Source}"` or `f"{Source} at {Any:ts}"` | +| Positional args in `define(Train(...))` don't match template | Template and population call must align | Match the order: source, [timestamp], [label/target] | +| Self-referential edge without `.ref()` | Same concept on both sides creates ambiguity | Use `PostRef = Post.ref()` for the destination | +| `time_col` fields not in `datetime` list | Both lists must include the field | Add time columns to both `datetime=[...]` and `time_col=[...]` | +| Task table concept used in edge definition | Only graph concepts participate in edges | Edges connect domain entities, not task tables | +| Missing type import | e.g. using `Date` without importing it | Add missing types to the import line | +| Column name has spaces or special characters | Python identifier rules prevent `Concept.weight(kg)` | Use `getattr(People, "weight(kg)")` to reference the field | +| `identify_by` key or property access doesn't match Snowflake column name | Typo or wrong column — matching is case-insensitive, but the column name must exist | Check `INFORMATION_SCHEMA.COLUMNS` / run `DESCRIBE TABLE` for the exact spelling | +| Train/Val/Test Relationships have different schemas | Test omits the label but also changes concept or timestamp structure | Train, Val, and Test must share the same concept and timestamp structure — only the label/target is omitted in Test | + +--- + +## Examples + +| Pattern | Description | File | +|---------|-------------|------| +| Node classification | Binary classification data model | [examples/node_classification_snowflake.py](examples/node_classification_snowflake.py) | +| Link prediction | Repeated link prediction data model | [examples/link_prediction_snowflake.py](examples/link_prediction_snowflake.py) | +| Regression | Regression-with-time data model | [examples/regression_snowflake.py](examples/regression_snowflake.py) | + +--- + +## Reference files + +| Reference | Description | File | +|-----------|-------------|------| +| Task relationships | Relationship template patterns for all task types with code examples | [references/task-relationships.md](references/task-relationships.md) | +| PropertyTransformer types | Full feature type reference, drop patterns, and guidelines | [references/property-transformer-types.md](references/property-transformer-types.md) | +| Auto-discovery | SQL templates for discovering PKs, FKs, edges, and task structure | [references/auto-discovery.md](references/auto-discovery.md) | diff --git a/skills/rai-predictive-modeling/examples/link_prediction_snowflake.py b/skills/rai-predictive-modeling/examples/link_prediction_snowflake.py new file mode 100644 index 0000000..c35a1d6 --- /dev/null +++ b/skills/rai-predictive-modeling/examples/link_prediction_snowflake.py @@ -0,0 +1,77 @@ +""" +GNN Link Prediction -- Data Modeling (Phases 1-6) +================================================= +Repeated link prediction on a bipartite User-Item graph with an Interaction +concept carrying timestamps. + +Demonstrates: concepts, population, task relationships (link prediction with +time), graph edges, and PropertyTransformer. + +For training and prediction, see `rai-predictive-training`. +""" + +# -- Phase 1: Imports & Model Setup -- +from relationalai.semantics import Model, select, define, Integer, Any +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +model = Model("gnn_link_prediction_example") +Concept, Table, Relationship = model.Concept, model.Table, model.Relationship + +# -- Phase 2: Define Concepts -- +# graph (node) concepts +User = Concept("User", identify_by={"user_id": Integer}) +Item = Concept("Item", identify_by={"item_id": Integer}) +Interaction = Concept("Interaction", identify_by={"interaction_id": Integer}) + +# task table concepts +train_table_concept = Concept("TrainTable") +val_table_concept = Concept("ValidationTable") +test_table_concept = Concept("TestTable") + +# -- Phase 3: Populate Concepts (from Snowflake) -- +define(User.new(Table("DB.SCHEMA.USERS").to_schema())) +define(Item.new(Table("DB.SCHEMA.ITEMS").to_schema())) +define(Interaction.new(Table("DB.SCHEMA.INTERACTIONS").to_schema())) + +define(train_table_concept.new(Table("DB.SCHEMA.TRAIN_LINK").to_schema())) +define(val_table_concept.new(Table("DB.SCHEMA.VAL_LINK").to_schema())) +define(test_table_concept.new(Table("DB.SCHEMA.TEST_LINK").to_schema())) + +# -- Phase 4: Setup Task Relationships -- repeated_link_prediction (with time) +# Train/Val carry the Target concept in the "has" clause (no {Any:label}). +# Test omits the target: the GNN predicts which Item each User links to. +Train = Relationship(f"{User} at {Any:timestamp} has {Item}") +define(Train(User, train_table_concept.timestamp, Item)).where( + User.user_id == train_table_concept.user_id, + Item.item_id == train_table_concept.item_id, +) + +Val = Relationship(f"{User} at {Any:timestamp} has {Item}") +define(Val(User, val_table_concept.timestamp, Item)).where( + User.user_id == val_table_concept.user_id, + Item.item_id == val_table_concept.item_id, +) + +Test = Relationship(f"{User} at {Any:timestamp}") +define(Test(User, test_table_concept.timestamp)).where( + User.user_id == test_table_concept.user_id, +) + +# -- Phase 5: Build Graph & Edges -- +gnn_graph = Graph(model, directed=True, weighted=False) +Edge = gnn_graph.Edge + +define(Edge.new(src=Interaction, dst=User)).where( + Interaction.user_id == User.user_id) +define(Edge.new(src=Interaction, dst=Item)).where( + Interaction.item_id == Item.item_id) + +# -- Phase 6: Configure PropertyTransformer -- +pt = PropertyTransformer( + category=[User.region, User.status, Item.category, Interaction.channel], + continuous=[User.age, Interaction.value], + text=[Item.name], + datetime=[Interaction.timestamp], + time_col=[Interaction.timestamp], +) diff --git a/skills/rai-predictive-modeling/examples/node_classification_snowflake.py b/skills/rai-predictive-modeling/examples/node_classification_snowflake.py new file mode 100644 index 0000000..39224fe --- /dev/null +++ b/skills/rai-predictive-modeling/examples/node_classification_snowflake.py @@ -0,0 +1,82 @@ +""" +GNN Node Classification -- Data Modeling (Phases 1-6) +===================================================== +Binary classification on user data from Snowflake with temporal features. +Demonstrates: concepts, population, task relationships, graph, and features. + +For training and prediction, see `rai-predictive-training`. +""" + +# -- Phase 1: Imports & Model Setup -- +from relationalai.semantics import Model, select, define, Integer, String, Any +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +model = Model("gnn_node_classification_example") +Concept, Table, Relationship = model.Concept, model.Table, model.Relationship + +# -- Phase 2: Define Concepts -- +# graph (node) concepts +User = Concept("User", identify_by={"user_id": Integer}) +Event = Concept("Event", identify_by={"event_id": Integer}) +EventAttendee = Concept("EventAttendee") + +# task table concepts +train_table_concept = Concept("TrainTable") +val_table_concept = Concept("ValidationTable") +test_table_concept = Concept("TestTable") + +# -- Phase 3: Populate Concepts (from Snowflake) -- +define(User.new(Table("DB.SCHEMA.USERS").to_schema())) +define(Event.new(Table("DB.SCHEMA.EVENTS").to_schema())) +define(EventAttendee.new(Table("DB.SCHEMA.EVENT_ATTENDEES").to_schema())) + +define(train_table_concept.new(Table("DB.SCHEMA.TRAIN").to_schema())) +define(val_table_concept.new(Table("DB.SCHEMA.VAL").to_schema())) +define(test_table_concept.new(Table("DB.SCHEMA.TEST").to_schema())) + +# -- Phase 4: Setup Task Relationships -- +Train = Relationship(f"{User} at {Any:timestamp} has {Any:target}") +define(Train(User, train_table_concept.timestamp, train_table_concept.target)).where( + User.user_id == train_table_concept.user_id +) + +Val = Relationship(f"{User} at {Any:timestamp} has {Any:target}") +define(Val(User, val_table_concept.timestamp, val_table_concept.target)).where( + User.user_id == val_table_concept.user_id +) + +Test = Relationship(f"{User} at {Any:timestamp}") +define(Test(User, test_table_concept.timestamp)).where( + User.user_id == test_table_concept.user_id +) + +# -- Phase 5: Build Graph & Edges -- +gnn_graph = Graph(model, directed=True, weighted=False) +Edge = gnn_graph.Edge + +define(Edge.new(src=Event, dst=User)).where( + Event.user_id == User.user_id) +define(Edge.new(src=EventAttendee, dst=Event)).where( + EventAttendee.event == Event.event_id) +define(Edge.new(src=EventAttendee, dst=User)).where( + EventAttendee.user_id == User.user_id) + +# -- Phase 6: Configure PropertyTransformer -- +category_user = [User.locale, User.gender] +datetime_user = [User.joinedAt] +continuous_user = [User.birthyear] + +category_event = [Event.city, Event.state, Event.zip, Event.country] +datetime_event = [Event.start_time] +continuous_event = [Event.lat, Event.lng] + +category_event_attendee = [EventAttendee.status] +datetime_event_attendee = [EventAttendee.start_time] + +pt = PropertyTransformer( + category=[*category_user, *category_event, *category_event_attendee], + datetime=[*datetime_user, *datetime_event, *datetime_event_attendee], + continuous=[*continuous_user, *continuous_event], + time_col=[Event.start_time], +) diff --git a/skills/rai-predictive-modeling/examples/regression_snowflake.py b/skills/rai-predictive-modeling/examples/regression_snowflake.py new file mode 100644 index 0000000..0550531 --- /dev/null +++ b/skills/rai-predictive-modeling/examples/regression_snowflake.py @@ -0,0 +1,85 @@ +""" +GNN Regression -- Data Modeling (Phases 1-6) +============================================= +Regression with temporal features on a bipartite User-Item graph. +The source concept (Interaction) carries the numeric target to predict. + +Demonstrates: concepts, population, regression task relationships with +`{Any:value}`, graph edges, and PropertyTransformer with time_col. + +For training and prediction, see `rai-predictive-training`. +""" + +# -- Phase 1: Imports & Model Setup -- +from relationalai.semantics import Model, select, define, Integer, Any +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +model = Model("gnn_regression_example") +Concept, Table, Relationship = model.Concept, model.Table, model.Relationship + +# -- Phase 2: Define Concepts -- +# graph concepts -- the source concept (the one being predicted on) needs its +# own primary key. If the source table lacks one, add a row_number column in +# Snowflake first (e.g. via a view or derived table). +User = Concept("User", identify_by={"user_id": Integer}) +Item = Concept("Item", identify_by={"item_id": Integer}) +Interaction = Concept("Interaction", identify_by={"interaction_id": Integer}) + +# task table concepts +train_table_concept = Concept("TrainTable") +val_table_concept = Concept("ValidationTable") +test_table_concept = Concept("TestTable") + +# -- Phase 3: Populate Concepts (from Snowflake) -- +define(User.new(Table("DB.SCHEMA.USERS").to_schema())) +define(Item.new(Table("DB.SCHEMA.ITEMS").to_schema())) +define(Interaction.new(Table("DB.SCHEMA.INTERACTIONS").to_schema())) + +define(train_table_concept.new(Table("DB.SCHEMA.TRAIN").to_schema())) +define(val_table_concept.new(Table("DB.SCHEMA.VAL").to_schema())) +define(test_table_concept.new(Table("DB.SCHEMA.TEST").to_schema())) + +# -- Phase 4: Setup Task Relationships -- regression (with time) +# Train/Val carry the numeric target in the "has" clause as {Any:value}. +# Test omits the target: the GNN predicts it. +Train = Relationship(f"{Interaction} at {Any:timestamp} has {Any:value}") +define(Train(Interaction, train_table_concept.timestamp, train_table_concept.value)).where( + Interaction.interaction_id == train_table_concept.interaction_id, +) + +Val = Relationship(f"{Interaction} at {Any:timestamp} has {Any:value}") +define(Val(Interaction, val_table_concept.timestamp, val_table_concept.value)).where( + Interaction.interaction_id == val_table_concept.interaction_id, +) + +Test = Relationship(f"{Interaction} at {Any:timestamp}") +define(Test(Interaction, test_table_concept.timestamp)).where( + Interaction.interaction_id == test_table_concept.interaction_id, +) + +# -- Phase 5: Build Graph & Edges -- +gnn_graph = Graph(model, directed=True, weighted=False) +Edge = gnn_graph.Edge + +define(Edge.new(src=Interaction, dst=User)).where( + Interaction.user_id == User.user_id, +) +define(Edge.new(src=Interaction, dst=Item)).where( + Interaction.item_id == Item.item_id, +) + +# -- Phase 6: Configure PropertyTransformer -- +# Drop PKs/FKs explicitly -- fields not listed in any category get auto-inferred +# as features, so PKs/FKs must be in `drop=[...]` to actually be excluded. +pt = PropertyTransformer( + category=[User.region, User.status, Item.category, Interaction.channel], + continuous=[User.age], + text=[Item.name], + datetime=[Interaction.timestamp], + time_col=[Interaction.timestamp], + drop=[ + User.user_id, Item.item_id, Interaction.interaction_id, + Interaction.user_id, Interaction.item_id, + ], +) diff --git a/skills/rai-predictive-modeling/references/auto-discovery.md b/skills/rai-predictive-modeling/references/auto-discovery.md new file mode 100644 index 0000000..aa56104 --- /dev/null +++ b/skills/rai-predictive-modeling/references/auto-discovery.md @@ -0,0 +1,176 @@ +# Auto-Discovery + +After the user provides table names, the agent automatically discovers schema details by querying Snowflake. This reference documents the conversation templates and discovery process. + +## How to use this workflow + +Walk through each phase **sequentially**. For each phase, use the **exact question template** below -- do not rephrase, reorder, or add extra questions. Wait for the user's answers before proceeding to the next phase. If the user provides information that covers multiple phases, acknowledge it and skip to the next uncovered phase. + +## Conversation Templates + +**Phase 1 is split into three sub-steps. Ask each one separately and wait for the user's response before moving to the next.** + +### Phase 1a -- Source Tables + +Ask exactly this: + +``` +Phase 1a: Source Tables + +What are your **source table** fully qualified names? +(e.g., `MY_DB.MY_SCHEMA.CUSTOMERS`, `MY_DB.MY_SCHEMA.TRANSACTIONS`) + +If you have a schema diagram or image, feel free to share it and I'll extract the details. +``` + +### Phase 1b -- Task Tables + +Ask exactly this (after user responds to 1a): + +``` +Phase 1b: Task Tables + +What are your **task table** fully qualified names for train/val/test? +(e.g., `MY_DB.TASKS.TRAIN`, `MY_DB.TASKS.VAL`, `MY_DB.TASKS.TEST`) +``` + +### Phase 1c -- Experiment Tracking + +Ask exactly this (after user responds to 1b): + +``` +Phase 1c: Experiment Tracking + +What Snowflake database and schema should we use for **experiment tracking**? +(e.g., `MY_DB.EXPERIMENTS`) +``` + +## What to Auto-Discover (and what NOT to ask) + +The user-input boundary is the 3 prompts above (source FQNs, task FQNs, experiment db and schema). **Do not ask the user** for column names, PKs, FKs, label/target columns, timestamp columns, task type, or feature types — those are friction the user often can't answer without checking the schema themselves. Use the in-skill helper below first (`get_table_schema(table_name, database, schema)`), then infer: + +1. **Column names and types** for all source and task tables +2. **Primary keys** -- identify PK columns +3. **Foreign key relationships** -- detect FK columns by matching column names across tables (e.g., `customer_id` in `TRANSACTIONS` matches `customer_id` PK in `CUSTOMERS`) +4. **Graph concepts** -- each source table becomes a concept (use singular form of table name) +5. **Edges** -- derived from FK relationships found above +6. **Task structure** -- from task table columns, infer: + - Join key (column matching a source concept PK) + - Label/target column (non-key, non-timestamp column) or target concept (for link prediction) + - Time column (columns with DATE/TIMESTAMP type) +7. **Task type** -- infer from the label column: + - Binary/boolean or 2-value categorical -> `binary_classification` + - Multi-value categorical -> `multiclass_classification` + - Multiple label columns for the same row, or array/list-of-labels target -> `multilabel_classification` + - Numeric/float -> `regression` + - Column matching another concept's PK -> `link_prediction` (ask user to confirm) + +**Note: multiclass vs multilabel** +- **Multiclass**: each row has exactly one label chosen from many classes (e.g., `sports` *or* `news` *or* `finance`). +- **Multilabel**: each row can have multiple labels at the same time (e.g., `sports` *and* `news`). + +### Required execution pattern (Snowpark first, then helper) + +Set up a Snowpark session once (follow `rai-setup`), then reuse it for schema discovery. + +Use this implementation directly: + +```python +import re +from relationalai.config import SnowflakeConnection, create_config +from snowflake import snowpark + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_]+$") +session: snowpark.Session = create_config().get_session(SnowflakeConnection) + + +def get_table_schema(table_name: str, database: str, schema: str) -> list[dict]: + """Return Snowflake table columns as [{'column_name': ..., 'data_type': ...}].""" + table_name = table_name.strip() + database = database.strip() + schema = schema.strip() + + if not table_name or not database or not schema: + return [{"error": "table_name, database, and schema are required and cannot be empty."}] + + for field_name, value in [("database", database), ("schema", schema), ("table_name", table_name)]: + if not _IDENTIFIER_RE.fullmatch(value): + return [{"error": f"Invalid {field_name}: '{value}'. Use only letters, numbers, and underscores."}] + + query = """ + SELECT COLUMN_NAME, DATA_TYPE + FROM {database}.INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = '{schema}' + AND TABLE_NAME = '{table}' + ORDER BY ORDINAL_POSITION + """.format( + database=database.upper(), + schema=schema.upper(), + table=table_name.upper(), + ) + + try: + rows = session.sql(query).collect() + except Exception as exc: + return [{"error": f"Snowflake query failed: {exc}"}] + + if not rows: + return [{"error": f"No columns found for {database}.{schema}.{table_name}. Check the name and permissions."}] + + return [{"column_name": row["COLUMN_NAME"], "data_type": row["DATA_TYPE"]} for row in rows] +``` + +For each user-provided fully qualified table name `DB.SCHEMA.TABLE`: + +1. Parse into `database=DB`, `schema=SCHEMA`, `table_name=TABLE`. +2. Call the in-skill helper `get_table_schema(table_name=TABLE, database=DB, schema=SCHEMA)`. +3. Treat the returned `column_name` values as canonical Snowflake column names (case-insensitive matching allowed for concept/property references, but spelling must match). +4. If the helper returns an `error`, retry once after uppercasing parts; if still failing, ask the user for `DESCRIBE TABLE` output for only the failing table. + +Use `DESCRIBE TABLE` / manual SQL only as fallback when the Snowpark helper cannot return schema. + +## Link Prediction Detection + +If link prediction is detected, after presenting the discovery summary, ask the user: + +``` +I detected a **link prediction** task. One more question: + +Are you predicting **new** links (connections that don't exist yet) or **repeated** interactions (e.g., a customer re-purchasing an item they've bought before)? + +- **New links** -> `link_prediction` +- **Repeated interactions** -> `repeated_link_prediction` +``` + +## Summary Table Template + +Present the discovery results to the user as a summary table for confirmation before proceeding: + +``` +Here's what I discovered from your tables: + +**Source Tables & Concepts:** +| Table | Concept | PK | Other Columns | +|-------|---------|-----|---------------| +| ... | ... | ... | ... | + +**Edges (FK relationships):** +| From | To | Join Condition | +|------|-----|---------------| +| ... | ... | ... | + +**Task Tables:** +| Split | Table | Join Key -> Concept | Label/Target | Time Column | +|-------|-------|-------------------|--------------|-------------| +| Train | ... | ... | ... | ... | +| Val | ... | ... | ... | ... | +| Test | ... | ... | ... (none) | ... | + +**Inferred task type:** `` + +Does this look correct? I'll proceed with this structure. +``` + +## Fallback + +If the helper cannot connect to Snowflake or auto-discovery fails, fall back to asking the user for `DESCRIBE TABLE` output (or column lists with types) for only the affected tables. diff --git a/skills/rai-predictive-modeling/references/property-transformer-types.md b/skills/rai-predictive-modeling/references/property-transformer-types.md new file mode 100644 index 0000000..a778552 --- /dev/null +++ b/skills/rai-predictive-modeling/references/property-transformer-types.md @@ -0,0 +1,66 @@ +# PropertyTransformer Feature Types + +The `PropertyTransformer` class specifies how concept fields are transformed into GNN-compatible features. + +## Feature Types + +| Type | PropertyTransformer kwarg | Description | Example fields | +|------|--------------------------|-------------|----------------| +| Category | `category=[...]` | Discrete categorical values (int or string) | Gender, product code, membership status | +| Continuous | `continuous=[...]` | Numeric continuous values (float) | Age, price, rating | +| Text | `text=[...]` | Text strings (embedded via language model) | Product name, description, comment | +| Datetime | `datetime=[...]` | Timestamps or dates | Transaction date, creation date | +| Integer | `integer=[...]` | Whole-number counts or ordinal values (not IDs) | Review counts, position ranks | +| Drop | `drop=[...]` | Exclude field from model entirely | Foreign keys, sensitive data, redundant IDs | + +## Special Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `time_col` | list or single | Time column(s) for temporal models. Must NOT appear in `drop`. When multiple concepts have date fields, list all of them: `time_col=[Interaction.timestamp, Session.started_at, Order.placed_at, ...]`. Fields in `time_col` must also appear in `datetime`. | + +## Usage + +```python +from relationalai.semantics.reasoners.predictive import PropertyTransformer + +pt = PropertyTransformer( + category=[User.status, Item.category, Interaction.channel], + continuous=[User.age, Interaction.value], + text=[Item.name], + datetime=[Interaction.timestamp], + drop=[User.internal_code, Item.legacy_sku], + time_col=[Interaction.timestamp], +) +``` + +## Drop Patterns + +### Drop specific fields +```python +drop=[Item.legacy_sku, Item.internal_code] +``` + +### Drop all fields of a concept (identifier columns) +```python +drop=[User] # drops all User fields (including primary key) +``` + +### Mixed: drop entire concept + specific fields from another +```python +drop=[Interaction, Item.legacy_sku, Item.internal_code] +``` + +## Default Behavior + +Fields not mentioned in any category are auto-inferred by the GNN engine (equivalent to the `Infer` embedding type). This is usually fine for most fields, but explicitly annotating them improves reproducibility. + +## Guidelines + +- **Primary key / identifier fields**: Usually `drop` (they don't carry predictive signal) +- **Foreign key join columns**: Usually `drop` (the graph structure captures the relationship) +- **Numeric IDs that encode meaning** (e.g. product_code): Use `category` +- **Free-form text**: Use `text` +- **Dates/timestamps**: Use `datetime`. If it's the temporal ordering column, also add to `time_col` +- **Boolean flags**: Use `category` +- **Continuous measurements**: Use `continuous` diff --git a/skills/rai-predictive-modeling/references/task-relationships.md b/skills/rai-predictive-modeling/references/task-relationships.md new file mode 100644 index 0000000..c871e0d --- /dev/null +++ b/skills/rai-predictive-modeling/references/task-relationships.md @@ -0,0 +1,126 @@ +# Task Relationships + +Relationships encode the task structure using a template string with three parts: +- **Head** = source concept (the concept being predicted on) +- **"at" clause** = optional timestamp field +- **"has" clause** = label (classification/regression) or target concept (link prediction) + +## Relationship Arity Rules + +| Task Type | Train/Val template | Test template | +|-----------|-------------------|---------------| +| classification (no time) | `f"{Source} has {Any:label}"` | `f"{Source}"` | +| classification (with time) | `f"{Source} at {Any:ts} has {Any:label}"` | `f"{Source} at {Any:ts}"` | +| regression (no time) | `f"{Source} has {Any:value}"` | `f"{Source}"` | +| regression (with time) | `f"{Source} at {Any:ts} has {Any:value}"` | `f"{Source} at {Any:ts}"` | +| link_prediction | `f"{Source} has {Target}"` | `f"{Source}"` | +| repeated_link_prediction | `f"{Source} at {Any:ts} has {Target}"` | `f"{Source} at {Any:ts}"` | + +## Node Classification (with time) + +```python +Train = Relationship(f"{User} at {Any:timestamp} has {Any:target}") +define(Train(User, train_table_concept.timestamp, train_table_concept.target)).where( + User.user_id == train_table_concept.user_id +) + +Val = Relationship(f"{User} at {Any:timestamp} has {Any:target}") +define(Val(User, val_table_concept.timestamp, val_table_concept.target)).where( + User.user_id == val_table_concept.user_id +) + +Test = Relationship(f"{User} at {Any:timestamp}") +define(Test(User, test_table_concept.timestamp)).where( + User.user_id == test_table_concept.user_id +) +``` + +## Node Classification (no time) + +```python +Train = Relationship(f"{User} has {Any:target}") +Val = Relationship(f"{User} has {Any:target}") +Test = Relationship(f"{User}") +``` + +## Regression (with time) + +Numeric target on the source concept (e.g. a per-row value). + +```python +Train = Relationship(f"{Interaction} at {Any:timestamp} has {Any:value}") +define(Train(Interaction, train_table_concept.timestamp, train_table_concept.value)).where( + Interaction.interaction_id == train_table_concept.interaction_id, +) + +Val = Relationship(f"{Interaction} at {Any:timestamp} has {Any:value}") +define(Val(Interaction, val_table_concept.timestamp, val_table_concept.value)).where( + Interaction.interaction_id == val_table_concept.interaction_id, +) + +Test = Relationship(f"{Interaction} at {Any:timestamp}") +define(Test(Interaction, test_table_concept.timestamp)).where( + Interaction.interaction_id == test_table_concept.interaction_id, +) +``` + +## Regression (no time) + +```python +Train = Relationship(f"{Interaction} has {Any:value}") +Val = Relationship(f"{Interaction} has {Any:value}") +Test = Relationship(f"{Interaction}") +``` + +## Link Prediction (with time / repeated_link_prediction) + +```python +Train = Relationship(f"{User} at {Any:timestamp} has {Item}") +define(Train(User, train_table_concept.timestamp, Item)).where( + User.user_id == train_table_concept.user_id, + Item.item_id == train_table_concept.item_id, +) + +Val = Relationship(f"{User} at {Any:timestamp} has {Item}") +define(Val(User, val_table_concept.timestamp, Item)).where( + User.user_id == val_table_concept.user_id, + Item.item_id == val_table_concept.item_id, +) + +Test = Relationship(f"{User} at {Any:timestamp}") +define(Test(User, test_table_concept.timestamp)).where( + User.user_id == test_table_concept.user_id, +) +``` + +## Link Prediction (no time) + +```python +Train = Relationship(f"{User} has {Item}") +Val = Relationship(f"{User} has {Item}") +Test = Relationship(f"{User}") +``` + +## Alternative: select() fragments + +Instead of `Relationship` + `define()`, you can use `select()` directly. Both forms are accepted by the GNN constructor: + +```python +Train = select(User, train_table_concept.timestamp, Item).where( + User.user_id == train_table_concept.user_id, + Item.item_id == train_table_concept.item_id, +) + +Val = select(User, val_table_concept.timestamp, Item).where( + User.user_id == val_table_concept.user_id, + Item.item_id == val_table_concept.item_id, +) + +Test = select(User, test_table_concept.timestamp).where( + User.user_id == test_table_concept.user_id, +) +``` + +## Post-training aggregation (rollup shape) + +A common real-world shape is: train the GNN on fine-grained events (e.g. a `Transaction` source), then aggregate predictions up to a coarser entity (e.g. `Article`) for downstream rules or optimization. This lives on the **consumption side**, not in the Relationship template -- see `rai-predictive-training` § Aggregation and bridge concepts for the `aggregates.(Source.predictions.).per(Target).where(...)` pattern. diff --git a/skills/rai-predictive-training/SKILL.md b/skills/rai-predictive-training/SKILL.md new file mode 100644 index 0000000..fdec466 --- /dev/null +++ b/skills/rai-predictive-training/SKILL.md @@ -0,0 +1,488 @@ +--- +name: rai-predictive-training +description: Configure and train GNN models, generate predictions, evaluate results, and manage trained models. Use after building the data model with rai-predictive-modeling, when ready to run training, evaluate, or manage GNN models. +--- + +# Predictive Training + + +> **Early access.** The RAI predictive reasoner (GNN) is in early access — APIs, engine requirements, and behavior may change. Confirm the latest surface with the RelationalAI team before production use. + +## Summary + +**What:** Training, evaluation, and model management workflow for GNN pipelines. + +**When to use:** +- Configuring the GNN estimator and hyperparameters +- Training models with `fit()` +- Generating predictions on test data +- Evaluating and debugging results +- Registering or loading saved models + +**When NOT to use:** +- Defining concepts, loading data, building graphs -- see `rai-predictive-modeling` + +**Overview:** 4 steps: configure GNN -> train -> predict/evaluate -> optional: register/load. + +**By user intent — sections to focus on:** +- Train + read validation metric → Quick Reference + GNN Constructor + `gnn.fit()` +- + predict + downstream rule / optimization → also Predictions + Using Predictions Downstream +- + register + reload across sessions → also Model Management + +## Quick Reference + +### Node Classification (minimal) + +```python +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + train=Train, validation=Val, + task_type="binary_classification", eval_metric="roc_auc", + has_time_column=True, device="cuda", n_epochs=5, +) +gnn.fit() +User.predictions = gnn.predictions(domain=Test) +``` + +### Default Metrics + +| Task Type | Suggested Metric | +|-----------|-----------------| +| binary_classification | `roc_auc` | +| multiclass_classification | `accuracy` | +| multilabel_classification | `multilabel_auprc_macro` | +| regression | `rmse` | +| link_prediction | `link_prediction_precision@5` | +| repeated_link_prediction | `link_prediction_precision@5` | + +### Prediction Attributes + +| Task Type | Attributes | +|-----------|-----------| +| classification | `.probs`, `.predicted_labels` | +| regression | `.predicted_value` | +| link prediction | `.rank`, `.scores`, `.predicted_` | + +--- + +## GNN Constructor + +### Required Parameters + +| Parameter | Description | +|-----------|-------------| +| `exp_database`, `exp_schema` | Snowflake location for experiment artifacts | +| `graph` | Graph object with edges defined | +| `train`, `validation` | Relationship objects | +| `task_type` | Task type string | +| `eval_metric` | Evaluation metric string | + +### Optional Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `property_transformer` | None | PropertyTransformer instance (omit for auto-inference) | +| `has_time_column` | False | Set `True` when Relationships use the "at" keyword | +| `dataset_alias` | None | Custom alias for the dataset | +| `stream_logs` | True | Stream training logs to console. Set `False` if log streaming is slow or unreliable — training continues server-side regardless | +| `parallel_reasoners_init` | True | Initialize reasoners in parallel at construction time | + +### Node Classification Example + +```python +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + train=Train, validation=Val, + task_type="binary_classification", + eval_metric="roc_auc", + has_time_column=True, + device="cuda", n_epochs=5, lr=0.005, +) +gnn.fit() +``` + +### Link Prediction Example (temporal) + +```python +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + train=Train, validation=Val, + task_type="repeated_link_prediction", + eval_metric="link_prediction_precision@5", + has_time_column=True, + device="cuda", n_epochs=5, lr=0.005, + head_layers=2, num_negative=20, label_smoothing=True, +) +gnn.fit() +``` + +**Note:** `gnn.fit()` trains at most once per GNN instance. If training has already completed (or is in progress), subsequent calls to `fit()` are silent no-ops. To retrain -- e.g. with different hyperparameters -- construct a new `GNN` instance. + +**Multi-GNN pipelines on the same model.** Train multiple GNNs over the same entity set (e.g. regression + classification + link-prediction on the same graph) by reusing one `Graph` and one `PropertyTransformer` across all `GNN` instances; vary `task_type`, `eval_metric`, `train`/`validation`, and the source/target concepts. Bind each task's predictions to a **distinct attribute name** -- the convention `Source.predictions` collides if one source concept hosts more than one task. + +```python +shared = dict(graph=gnn_graph, property_transformer=pt) +gnn_a = GNN(**shared, train=TrainA, validation=ValA, task_type="regression", eval_metric="rmse", ...) +gnn_b = GNN(**shared, train=TrainB, validation=ValB, task_type="binary_classification", eval_metric="roc_auc", ...) +gnn_c = GNN(**shared, train=TrainC, validation=ValC, task_type="repeated_link_prediction", eval_metric="link_prediction_precision@5", ...) +for g in (gnn_a, gnn_b, gnn_c): g.fit() + +# Distinct attributes when a source concept hosts multiple predictions: +Item.value_predictions = gnn_a.predictions(domain=TestA) +User.label_predictions = gnn_b.predictions(domain=TestB) +User.link_predictions = gnn_c.predictions(domain=TestC) +``` + +Hyperparameters can also be passed as a dictionary: + +```python +train_config = {"device": "cuda", "n_epochs": 10, "lr": 0.001, "train_batch_size": 512} +gnn = GNN(exp_database="DB", exp_schema="EXPERIMENTS", ..., **train_config) +gnn.fit() +``` + +--- + +## Common Hyperparameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `device` | `"cuda"` | `"cuda"` (GPU) or `"cpu"` | +| `n_epochs` | 5 | Number of training epochs | +| `lr` | 0.005 | Learning rate | +| `train_batch_size` | 256 | Training batch size | + +For link prediction, also consider: `head_layers=2`, `num_negative=20`, `label_smoothing=True`. + +**`device="cuda"` is a paired requirement.** The client-side flag alone is not enough — the predictive reasoner engine must also be GPU-sized in `raiconfig.yaml`. Configure both or neither; mismatched settings silently fall back or fail. Heuristic: CPU HIGHMEM tiers trade training speed for more RAM; GPU is faster per epoch when the dataset fits in the GPU VM's CPU memory, HIGHMEM otherwise. + +**A GNN workflow touches multiple reasoner engines — size each per its role.** At minimum, the **predictive** engine runs `fit()` and prediction jobs, and the **logic** engine runs model definitions, queries, and downstream rule evaluation; predict-then-optimize pipelines also need the **prescriptive** engine. Each is configured independently in `raiconfig.yaml` under `reasoners:` with its own `name` and `size` — appropriate sizing differs per role (GPU for predictive training, HIGHMEM CPU for logic query and rule workloads, and per-problem sizing for prescriptive). Mis-sizing one engine doesn't error loudly; the workflow still runs and silently under-performs or hits memory limits on that engine's step. + +**Auto-suspend during iteration.** Set a low `auto_suspend_mins` on every engine you're using — idle pool cost can dominate total spend on small workloads. Warm pools make sense only for scheduled/production cadence. Specific tier names and per-cloud memory-vs-compute tradeoffs change over time — ask the RelationalAI team for current sizing. Full `raiconfig.yaml` structure (including the `reasoners:` block for all engine types) lives in the RAI configuration/setup skill. + +For all hyperparameters and tuning guidance, see [references/hyperparameters.md](references/hyperparameters.md). + +--- + +## Training + +### fit() Stages + +`gnn.fit()` runs three stages internally: +1. Data preparation and feature extraction +2. Model training over `n_epochs` +3. Evaluation on the validation set + +--- + +## Predictions + +After training, generate predictions on the test set. Two valid binding patterns: + +```python +# Pattern 1 — bind to a concept attribute (queryable via select()): +Source.predictions = gnn.predictions(domain=Test) + +# Pattern 2 — assign to a plain Python variable (re-callable): +predictions = gnn.predictions(domain=Test) +``` + +Each concept-attribute name can be assigned **once per session** — re-binding `Source.predictions` raises `[Duplicate relationship]`. To call `predictions()` multiple times in one session, use Pattern 2 or a fresh attribute name (e.g. `Source.predictions_v2`). + +### Classification (binary, multiclass, multilabel) + +```python +User.predictions = gnn.predictions(domain=Test) + +select( + User.user_id, + User.predictions.probs, + User.predictions.predicted_labels, +).where(User.predictions).inspect() +``` + +### Regression + +```python +Unit.predictions = gnn.predictions(domain=Test) + +select( + Unit.unit_id, + Unit.predictions.predicted_value, +).where(Unit.predictions).inspect() +``` + +### Link Prediction + +```python +User.predictions = gnn.predictions(domain=Test) + +select( + User.user_id, + Item.item_id, + User.predictions.rank, + User.predictions.scores, +).where( + User.predictions.predicted_item == Item, +).inspect() +``` + +The `predicted_` attribute name is always lowercase: Target `Item` -> `.predicted_item`. + +### As DataFrame + +Replace `.inspect()` with `.to_df()` to get a pandas DataFrame: + +```python +df = select( + User.user_id, + User.predictions.probs, + User.predictions.predicted_labels, +).where(User.predictions).to_df() +``` + +### Dictionary-Style Field Indexing + +The prediction relation also supports dictionary-style field indexing, useful when the source concept name conflicts with an existing attribute: + +```python +PredRelation = gnn.predictions(domain=Test) +select( + PredRelation["beer"].name, + PredRelation["timestamp"], + PredRelation["prediction"].predicted_labels, + PredRelation["prediction"].probs, +).inspect() +``` + +**Direct access via `gnn.prediction_concept`.** Exposes the underlying prediction concept without binding to a source attribute — useful when the source concept name conflicts with an existing attribute. Use it as the head in `select(...)`: `select(Source.source_id, gnn.prediction_concept.predicted_labels).where(Source.predictions(DateTime, gnn.prediction_concept)).inspect()`. + +For the full prediction attributes reference (per-task attribute types, code shapes), see [references/prediction-attributes.md](references/prediction-attributes.md). The summary table is in Quick Reference above. + +--- + +## Using Predictions Downstream + +Once `Source.predictions = gnn.predictions(...)` runs, predictions are bound to the source concept and accessible via `Source.predictions.` throughout the **same `Model`**. Other reasoners (rules, prescriptive, graph) consume them by deriving new properties from those attributes. + +### Same-model pattern (default) + +Keep training, prediction, and downstream reasoning in one `Model`. This is the idiomatic RAI flow for predict-then-optimize and predict-then-rules chains: + +```python +# 1. Train and bind predictions +Item.predictions = gnn.predictions(domain=Test) + +# 2. Derive a regular property from the prediction +Item.predicted_value = model.Property(f"{Item} has {Float:predicted_value}") +model.define(Item.predicted_value(Item.predictions.predicted_value)) + +# 3a. Predictive -> Rules: boolean flag +Item.is_high = model.Relationship(f"{Item} is high") +model.where(Item.predicted_value > threshold).define(Item.is_high()) + +# 3b. Predictive -> Prescriptive: Item.predicted_value can appear in +# Problem(model, Float) constraint / objective expressions. +``` + +### Cross-session pattern (explicit persistence) + +If training and downstream reasoning run in separate processes, persist predictions to Snowflake and reload them as a fresh `Concept`: + +```python +# Training session: save predictions DataFrame +df = select(Source.source_id, Source.predictions.predicted_value) \ + .where(Source.predictions).to_df() +# Then write_pandas(conn, df, "MY_PREDICTIONS", auto_create_table=True, overwrite=True) +# Grant SELECT on MY_PREDICTIONS to APPLICATION RELATIONALAI. + +# Downstream session: load as a Concept in a new Model +Prediction = Concept("Prediction", identify_by={"source_id": Integer}) +model.define(Prediction.new(Table("DB.SCHEMA.MY_PREDICTIONS").to_schema())) +# Derive properties from Prediction, apply rules, run a solver, etc. +``` + +`database=` and `schema=` on `GNN(...)` are optional and omitted throughout this skill. For durable persistence, use the explicit `write_pandas` path above. + +### Aggregation and bridge concepts + +When the downstream reasoner's scope differs from the GNN source -- e.g. per-source predictions feeding a per-target optimizer -- aggregate predictions via `aggregates.(...).per(Target).where(join)` and attach the result to a **bridge concept** representing the downstream scope: + +```python +# GNN source predicts a value per Source (e.g. per-event regression); +# downstream scope is OptTarget, one row per coarser entity. +OptTarget = Concept("OptTarget", identify_by={"opt_target_id": Integer}) +OptTarget.total_predicted_value = model.Property(f"{OptTarget} has {Float:total_predicted_value}") + +agg = aggregates.sum(Source.predictions.predicted_value).per(OptTarget).where( + Interaction.target_id == OptTarget.opt_target_id, + Interaction.source_id == Source.source_id, +) +model.define(OptTarget.total_predicted_value(agg)) +``` + +The bridge concept (`OptTarget`) separates *what the GNN predicted at Source scope* from *what the downstream reasoner consumes at Target scope*. Skipping the bridge and trying to use `Source.predictions.predicted_value` directly in a Target-scoped constraint forces ad-hoc joins inside each rule or objective expression. For classification or link-prediction predictions, swap `sum`/`predicted_value` for `avg`/`probs` or `count`/`scores` per the rule below. + +**Choose the aggregation function by target shape.** Use `sum` for additive or count-like predictions (per-event regression values rolled up to an entity total), `avg` for proportional or probability-like predictions (mean predicted score across related source entities), `count` for link-prediction hits. Mixing them produces values that look numerically fine but don't mean what downstream expects. + +**Non-additive blending of multiple signals** (e.g. combining several GNN outputs, or a GNN probability with a rule-derived flag) is also a derived-property step, not a built-in `aggregates.`. Express it as ordinary arithmetic in the property definition: a multiplicative composite (`predicted_a * (1 - w * avg_b) * (1 + w * avg_c)`) for risk-uplift-style logic, or a weighted interpolation (`alpha * rule_signal + (1 - alpha) * gnn_probs`) for hybrid scoring. Keep the bridge concept distinct from the GNN source so the blend is a regular Property the downstream reasoner can consume. + +**Denormalize if the target was pre-scaled at training.** If the training target was normalized (e.g. to `[0, 1]`, or z-scored), raw predictions carry that scale too. Record the denormalization factor alongside the derived property and apply it before feeding into constraints or objectives that expect real-world units -- otherwise the downstream reasoner sees tiny numbers where it expected the real-world quantity. + +For a full predict-then-optimize example chaining multiple GNNs into optimizers with bridge + aggregation, see the `retail_planning` template in the templates repo. + +--- + +## Evaluation & Debugging + +After `gnn.fit()`, inspect what data the engine received: + +```python +# Visual schema with data types (requires pydot; omit show_dtypes for the simple variant) +graph_viz = gnn.visualize_dataset(show_dtypes=True) +graph_viz.write_png("dataset_schema.png") + +# Full metadata dict (debugging feature types) and data-config printout +config = gnn.dataset.metadata_dict +gnn.dataset.print_data_config() +``` + +If results are poor, see [references/evaluation-debugging.md](references/evaluation-debugging.md) § Tuning Poor Results for the ordered checklist (dataset inspection → text-feature reduction → hyperparameter tuning) plus regression-specific sanity checks, multi-metric framing, and leakage diagnostics. + +--- + +## Model Management + +### Register a Model + +After `gnn.fit()` completes: + +```python +gnn.register_model( + model_database="DB", + model_schema="MODEL_REGISTRY", + model_name="my_predictor", + version_name="v1", + comment="Initial training run", # optional +) +``` + +### Load by Registry Key + +```python +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + source_concept=User, + task_type="binary_classification", + has_time_column=True, + model_database="DB", model_schema="MODEL_REGISTRY", + model_name="my_predictor", version_name="v1", +) +gnn.load() +User.predictions = gnn.predictions(domain=Test) +``` + +### Load by Run ID + +Same as above, replacing the registry key params with `model_run_id=""`. + +### What to Include vs. Omit When Loading + +| Include | Omit | +|---------|------| +| `exp_database`, `exp_schema` | `database`, `schema` (now optional) | +| `graph`, `property_transformer` | `train`, `validation` | +| `source_concept` (required) | `eval_metric` | +| `task_type` (required) | hyperparameters (`device`, `n_epochs`, etc.) | +| `has_time_column=True` (if model was trained with time column) | | +| `target_concept` (required for link prediction only) | | +| model identifier (registry key or run ID) | | + +### Train-Register-Load Workflow + +**Session 1: Train and Register** + +```python +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + train=Train, validation=Val, + task_type="binary_classification", eval_metric="roc_auc", + has_time_column=True, device="cuda", n_epochs=5, +) +gnn.fit() +gnn.register_model( + model_database="DB", model_schema="MODEL_REGISTRY", + model_name="my_predictor", version_name="v1", +) +``` + +**Session 2: Load and Predict** + +```python +# Rebuild graph and property_transformer (same structure as training) +gnn_graph = Graph(model, directed=True, weighted=False) +# ... define edges ... +pt = PropertyTransformer(...) + +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + source_concept=User, + task_type="binary_classification", + has_time_column=True, + model_database="DB", model_schema="MODEL_REGISTRY", + model_name="my_predictor", version_name="v1", +) +gnn.load() +User.predictions = gnn.predictions(domain=Test) +``` + +--- + +## Common Pitfalls + +| Mistake | Cause | Fix | +|---------|-------|-----| +| Missing `has_time_column=True` | Templates with the "at" keyword require the flag so the trainer finds the time column | Set `has_time_column=True` when templates contain "at" | +| Using `.predicted_Item` (uppercase) | Target-attribute names are always lowercased from the Target concept name | Use `.predicted_item` | +| Invalid `task_type`/`eval_metric` combination | Not every metric applies to every task type | Check [references/task-types-and-metrics.md](references/task-types-and-metrics.md) for valid pairs | +| `register_model()` before `fit()` | Registration requires a trained model | Always call `gnn.fit()` before `gnn.register_model()` | +| `model_name` or `version_name` with spaces or special characters (e.g. `"my model"`, `"v1.0!"`) | Snowflake rejects non-identifier strings as model names, but validation only happens after full training completes | Use plain alphanumeric names with underscores only (e.g. `"my_model"`, `"V1"`) | +| Calling `register_model()` with a `(model_name, version_name)` pair that already exists in the registry | The registry enforces uniqueness — duplicate versions raise `ModelManagerError` | Use a new `version_name` (e.g. `"V2"`) or delete the existing version first | +| Calling `register_model()` on a GNN instance created in load mode | Load-mode GNN instances cannot re-register — only fit-mode instances can register models | Call `register_model()` on the `fit_gnn` instance after `fit()`, not on the `gnn` instance after `load()` | +| Omitting `graph`/`property_transformer` when loading | Load reconstructs against the same schema used during training | Provide the same `graph` and `property_transformer` used during training | +| Passing training-only params when loading | Load ignores training-time params | Omit `train`, `validation`, and hyperparameters when loading | +| Omitting `source_concept` when loading | Required to bind the loaded model to the source concept for prediction | Add `source_concept=` to the load constructor | +| Omitting `task_type` when loading | Not persisted in the registry | Add `task_type=""` to the load constructor | +| Omitting `target_concept` for link-prediction load | Required to resolve the prediction target concept | Add `target_concept=` for link prediction | +| Omitting `has_time_column` when loading a temporal model | Not persisted in the registry | Re-supply `has_time_column=True` at load time | +| Calling `fit()` on a GNN instance created in load mode | Load-mode GNN instances do not support training | Create a separate fit-mode GNN instance (with `train=`, `validation=`) and call `fit()` on that | +| Calling `load()` on a GNN instance created in fit mode (with `train=`, `validation=`) | Fit-mode GNN instances do not support `load()` | Create a separate load-mode GNN instance (with `source_concept=`, `model_name=`, `version_name=`) and call `load()` on that | +| Experiment schema not accessible by the RAI native app | RAI app needs explicit grants to write to the experiment schema | `GRANT USAGE ON DATABASE TO APPLICATION RELATIONALAI; GRANT USAGE ON SCHEMA . TO APPLICATION RELATIONALAI; GRANT CREATE EXPERIMENT ON SCHEMA . TO APPLICATION RELATIONALAI` | + +--- + +## Examples + +| Pattern | Description | File | +|---------|-------------|------| +| Node classification | Binary classification training + prediction | [examples/train_node_classification.py](examples/train_node_classification.py) | +| Link prediction | Repeated link prediction training + prediction | [examples/train_link_prediction.py](examples/train_link_prediction.py) | +| Regression | Regression training + prediction | [examples/train_regression.py](examples/train_regression.py) | +| Register and load | Complete train-register-load workflow across sessions | [examples/register_and_load.py](examples/register_and_load.py) | + +--- + +## Reference Files + +| Reference | Description | File | +|-----------|-------------|------| +| Task types and metrics | All valid (task_type, eval_metric) combinations | [references/task-types-and-metrics.md](references/task-types-and-metrics.md) | +| Hyperparameters | Full hyperparameter table with types, defaults, and tuning guidance | [references/hyperparameters.md](references/hyperparameters.md) | +| Prediction attributes | Prediction attributes by task type with usage examples | [references/prediction-attributes.md](references/prediction-attributes.md) | +| Evaluation & debugging | Dataset inspection, result checking, and tuning steps | [references/evaluation-debugging.md](references/evaluation-debugging.md) | diff --git a/skills/rai-predictive-training/evals/evals.json b/skills/rai-predictive-training/evals/evals.json new file mode 100644 index 0000000..3c877a8 --- /dev/null +++ b/skills/rai-predictive-training/evals/evals.json @@ -0,0 +1,18 @@ +{ + "skill_name": "rai-predictive-training", + "evals": [ + { + "id": 1, + "prompt": "Build me a churn predictor for H&M. Source tables: HM_MINI.PUBLIC.CUSTOMERS, HM_MINI.PUBLIC.ARTICLES, HM_MINI.PUBLIC.TRANSACTIONS_DEDUP. Task tables: HM_MINI.TASK_CHURN.{TRAIN, VAL, TEST}. Experiment artifacts go to F1_DB.F1_DID_NOT_FINISH. Walk through the rai-predictive-modeling and rai-predictive-training skills end-to-end: ask me only for what you actually need.\n\nReply with two sections:\n\n1. Script -- paste the full script source in one ```python``` code block. I cannot read your filesystem; only what you paste into this reply is visible to me, so a file path is not enough. Single runnable file: Concepts, Table-backed populates, Train/Val/Test Relationships, Graph + Edges, PropertyTransformer, GNN, fit, predictions on Test, final inspect.\n\n2. Run output -- first run `cat ` and then `python ` in the same shell, and paste the full terminal output of both commands. The `cat` output must appear before the `python` output. Do not stop until both jobs show JOB_COMPLETED and the prediction rows are visible.", + "expected_output": "The agent follows the two skills in order (modeling -> training) and asks only for high-level information that cannot be auto-discovered: the source/target task type if ambiguous, hyperparameter preferences (epochs, device), and confirmation of the label column name on the task tables. It does NOT ask for column lists, primary keys, foreign keys, or feature types -- those come from Snowflake schema introspection. It then produces a single end-to-end script with all phases (Concepts, Table-backed populates, Train/Val/Test Relationships using 'at', Graph + Edges, PropertyTransformer, GNN(...) with has_time_column=True, gnn.fit(), Customer.predictions = gnn.predictions(domain=Test), and a final select(...).inspect()). When executed against a live reasoner, the script runs to completion: training reaches JOB_COMPLETED, prediction reaches JOB_COMPLETED, and the final select returns a non-empty DataFrame with columns (c_customer_id, probs, predicted_labels) where probs are in [0, 1] and predicted_labels are in {0, 1}.", + "expectations": [ + "The produced script contains all phases in order: Concept declarations (Customer, Article, Transaction, task table concepts without identify_by), Table-backed define() calls, Train/Val/Test Relationships with 'at {Any:timestamp}' (Train/Val also 'has {Any:label}', Test omits the label), Graph with Edges, PropertyTransformer, GNN constructor, gnn.fit(), gnn.predictions(domain=Test) bound to Customer.predictions, and a final select(...).where(Customer.predictions).inspect()", + "Script executes without raising any Python exception or PyRel compile error end-to-end", + "Training job reaches JOB_COMPLETED status (visible in streamed logs); the agent does not declare success on JOB_START or partial logs", + "Prediction job reaches JOB_COMPLETED status and predictions are loaded back into the logic engine", + "Final select(...).inspect() (or .to_df()) returns a non-empty result with columns (c_customer_id, probs, predicted_labels), probs in [0, 1], predicted_labels in {0, 1}", + "The script does not introduce manual job polling, sleep loops, or status-check API calls around fit() or predictions() -- both are blocking and stream logs by default" + ] + } + ] +} diff --git a/skills/rai-predictive-training/examples/register_and_load.py b/skills/rai-predictive-training/examples/register_and_load.py new file mode 100644 index 0000000..76745bd --- /dev/null +++ b/skills/rai-predictive-training/examples/register_and_load.py @@ -0,0 +1,53 @@ +""" +GNN Model Management -- Register and Load Workflow +==================================================== +Demonstrates the train-register-load pattern across sessions. + +Session 1: Train a model and register it to Snowflake Model Registry. +Session 2: Load the registered model and generate predictions. +""" +from relationalai.semantics.reasoners.predictive import GNN + +# -- Session 1: Train and Register ------------------------------------------- +# Assumes data model from `rai-predictive-modeling`: +# gnn_graph, pt, Train, Val, Test, User + +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + train=Train, validation=Val, + task_type="binary_classification", eval_metric="roc_auc", + has_time_column=True, + device="cuda", n_epochs=5, +) +gnn.fit() + +gnn.register_model( + model_database="DB", + model_schema="MODEL_REGISTRY", + model_name="my_predictor", + version_name="v1", + comment="Initial training run", +) + + +# -- Session 2: Load and Predict --------------------------------------------- +# Rebuild graph and PropertyTransformer (same structure as training session) +# gnn_graph = Graph(model, directed=True, weighted=False) +# ... define edges ... +# pt = PropertyTransformer(...) + +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, property_transformer=pt, + source_concept=User, + task_type="binary_classification", + has_time_column=True, + model_database="DB", + model_schema="MODEL_REGISTRY", + model_name="my_predictor", + version_name="v1", +) +gnn.load() + +User.predictions = gnn.predictions(domain=Test) diff --git a/skills/rai-predictive-training/examples/train_link_prediction.py b/skills/rai-predictive-training/examples/train_link_prediction.py new file mode 100644 index 0000000..14ebacb --- /dev/null +++ b/skills/rai-predictive-training/examples/train_link_prediction.py @@ -0,0 +1,46 @@ +""" +GNN Link Prediction -- Training & Prediction +============================================== +Repeated link prediction training and prediction. + +Assumes data model from `rai-predictive-modeling`: + - gnn_graph: Graph with edges defined + - pt: PropertyTransformer instance (passed as `property_transformer=pt`) + - Train, Val, Test: Relationship objects + - User: source concept, Item: target concept +""" +from relationalai.semantics import select +from relationalai.semantics.reasoners.predictive import GNN + +# -- Train GNN --------------------------------------------------------------- +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, + property_transformer=pt, + train=Train, + validation=Val, + task_type="repeated_link_prediction", + eval_metric="link_prediction_precision@5", + has_time_column=True, + device="cuda", + n_epochs=5, + train_batch_size=256, + lr=0.005, + head_layers=2, + num_negative=20, + label_smoothing=True, +) +gnn.fit() + +# -- Predict & Inspect ------------------------------------------------------- +# .predicted_ attribute name is always lowercase: Target `Item` -> .predicted_item +User.predictions = gnn.predictions(domain=Test) + +select( + User.user_id, + Item.item_id, + User.predictions.rank, + User.predictions.scores, +).where( + User.predictions.predicted_item == Item, +).inspect() diff --git a/skills/rai-predictive-training/examples/train_node_classification.py b/skills/rai-predictive-training/examples/train_node_classification.py new file mode 100644 index 0000000..405ac32 --- /dev/null +++ b/skills/rai-predictive-training/examples/train_node_classification.py @@ -0,0 +1,45 @@ +""" +GNN Node Classification -- Training & Prediction +================================================== +Binary classification training and prediction on user data. + +Assumes data model from `rai-predictive-modeling`: + - gnn_graph: Graph with edges defined + - pt: PropertyTransformer instance (passed as `property_transformer=pt`) + - Train, Val, Test: Relationship objects + - User: source concept (head of Relationship template) +""" +from relationalai.semantics import select +from relationalai.semantics.reasoners.predictive import GNN + +# -- Train GNN --------------------------------------------------------------- +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, + property_transformer=pt, + train=Train, + validation=Val, + task_type="binary_classification", + eval_metric="roc_auc", + has_time_column=True, + device="cuda", + n_epochs=5, +) +gnn.fit() + +# -- Predict & Inspect ------------------------------------------------------- +User.predictions = gnn.predictions(domain=Test) + +select( + User.user_id, + User.predictions.probs, + User.predictions.predicted_labels, +).where(User.predictions).inspect() + +df = select( + User.user_id, + User.predictions.probs, + User.predictions.predicted_labels, +).where(User.predictions).to_df() + +print(f"Predictions: {len(df)} rows, {len(df.dropna())} after dropping NaNs") diff --git a/skills/rai-predictive-training/examples/train_regression.py b/skills/rai-predictive-training/examples/train_regression.py new file mode 100644 index 0000000..2ca1d42 --- /dev/null +++ b/skills/rai-predictive-training/examples/train_regression.py @@ -0,0 +1,46 @@ +""" +GNN Regression -- Training & Prediction +======================================== +Regression training and prediction with temporal features. + +Assumes data model from `rai-predictive-modeling`: + - gnn_graph: Graph with edges defined + - pt: PropertyTransformer instance (passed as `property_transformer=pt`) + - Train, Val, Test: Relationship objects + - Interaction: source concept (head of Relationship template) +""" +from relationalai.semantics import select +from relationalai.semantics.reasoners.predictive import GNN + +# -- Train GNN --------------------------------------------------------------- +# Regression typically needs more epochs than classification. Start with 20-50; +# 5 (the classification default) is a smoke test and usually plateaus at the mean. +gnn = GNN( + exp_database="DB", exp_schema="EXPERIMENTS", + graph=gnn_graph, + property_transformer=pt, + train=Train, + validation=Val, + task_type="regression", + eval_metric="rmse", + has_time_column=True, + device="cuda", + n_epochs=20, + lr=0.005, +) +gnn.fit() + +# -- Predict & Inspect ------------------------------------------------------- +Interaction.predictions = gnn.predictions(domain=Test) + +select( + Interaction.interaction_id, + Interaction.predictions.predicted_value, +).where(Interaction.predictions).inspect() + +df = select( + Interaction.interaction_id, + Interaction.predictions.predicted_value, +).where(Interaction.predictions).to_df() + +print(f"Predictions: {len(df)} rows") diff --git a/skills/rai-predictive-training/references/evaluation-debugging.md b/skills/rai-predictive-training/references/evaluation-debugging.md new file mode 100644 index 0000000..7c42ebd --- /dev/null +++ b/skills/rai-predictive-training/references/evaluation-debugging.md @@ -0,0 +1,138 @@ +# Evaluation & Debugging + +Detailed patterns for inspecting datasets, checking prediction results, and tuning model performance. + +## Inspecting the Dataset + +After `gnn.fit()`, inspect what data the engine received: + +```python +# Visual graph of the dataset schema (requires pydot) +graph_viz = gnn.visualize_dataset() +graph_viz.write_png("dataset_schema.png") + +# With data types shown +graph_viz = gnn.visualize_dataset(show_dtypes=True) +graph_viz.write_png("dataset_schema.png") +``` + +### Metadata and Data Config + +```python +# Export full metadata as a dictionary (useful for debugging feature types) +config = gnn.dataset.metadata_dict + +# Print the data config to console +gnn.dataset.print_data_config() +``` + +### Prediction-step timing expectations + +`gnn.predictions(...)` runs a 4-step sequence (prepare test table -> load model -> submit prediction job -> load results into the logic engine). This sequence carries fixed overhead independent of test-set size, so small test sets still incur meaningful wall-clock time. Subsequent predictions in the same session are faster due to caching; fresh `GNN` instances re-pay the full cost. Don't optimize feature choices based on a first-run prediction time. + +## Accessing Prediction Results + +### Via Source Concept Attribute + +```python +Source.predictions = gnn.predictions(domain=Test) + +select( + Source.id, + Source.predictions.probs, + Source.predictions.predicted_labels, +).where(Source.predictions).inspect() +``` + +### Via prediction_concept + +Access the underlying prediction concept directly -- useful when you need to reference it without binding it to a source concept attribute: + +```python +PredResult = gnn.prediction_concept +select(Source.source_id, PredResult.predicted_labels, PredResult.probs).where( + Source.predictions(DateTime, PredResult) +).inspect() +``` + +### Dictionary-Style Field Indexing + +Useful when the source concept name conflicts with an existing attribute: + +```python +PredRelation = gnn.predictions(domain=Test) +select( + PredRelation["beer"].name, + PredRelation["timestamp"], + PredRelation["prediction"].predicted_labels, + PredRelation["prediction"].probs, +).inspect() +``` + +## Evaluating Results + +### What "good" means + +Ultimately a prediction is good if it supports the business question. That's the ground truth. Business-utility is hard to measure upfront, though, so training-time evaluation relies on intrinsic metrics as proxies. Pick the proxy that most resembles downstream use -- RMSE if the answer is a numeric value, Spearman rho if the answer is a rank-ordering, recall-at-precision if the answer is a gated decision -- and triangulate with the others. + +### Reading the training loss + +`gnn.fit()` prints per-epoch train and validation loss. The trajectory diagnoses training health before any test-set metric: + +| Loss pattern | Likely cause | Action | +|--------------|-------------|--------| +| Both losses still decreasing at the last epoch | Not converged | Train longer (bump `n_epochs`) | +| Train loss decreasing, val loss plateau or rising | Overfitting | Stop earlier, reduce capacity, or add regularization | +| Both losses flat at a high value | Under-capacity, weak features, or LR too small | Check features; try a larger `lr` | +| Long plateau then step-change improvement | Model just learned a structural pattern | Keep training past the plateau -- best epoch may come late | + +### Use multiple metrics + +No single number describes quality. Before trusting predictions, look at: + +- **Task metric vs a predict-baseline.** Predict-mean for regression, majority-class for classification. Compute the lift: `(baseline - model) / baseline`. Near-zero lift means the model hasn't learned anything useful. +- **Correlation** (Pearson + Spearman). Can be high even when absolute error is poor -- the model may have learned ranking but not magnitudes. +- **Prediction-range vs target-range.** `stddev(predicted) / stddev(target)`. A tight prediction band means the model is hedging toward the mean. +- **Error distribution.** Look at the residual histogram, not just aggregates -- a few huge errors can dominate RMSE while most predictions are fine. +- **Sanity-check the prediction DataFrame before using it downstream.** After `.to_df()`, verify the predicted column is free of NaN and stays in the expected range (`predicted_value >= 0` for non-negative targets, `probs` in `[0, 1]` for classification, `scores` non-null for link prediction). Silent NaN/garbage can propagate through a derived property or optimizer constraint and surface as a cryptic solver failure later. + + Pattern (warn-not-block — keeps the pipeline running while flagging suspicious output): + + ```python + df = select(Source.id, Source.predictions.).where(Source.predictions).to_df() + col = df[""] + if col.isna().any() or (col < 0).any(): # adjust bounds per task type + print(f"WARNING: {Source.__name__} predictions contain NaN or out-of-range values") + ``` + + Run a check per GNN in a multi-GNN pipeline; cheap and catches silent failures before they reach derived properties or solver constraints. + +## Tuning Poor Results + +If results are significantly worse than expected, check these in order: + +1. **Inspect the dataset** -- run `gnn.visualize_dataset(show_dtypes=True)` and `gnn.dataset.print_data_config()` to verify feature types and edges match expectations. +2. **Reduce text features** -- too many text fields dilute signal. Start with 3-5 key text fields, add more only if metrics improve. In practice, reducing ~30 text fields to 5 improved AUROC from 57% to 68%. +3. **Adjust hyperparameters** -- see [hyperparameters.md](hyperparameters.md) "Tuning When Results Are Poor" section for symptom-based guidance. + +### Regression-specific sanity checks + +Regression typically needs **more epochs than classification** -- `n_epochs=5` (the quickstart default) is a smoke-test, not a training run. For a first real attempt, bump well above the default and let the loss trajectory (see "Reading the training loss" above) tell you when to stop -- if val-loss is still decreasing at the last epoch, you need more. + +**Under-fitting checklist** (cheapest diagnostic first): + +- **Profile the target distribution before training** -- `SELECT MIN, MAX, AVG, STDDEV FROM ` anchors what RMSE values mean. The same RMSE that's tight on a [0,1]-normalized target is meaningless on an unnormalized one. +- **Val-RMSE vs `stddev(target)`** -- if val-RMSE plateaus at or above the target's stddev, the model has collapsed to the mean. +- **Prediction-band vs target-band** -- if `stddev(predicted)` is noticeably narrower than `stddev(target)`, the model is hedging toward the mean. Under-trained regardless of RMSE. +- **Ranking vs magnitudes** -- if Pearson/Spearman correlation is moderate (>0.3) but RMSE doesn't beat the predict-mean baseline, the model has learned *ranking* but not *magnitudes*. This is under-fitting, not a feature problem -- train longer. +- **R² < 0 early in training is normal** -- it clears as the model learns the target's scale. If it persists past the early training phase, revisit features or learning rate. + +### Suspiciously-good results + +If a first-pass GNN returns R² > 0.95 (regression), AUROC > 0.98, or accuracy > 0.95 (classification), pause and check for leakage before trusting the model: + +- Is the target/label column also listed in the `PropertyTransformer` (category/continuous/...) by accident? +- Is a feature a near-duplicate of the label (a derived property that encodes the target)? +- Does the train/val/test split share entities in ways that let the model memorize — e.g., the same source entity appears in all three splits with the label tied to that entity? Especially common in `repeated_link_prediction`, where the same (source, target) pair can recur across splits. + +Strong features can legitimately produce high scores, but a cheap verification pass prevents shipping a leaky model. diff --git a/skills/rai-predictive-training/references/hyperparameters.md b/skills/rai-predictive-training/references/hyperparameters.md new file mode 100644 index 0000000..3058603 --- /dev/null +++ b/skills/rai-predictive-training/references/hyperparameters.md @@ -0,0 +1,91 @@ +# GNN Hyperparameters + +Hyperparameters are passed as `**train_params` kwargs to the `GNN(...)` constructor. + +## Common Hyperparameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `device` | str | `"cuda"` | Compute device: `"cuda"` (GPU) or `"cpu"` | +| `n_epochs` | int | 5 | Number of training epochs | +| `lr` | float | 0.005 | Learning rate | +| `train_batch_size` | int | 256 | Training batch size | +| `head_layers` | int | 2 | Number of prediction head layers | +| `seed` | int | - | Random seed for reproducibility | +| `channels` | int | 64 | Hidden channel dimension | + +## Link Prediction Hyperparameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `num_negative` | int | 20 | Number of negative samples per positive | +| `label_smoothing` | bool | True | Apply label smoothing during training | + +## Advanced Hyperparameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `temporal_strategy` | str | - | Temporal modeling strategy (e.g. `"last"`) | +| `text_embedder` | str | - | Text embedding model (e.g. `"model2vec-potion-base-4M"`) | +| `max_iters` | int | - | Maximum training iterations | + +## GNN Constructor Operational Flags + +These are named parameters on `GNN(...)`, not train_params: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `test_batch_size` | int | None | Batch size for prediction/inference | +| `stream_logs` | bool | True | Stream training logs to console | +| `use_current_time` | bool | True | Use current time for temporal models | + +## Example Configurations + +### Node Classification (small dataset) +```python +train_params = {"device": "cpu", "n_epochs": 10, "seed": 42} +``` + +### Node Classification (large dataset) +```python +train_params = {"device": "cuda", "n_epochs": 5, "lr": 0.005, "train_batch_size": 256} +``` + +### Link Prediction +```python +train_params = { + "device": "cuda", + "n_epochs": 5, + "train_batch_size": 256, + "lr": 0.005, + "head_layers": 2, + "num_negative": 20, + "label_smoothing": True, +} +``` + +### Regression with Temporal Data +```python +train_params = { + "device": "cuda", + "n_epochs": 5, + "channels": 64, + "head_layers": 2, + "temporal_strategy": "last", +} +``` + +## Tuning When Results Are Poor + +Two heuristics before the symptom table: + +- **`lr` is usually the first knob to sweep.** The default is a starting point, not a recommendation. If training isn't producing learning (flat losses, no convergence), try `lr` above and below the default before concluding features are the problem. +- **Message-passing depth vs graph diameter.** A GNN propagates signal one hop per layer (or one level per neighbor-sampling step). If the source concept sits far from the concepts carrying predictive signal in the schema, the model's depth must reach them -- otherwise distant nodes never contribute. The depth parameter is passed through `train_params` to the trainer; inspect the trainer's accepted kwargs (e.g. on `gnn.trainer` after construction) to find the exact name. + +| Symptom | Likely cause | Action | +|---------|-------------|--------| +| Validation metric still improving at last epoch | Not enough training | Increase `n_epochs` | +| Training loss oscillates or diverges | Learning rate too high | Lower `lr` | +| Good training metric, poor validation metric | Overfitting | Reduce `n_epochs`, reduce text features, or increase `train_batch_size` | +| Very slow convergence on large dataset | Batch too small or lr too high | Increase `train_batch_size`, decrease `lr` | +| Poor results despite hyperparameter sweeps | Signal can't reach the source concept | Check the graph depth matches the schema's diameter; otherwise reduce noisy features (drop PKs/FKs, trim text fields) | diff --git a/skills/rai-predictive-training/references/prediction-attributes.md b/skills/rai-predictive-training/references/prediction-attributes.md new file mode 100644 index 0000000..a32a305 --- /dev/null +++ b/skills/rai-predictive-training/references/prediction-attributes.md @@ -0,0 +1,67 @@ +# Prediction Attributes by Task Type + +After calling `gnn.predictions(domain=Test)`, the prediction results are attached to the source concept (head of the Relationship) and accessed via `select(...)`. + +## Classification (binary, multiclass, multilabel) + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Source.predictions.probs` | float/array | Probability distribution over classes | +| `Source.predictions.predicted_labels` | int/str | Predicted class label (argmax of probs) | + +```python +Source.predictions = gnn.predictions(domain=Test) +select( + Source.id, + Source.predictions.probs, + Source.predictions.predicted_labels, +).where(Source.predictions).inspect() +``` + +## Regression + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Source.predictions.predicted_value` | float | Predicted continuous value | + +```python +Source.predictions = gnn.predictions(domain=Test) +select( + Source.id, + Source.predictions.predicted_value, +).where(Source.predictions).inspect() +``` + +## Link Prediction (link_prediction, repeated_link_prediction) + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Source.predictions.rank` | int | Ranking position (1, 2, 3, ...) | +| `Source.predictions.scores` | float | Relevance/similarity score | +| `Source.predictions.predicted_` | reference | Predicted target concept instance | + +The `predicted_` attribute name is derived from the target concept in the Relationship template. For example, if the Relationship tail is `Item`, the attribute is `predicted_item`. + +```python +Source.predictions = gnn.predictions(domain=Test) +select( + Source.source_id, + Target.target_id, + Source.predictions.rank, + Source.predictions.scores, +).where( + Source.predictions.predicted_target == Target, +).inspect() +``` + +## Using `.to_df()` Instead of `.inspect()` + +Replace `.inspect()` with `.to_df()` to get a pandas DataFrame: + +```python +df = select( + Source.id, + Source.predictions.probs, + Source.predictions.predicted_labels, +).where(Source.predictions).to_df() +``` diff --git a/skills/rai-predictive-training/references/task-types-and-metrics.md b/skills/rai-predictive-training/references/task-types-and-metrics.md new file mode 100644 index 0000000..b1b0751 --- /dev/null +++ b/skills/rai-predictive-training/references/task-types-and-metrics.md @@ -0,0 +1,68 @@ +# Task Types and Evaluation Metrics + +Valid `(task_type, eval_metric)` combinations for the GNN constructor. + +## Binary Classification + +| task_type | eval_metric | +|-----------|-------------| +| `"binary_classification"` | `"accuracy"` | +| `"binary_classification"` | `"f1"` | +| `"binary_classification"` | `"roc_auc"` | +| `"binary_classification"` | `"average_precision"` | + +## Multiclass Classification + +| task_type | eval_metric | +|-----------|-------------| +| `"multiclass_classification"` | `"accuracy"` | +| `"multiclass_classification"` | `"macro_f1"` | +| `"multiclass_classification"` | `"micro_f1"` | + +## Multilabel Classification + +| task_type | eval_metric | +|-----------|-------------| +| `"multilabel_classification"` | `"multilabel_auprc_micro"` | +| `"multilabel_classification"` | `"multilabel_auroc_micro"` | +| `"multilabel_classification"` | `"multilabel_precision_micro"` | +| `"multilabel_classification"` | `"multilabel_auprc_macro"` | +| `"multilabel_classification"` | `"multilabel_auroc_macro"` | +| `"multilabel_classification"` | `"multilabel_precision_macro"` | + +## Regression + +| task_type | eval_metric | +|-----------|-------------| +| `"regression"` | `"r2"` | +| `"regression"` | `"mae"` | +| `"regression"` | `"rmse"` | + +## Link Prediction + +| task_type | eval_metric | +|-----------|-------------| +| `"link_prediction"` | `"link_prediction_precision@k"` | +| `"link_prediction"` | `"link_prediction_recall@k"` | +| `"link_prediction"` | `"link_prediction_map@k"` | + +## Repeated Link Prediction (temporal) + +| task_type | eval_metric | +|-----------|-------------| +| `"repeated_link_prediction"` | `"link_prediction_precision@k"` | +| `"repeated_link_prediction"` | `"link_prediction_recall@k"` | +| `"repeated_link_prediction"` | `"link_prediction_map@k"` | + +`@k` is optional. Omit it to evaluate without a top-k cutoff (e.g. `"link_prediction_precision"`), or append a value to restrict to the top k results (e.g. `"link_prediction_precision@5"`). + +## Task Type Summary + +| Task Type | has_time_column | Train Relationship template | Test Relationship template | +|-----------|-----------------|---------------------------|--------------------------| +| binary_classification | optional | `f"{Source} has {Any:label}"` | `f"{Source}"` | +| multiclass_classification | optional | `f"{Source} has {Any:label}"` | `f"{Source}"` | +| multilabel_classification | optional | `f"{Source} has {Any:label}"` | `f"{Source}"` | +| regression | optional | `f"{Source} has {Any:value}"` | `f"{Source}"` | +| link_prediction | False | `f"{Source} has {Target}"` | `f"{Source}"` | +| repeated_link_prediction | True | `f"{Source} at {Any:ts} has {Target}"` | `f"{Source} at {Any:ts}"` |