From 58eb2d471ba38703baf505b92fa2d4bf15c28b79 Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:36:10 -0400 Subject: [PATCH 1/9] optimize: remove dead code and fix dangling refs (agent 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pull-x.mjs: drop unused `dirname` import - fetch_x.py: drop unused HANDLE constant and _slug() (and the now-orphaned `re` import that _slug was the sole consumer of) - weak-model-test.mjs: write results to tests/, not examples/ — the file has only ever lived in tests/, all three docs point there, and the steipete twin already does this correctly - karpathy/README.md: ../../pull-x.mjs did not exist; point at its real home - .gitignore: add __pycache__/ and *.pyc (the repo's own documented verify command generates them) --- .gitignore | 2 ++ examples/garry-tan/scripts/fetch_x.py | 6 ------ examples/karpathy/README.md | 2 +- examples/karpathy/scripts/weak-model-test.mjs | 2 +- examples/steipete/scripts/pull-x.mjs | 1 - 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index e43b0f9..2348c91 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .DS_Store +__pycache__/ +*.pyc diff --git a/examples/garry-tan/scripts/fetch_x.py b/examples/garry-tan/scripts/fetch_x.py index 63db7ff..48fccbd 100644 --- a/examples/garry-tan/scripts/fetch_x.py +++ b/examples/garry-tan/scripts/fetch_x.py @@ -27,22 +27,16 @@ import json import os import pathlib -import re import sys import time import urllib.parse import urllib.request -HANDLE = "garrytan" USER_ID = "19301828" # @garrytan, stable numeric id OUT = pathlib.Path(__file__).parent.parent / "data" / "x" OUT.mkdir(parents=True, exist_ok=True) -def _slug(s: str, n: int = 40) -> str: - return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")[:n] - - def _write(tweet_id: str, date: str, text: str, source: str, reply_to: str | None) -> None: safe_date = date.split("T", 1)[0] path = OUT / f"{safe_date}_{tweet_id}.md" diff --git a/examples/karpathy/README.md b/examples/karpathy/README.md index f13ac44..9534360 100644 --- a/examples/karpathy/README.md +++ b/examples/karpathy/README.md @@ -115,7 +115,7 @@ Re-run the pipeline to refresh: ```bash bash scripts/fetch-data.sh # blogs + YouTube + GitHub -SURF_API_KEY=sk-surf-... node ../../pull-x.mjs # X refresh (see data/x/README.md) +SURF_API_KEY=sk-surf-... node ../steipete/scripts/pull-x.mjs # X refresh (see data/x/README.md) ``` --- diff --git a/examples/karpathy/scripts/weak-model-test.mjs b/examples/karpathy/scripts/weak-model-test.mjs index 019398e..4a61a69 100644 --- a/examples/karpathy/scripts/weak-model-test.mjs +++ b/examples/karpathy/scripts/weak-model-test.mjs @@ -304,7 +304,7 @@ if (openaiKey) { const result = await runTest(apiKey, model, backend); // Write results markdown -const resultPath = join(ROOT, 'examples', 'weak-model-results.md'); +const resultPath = join(ROOT, 'tests', 'weak-model-results.md'); let md = `# Weak Model Test Results\n\n`; md += `**Model**: \`${result.model}\` (${result.backend})\n`; md += `**Date**: ${new Date().toISOString().split('T')[0]}\n`; diff --git a/examples/steipete/scripts/pull-x.mjs b/examples/steipete/scripts/pull-x.mjs index 26f3821..12b4a7c 100644 --- a/examples/steipete/scripts/pull-x.mjs +++ b/examples/steipete/scripts/pull-x.mjs @@ -5,7 +5,6 @@ import { spawn } from 'child_process'; import { writeFileSync, mkdirSync } from 'fs'; -import { dirname } from 'path'; const API_KEY = process.env.SURF_API_KEY || ''; if (!API_KEY) { console.error('SURF_API_KEY required'); process.exit(1); } From 16fa9597f87844d6b46a1de74ee67d5d149a9cba Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:37:18 -0400 Subject: [PATCH 2/9] optimize: collapse stale legacy paths (agent 7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CONTRIBUTING/SECURITY: the soul-builder skill is BUILD.md, not SKILL.md. Fossil from when SKILL.md was deleted (0800de6) and later re-added in a different role (d750ab3); both docs were written against the middle state. - BUILD.md: list examples/bad-outputs.md in Output — the template exists and both README and CONTRIBUTING require it. - garry-tan/weak_model_test.md: the documented command could not run (no --prompts flag; --style/--good/--bad are required). Replaced with the verified invocation from scores_gpt-4o-mini.md. - garry-tan/weak_model_test.md: Status claimed the live run was pending; it shipped 2026-04-14 at 38.5/48. Corrected, and 13 prompts -> 12 (48 max / 4). --- .github/CONTRIBUTING.md | 2 +- .github/SECURITY.md | 3 ++- BUILD.md | 1 + examples/garry-tan/tests/weak_model_test.md | 22 +++++++++++---------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 9c1bde8..5f42319 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -12,7 +12,7 @@ reading it could predict its subject's take on a new topic.** or [`STYLE.template.md`](../STYLE.template.md), improve a `_GUIDE.md`, fix a factual error or broken link. -Build your soul first with the [soul-builder skill](../SKILL.md) (`/soul-builder`) +Build your soul first with the [soul-builder skill](../BUILD.md) (`/soul-builder`) or by hand from the templates. See [`examples/_GUIDE.md`](../examples/_GUIDE.md) for what goes in the calibration files. diff --git a/.github/SECURITY.md b/.github/SECURITY.md index e8d817d..d829019 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -24,7 +24,8 @@ that part isn't a secret and we'd rather act on it fast. Please include what you can: -- The file or component affected — `SKILL.md` (the builder), a `scripts/*.mjs` +- The file or component affected — `BUILD.md` (the builder), `SKILL.md` (the + embodier), a `scripts/*.mjs` helper, a template, or a specific `examples//` soul. - A minimal reproduction (for injection: the source text that triggers it). - The impact you can demonstrate. diff --git a/BUILD.md b/BUILD.md index 5129c7f..5037f49 100644 --- a/BUILD.md +++ b/BUILD.md @@ -189,6 +189,7 @@ When done, you should have created: - `STYLE.md` — Their voice - `MEMORY.md` — Empty memory log - `examples/good-outputs.md` — Calibration examples +- `examples/bad-outputs.md` — Negative calibration examples - Optionally: `data/influences.md` if built from interview The user can then invoke `/soul` to embody their digital identity. diff --git a/examples/garry-tan/tests/weak_model_test.md b/examples/garry-tan/tests/weak_model_test.md index dede91f..30c986e 100644 --- a/examples/garry-tan/tests/weak_model_test.md +++ b/examples/garry-tan/tests/weak_model_test.md @@ -48,20 +48,20 @@ For each response, grade: - **Stance match**: 0–2 (opinion aligned with SOUL.md) - **Anti-pattern hits**: subtract 1 per `bad-outputs.md` tell that appears (e.g., "that's a great question," emoji spam, hedging stack). -Formula: `score = (voice + stance) - anti_pattern_hits`. Max 4 per prompt. Pass threshold: **average ≥ 3.0 across 13 prompts**. +Formula: `score = (voice + stance) - anti_pattern_hits`. Max 4 per prompt. Pass threshold: **average ≥ 3.0 across 12 prompts**. ### 5. Run script ```bash -# OPENAI_API_KEY required -python tests/run_weak_model.py \ - --model gpt-4o-mini \ - --prompts tests/prediction_test.md,tests/platform_tests.md \ - --soul /tmp/garry_prompt.md \ +OPENROUTER_API_KEY=... python3 tests/run_weak_model.py \ + --model openai/gpt-4o-mini \ + --soul SOUL.md --style STYLE.md \ + --good examples/good-outputs.md --bad examples/bad-outputs.md \ --out tests/results_gpt-4o-mini.md ``` -`run_weak_model.py` is a 30-line stub below. +The prompts are defined in `run_weak_model.py` itself; `--style`, `--good`, and +`--bad` are required. --- @@ -155,7 +155,9 @@ Run it, grade it, commit the result. A passing run is part of the deliverable. - ✅ Protocol defined - ✅ Prompts authored (prediction_test.md + platform_tests.md) -- ✅ Run script reference implementation (above — saveable as-is) -- ⏳ Live run pending: requires `OPENAI_API_KEY`. Grading is manual until we add an LLM-judge step. +- ✅ Run script shipped: `tests/run_weak_model.py` +- ✅ Live run 2026-04-14 on `openai/gpt-4o-mini`: 38.5/48 (PASS). Full transcript + in [`results_gpt-4o-mini.md`](results_gpt-4o-mini.md), per-prompt grades in + [`scores_gpt-4o-mini.md`](scores_gpt-4o-mini.md). -When run live, append actual transcript + scores below. +Grading is manual until we add an LLM-judge step. From bceebf759d150695e50bc3a83dc5f84142617f5e Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:38:30 -0400 Subject: [PATCH 3/9] optimize: stop silent failures from corrupting the corpus (agent 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit karpathy/fetch-data.sh used `curl -sL` without -f, so HTTP errors were written to disk as the response body and curl still exited 0 — the author's own `|| echo "[WARN]"` could never fire, and the `[ -f ]` cache guard meant the poisoned file was never retried. Verified against the live URL: `curl -sL` exits 0 and writes a 9379-byte 404 page; `curl -fsSL` exits 56 and writes nothing. This already corrupted three committed files (data fix is separate): - data/writing/software-2.0.html -> GitHub Pages 404 (9379 B) - data/github/karpathy_char-rnn_README.md -> "404: Not Found" (14 B) - steipete data/x/steipete.replies.json -> [] (2 B) Changes: - karpathy/fetch-data.sh: -sL -> -fsSL --max-time 30 on both fetch sites; cache guards now also test -s. This also revives the master->main README fallback, which was unreachable because the first curl never failed. - karpathy/fetch-data.sh: stop discarding yt-dlp stderr; "No subs" was asserted for what may have been a network or geo failure. - steipete/fetch-data.sh: bare `|| true` -> report the failure. - pull-x.mjs: a non-JSON line on the MCP stdout channel is a protocol violation; log it instead of discarding it silently. - both weak-model-test.mjs: send errors to stderr, not stdout. --- examples/karpathy/scripts/fetch-data.sh | 12 ++++++------ examples/karpathy/scripts/weak-model-test.mjs | 6 +++--- examples/steipete/scripts/fetch-data.sh | 2 +- examples/steipete/scripts/pull-x.mjs | 2 +- examples/steipete/scripts/weak-model-test.mjs | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/karpathy/scripts/fetch-data.sh b/examples/karpathy/scripts/fetch-data.sh index be79ff3..4cc6136 100644 --- a/examples/karpathy/scripts/fetch-data.sh +++ b/examples/karpathy/scripts/fetch-data.sh @@ -31,11 +31,11 @@ set -- $BLOG_URLS for slug in $BLOG_SLUGS; do url="$1"; shift outfile="$DATA/writing/${slug}.html" - if [ -f "$outfile" ]; then + if [ -f "$outfile" ] && [ -s "$outfile" ]; then echo " [cached] $slug" else echo " [fetch] $slug" - curl -sL "$url" -o "$outfile" || echo " [WARN] Failed to fetch $slug" + curl -fsSL --max-time 30 "$url" -o "$outfile" || echo " [WARN] Failed to fetch $slug" sleep 0.5 fi done @@ -73,7 +73,7 @@ if command -v yt-dlp &>/dev/null; then echo " [fetch] $vid" yt-dlp --write-auto-sub --sub-lang en --skip-download \ --output "$DATA/transcripts/${vid}" \ - "https://www.youtube.com/watch?v=${vid}" 2>/dev/null || echo " [WARN] No subs for $vid" + "https://www.youtube.com/watch?v=${vid}" || echo " [WARN] Failed: $vid" sleep 1 fi done @@ -114,12 +114,12 @@ REPOS=( for repo in "${REPOS[@]}"; do slug="${repo//\//_}" outfile="$DATA/github/${slug}_README.md" - if [ -f "$outfile" ]; then + if [ -f "$outfile" ] && [ -s "$outfile" ]; then echo " [cached] $repo" else echo " [fetch] $repo" - curl -sL "https://raw.githubusercontent.com/${repo}/master/README.md" -o "$outfile" 2>/dev/null \ - || curl -sL "https://raw.githubusercontent.com/${repo}/main/README.md" -o "$outfile" 2>/dev/null \ + curl -fsSL --max-time 30 "https://raw.githubusercontent.com/${repo}/master/README.md" -o "$outfile" \ + || curl -fsSL --max-time 30 "https://raw.githubusercontent.com/${repo}/main/README.md" -o "$outfile" \ || echo " [WARN] Failed to fetch README for $repo" sleep 0.3 fi diff --git a/examples/karpathy/scripts/weak-model-test.mjs b/examples/karpathy/scripts/weak-model-test.mjs index 4a61a69..f5a5fc5 100644 --- a/examples/karpathy/scripts/weak-model-test.mjs +++ b/examples/karpathy/scripts/weak-model-test.mjs @@ -187,7 +187,7 @@ HARD RULES: }); const data = await response.json(); if (data.error) { - console.log(` ERROR: ${JSON.stringify(data.error)}`); + console.error(` ERROR: ${JSON.stringify(data.error)}`); results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); continue; } @@ -211,7 +211,7 @@ HARD RULES: }); const data = await response.json(); if (data.error) { - console.log(` ERROR: ${JSON.stringify(data.error)}`); + console.error(` ERROR: ${JSON.stringify(data.error)}`); results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); continue; } @@ -263,7 +263,7 @@ HARD RULES: antiFound }); } catch (err) { - console.log(` FETCH ERROR: ${err.message}`); + console.error(` FETCH ERROR: ${err.message}`); results.push({ ...test, score: 0, error: err.message }); } } diff --git a/examples/steipete/scripts/fetch-data.sh b/examples/steipete/scripts/fetch-data.sh index 8da50e9..b22dde4 100755 --- a/examples/steipete/scripts/fetch-data.sh +++ b/examples/steipete/scripts/fetch-data.sh @@ -97,7 +97,7 @@ for entry in "${BLOG_POSTS[@]}"; do done # Index page (lists posts) -[ -f "$DATA/writing/index.html" ] || curl -fsSL "https://steipete.me/" -o "$DATA/writing/index.html" || true +[ -f "$DATA/writing/index.html" ] || curl -fsSL "https://steipete.me/" -o "$DATA/writing/index.html" || echo " ⚠ failed: index.html" echo "" diff --git a/examples/steipete/scripts/pull-x.mjs b/examples/steipete/scripts/pull-x.mjs index 12b4a7c..8a6014b 100644 --- a/examples/steipete/scripts/pull-x.mjs +++ b/examples/steipete/scripts/pull-x.mjs @@ -35,7 +35,7 @@ mcp.stdout.on('data', (d) => { pending.delete(msg.id); resolve(msg); } - } catch (e) { /* ignore */ } + } catch { console.error(`non-JSON line from surf-mcp: ${line}`); } } }); diff --git a/examples/steipete/scripts/weak-model-test.mjs b/examples/steipete/scripts/weak-model-test.mjs index 1adf11d..2f6c4b8 100644 --- a/examples/steipete/scripts/weak-model-test.mjs +++ b/examples/steipete/scripts/weak-model-test.mjs @@ -177,7 +177,7 @@ HARD RULES: }); const data = await response.json(); if (data.error) { - console.log(` ERROR: ${JSON.stringify(data.error)}`); + console.error(` ERROR: ${JSON.stringify(data.error)}`); results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); continue; } @@ -201,7 +201,7 @@ HARD RULES: }); const data = await response.json(); if (data.error) { - console.log(` ERROR: ${JSON.stringify(data.error)}`); + console.error(` ERROR: ${JSON.stringify(data.error)}`); results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); continue; } @@ -254,7 +254,7 @@ HARD RULES: hasAnchor }); } catch (err) { - console.log(` FETCH ERROR: ${err.message}`); + console.error(` FETCH ERROR: ${err.message}`); results.push({ ...test, score: 0, error: err.message }); } } From ca3c600859efab1b31e0dcf7ea3904d53bb96b0e Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:39:19 -0400 Subject: [PATCH 4/9] optimize: collapse duplicated fetch branches (agent 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both weak-model-test.mjs had an if/else whose two arms were byte-identical except for the endpoint URL. Hoisted the URL into a ternary and kept one fetch. -48 lines, no behavior change. Deliberately NOT consolidating the two scripts into a shared module despite a 24.76% clone rate: each examples// folder is meant to be copied out whole, its docs use folder-relative commands with no repo-root dependency, and the parts that would need parameterizing (the 12 prompts, system prompt, scoring regexes) are the persona itself. jscpd found 0 clones between the two fetch-data.sh files — those are independently written, not copy-paste. --- examples/karpathy/scripts/weak-model-test.mjs | 74 +++++++------------ examples/steipete/scripts/weak-model-test.mjs | 74 +++++++------------ 2 files changed, 50 insertions(+), 98 deletions(-) diff --git a/examples/karpathy/scripts/weak-model-test.mjs b/examples/karpathy/scripts/weak-model-test.mjs index f5a5fc5..eba2e36 100644 --- a/examples/karpathy/scripts/weak-model-test.mjs +++ b/examples/karpathy/scripts/weak-model-test.mjs @@ -167,56 +167,32 @@ HARD RULES: console.log(`\n--- [${test.id}] ${test.topic} ---`); try { - let output; - if (backend === 'openai') { - const response = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - max_tokens: 500, - temperature: 0.7, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: test.prompt } - ] - }) - }); - const data = await response.json(); - if (data.error) { - console.error(` ERROR: ${JSON.stringify(data.error)}`); - results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); - continue; - } - output = data.choices[0].message.content; - } else { - const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - max_tokens: 500, - temperature: 0.7, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: test.prompt } - ] - }) - }); - const data = await response.json(); - if (data.error) { - console.error(` ERROR: ${JSON.stringify(data.error)}`); - results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); - continue; - } - output = data.choices[0].message.content; + const endpoint = backend === 'openai' + ? 'https://api.openai.com/v1/chat/completions' + : 'https://openrouter.ai/api/v1/chat/completions'; + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model, + max_tokens: 500, + temperature: 0.7, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: test.prompt } + ] + }) + }); + const data = await response.json(); + if (data.error) { + console.error(` ERROR: ${JSON.stringify(data.error)}`); + results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); + continue; } + const output = data.choices[0].message.content; // Structured scoring: Voice (0-2) + Stance (0-2) - Anti-pattern penalty (max -1) const lower = output.toLowerCase(); diff --git a/examples/steipete/scripts/weak-model-test.mjs b/examples/steipete/scripts/weak-model-test.mjs index 2f6c4b8..ec3385d 100644 --- a/examples/steipete/scripts/weak-model-test.mjs +++ b/examples/steipete/scripts/weak-model-test.mjs @@ -157,56 +157,32 @@ HARD RULES: console.log(`\n--- [${test.id}] ${test.topic} ---`); try { - let output; - if (backend === 'openai') { - const response = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - max_tokens: 600, - temperature: 0.7, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: test.prompt } - ] - }) - }); - const data = await response.json(); - if (data.error) { - console.error(` ERROR: ${JSON.stringify(data.error)}`); - results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); - continue; - } - output = data.choices[0].message.content; - } else { - const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - max_tokens: 600, - temperature: 0.7, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: test.prompt } - ] - }) - }); - const data = await response.json(); - if (data.error) { - console.error(` ERROR: ${JSON.stringify(data.error)}`); - results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); - continue; - } - output = data.choices[0].message.content; + const endpoint = backend === 'openai' + ? 'https://api.openai.com/v1/chat/completions' + : 'https://openrouter.ai/api/v1/chat/completions'; + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model, + max_tokens: 600, + temperature: 0.7, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: test.prompt } + ] + }) + }); + const data = await response.json(); + if (data.error) { + console.error(` ERROR: ${JSON.stringify(data.error)}`); + results.push({ ...test, score: 0, error: JSON.stringify(data.error) }); + continue; } + const output = data.choices[0].message.content; const lower = output.toLowerCase(); From b5360a63d13fbedcc18219be650d93d910360b37 Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:40:31 -0400 Subject: [PATCH 5/9] optimize: remove comment noise (agent 8) - weak-model-test.mjs: the scoring comment said "Anti-pattern penalty (max -1)" but the code is Math.min(antiFound.length, 2) -> max -2. README:95 and the script's own generated methodology both say -2; the comment was the outlier. - Drop comments that only restate the line below ("Also check for general Karpathy voice markers", "Anti-pattern penalty", "Write results markdown", "// Main"). Kept the ones carrying intent, e.g. the scoring rubric. - steipete: trim the inline "leverage" note to the part that is true; it promised a verb/noun distinction the code never implemented. - check-links.mjs: fix a dangling article ("the markdownlint handles"). Not done: the "# --- N. Title ---" dividers in fetch-data.sh. They mirror the echo below them but serve a different reader, and they are what makes a 174-line shell script navigable. --- examples/karpathy/scripts/weak-model-test.mjs | 6 +----- examples/steipete/scripts/weak-model-test.mjs | 4 +--- scripts/check-links.mjs | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/examples/karpathy/scripts/weak-model-test.mjs b/examples/karpathy/scripts/weak-model-test.mjs index eba2e36..9dc1aee 100644 --- a/examples/karpathy/scripts/weak-model-test.mjs +++ b/examples/karpathy/scripts/weak-model-test.mjs @@ -194,7 +194,7 @@ HARD RULES: } const output = data.choices[0].message.content; - // Structured scoring: Voice (0-2) + Stance (0-2) - Anti-pattern penalty (max -1) + // Structured scoring: Voice (0-2) + Stance (0-2) - Anti-pattern penalty (max -2) const lower = output.toLowerCase(); const signalsFound = test.expectedSignals.filter(s => lower.includes(s.toLowerCase())); @@ -205,7 +205,6 @@ HARD RULES: if (signalsFound.length >= 2) voice = 2; else if (signalsFound.length >= 1) voice = 1; - // Also check for general Karpathy voice markers const voiceMarkers = /\b(from scratch|let's (build|think|see|code)|the bitter lesson|next-token|okay so|this is the way|loss went down|interesting|the thing is|basically|spelled out|under the hood|what (most )?people miss)\b/i; if (voice < 2 && voiceMarkers.test(output)) voice = Math.min(2, voice + 1); @@ -217,7 +216,6 @@ HARD RULES: if (!hasHedge && (hasSpecificity || hasOpinion)) stance = 2; else if (hasSpecificity || hasOpinion || !hasHedge) stance = 1; - // Anti-pattern penalty const antiPenalty = Math.min(antiFound.length, 2); const score = Math.max(0, voice + stance - antiPenalty); @@ -256,7 +254,6 @@ HARD RULES: return { model, backend, totalScore, maxScore, avgScore, results, passThreshold }; } -// Main const openaiKey = process.env.OPENAI_API_KEY; const openrouterKey = process.env.OPENROUTER_API_KEY; @@ -279,7 +276,6 @@ if (openaiKey) { const result = await runTest(apiKey, model, backend); -// Write results markdown const resultPath = join(ROOT, 'tests', 'weak-model-results.md'); let md = `# Weak Model Test Results\n\n`; md += `**Model**: \`${result.model}\` (${result.backend})\n`; diff --git a/examples/steipete/scripts/weak-model-test.mjs b/examples/steipete/scripts/weak-model-test.mjs index ec3385d..93a16a8 100644 --- a/examples/steipete/scripts/weak-model-test.mjs +++ b/examples/steipete/scripts/weak-model-test.mjs @@ -78,7 +78,7 @@ const testPrompts = [ id: "ios_or_ai_2026", topic: "Should I learn iOS or AI in 2026", prompt: "A 19-year-old DMs you: 'Should I learn iOS dev in 2026 or go straight to AI/agents?' Your reply, 3-5 sentences.", - expectedSignals: ["ios", "pspdfkit", "agent", "ship", "you can just do things", "14 years", "leverage" /* note: he uses "leverage" only as a noun in math sense, not the verb. We allow noun usage but penalize verb usage in antiSignals separately if needed */], + expectedSignals: ["ios", "pspdfkit", "agent", "ship", "you can just do things", "14 years", "leverage" /* noun (math sense) only; the verb is an anti-signal */], antiSignals: ["follow your passion", "do what you love", "stakeholders", "synergy", "revolutionize"] }, { @@ -247,7 +247,6 @@ HARD RULES: return { model, backend, totalScore, maxScore, avgScore, results, passThreshold }; } -// Main const openaiKey = process.env.OPENAI_API_KEY; const openrouterKey = process.env.OPENROUTER_API_KEY; @@ -270,7 +269,6 @@ if (openaiKey) { const result = await runTest(apiKey, model, backend); -// Write results markdown const resultPath = join(ROOT, 'tests', 'weak-model-results.md'); let md = `# Weak Model Test Results — steipete soul file\n\n`; md += `**Model**: \`${result.model}\` (${result.backend})\n`; diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs index 9f35f72..87a4d09 100644 --- a/scripts/check-links.mjs +++ b/scripts/check-links.mjs @@ -2,7 +2,7 @@ // Verifies that relative links and image sources in the repo's maintained // docs resolve to real files. This catches the kind of dangling reference // that has slipped in before (e.g. a README pointing at a file that no longer -// exists). External URLs and same-page anchors are out of scope here — the +// exists). External URLs and same-page anchors are out of scope here — // markdownlint handles link *syntax*; this handles link *targets*. // // Scope: top-level markdown only. The examples/ subtrees carry raw, third-party From 79323af10cfc03965baebb471c742507a7513aa5 Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:45:15 -0400 Subject: [PATCH 6/9] optimize: make the CI link check real, unify SURF_API_KEY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-links.mjs scanned only root-level *.md — five files that contain no relative links at all. It reported "0 relative link(s) across 5 doc(s)" and the CI job named "Check relative links" passed unconditionally. Cause: commit 9edda35 moved README/CONTRIBUTING/SECURITY into .github/ and the script's scope was never updated, so the comment describing what it catches ("a README pointing at a file that no longer exists") named the exact failure mode it could no longer see. Now scans root + .github/ + the folder guides + per-example README/QUICKSTART, still excluding examples/*/data/ and the persona files (both carry links that are not ours to keep resolvable). 20 links across 20 docs, green. Verified it fails correctly by injecting a broken link: exits 1 and names the target. Also unified SURF_AI_API_KEY -> SURF_API_KEY across fetch-data.sh and steipete/README.md. pull-x.mjs reads SURF_API_KEY, so following the documented instructions hit `exit(1)` and section 4 of the pipeline had never run. --- examples/steipete/README.md | 4 ++-- examples/steipete/scripts/fetch-data.sh | 8 +++---- scripts/check-links.mjs | 32 +++++++++++++++++++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/examples/steipete/README.md b/examples/steipete/README.md index 63d9d2e..019ac55 100644 --- a/examples/steipete/README.md +++ b/examples/steipete/README.md @@ -82,8 +82,8 @@ The good/bad-outputs files codify these signals so the model has concrete calibr # Blog posts (47 files), GitHub READMEs (16 files), YouTube transcripts (8 files) bash scripts/fetch-data.sh -# X corpus (requires SURF_AI_API_KEY for surf-ai MCP) -SURF_AI_API_KEY=... HANDLES="steipete,openclaw" node scripts/pull-x.mjs +# X corpus (requires SURF_API_KEY for surf-ai MCP) +SURF_API_KEY=... HANDLES="steipete,openclaw" node scripts/pull-x.mjs # Voice-holding test (requires OpenAI or OpenRouter key) OPENROUTER_API_KEY=sk-or-... MODEL=openai/gpt-4o-mini node scripts/weak-model-test.mjs diff --git a/examples/steipete/scripts/fetch-data.sh b/examples/steipete/scripts/fetch-data.sh index b22dde4..a947460 100755 --- a/examples/steipete/scripts/fetch-data.sh +++ b/examples/steipete/scripts/fetch-data.sh @@ -6,7 +6,7 @@ # - Blog posts (steipete.me) # - GitHub repo READMEs / VISION.md / AGENTS.md # - Podcast / talk transcripts (via yt-dlp) -# - X/Twitter (via separate scripts/pull-x.mjs — needs SURF_AI_API_KEY) +# - X/Twitter (via separate scripts/pull-x.mjs — needs SURF_API_KEY) # # Idempotent — skips already-cached files. # @@ -220,12 +220,12 @@ echo "" # 4. X/Twitter corpus via surf-ai MCP # ----------------------------------------------------------------------------- echo "--- X corpus (surf-ai MCP) ---" -if [ -n "${SURF_AI_API_KEY:-}" ] && [ -f "$ROOT/scripts/pull-x.mjs" ]; then +if [ -n "${SURF_API_KEY:-}" ] && [ -f "$ROOT/scripts/pull-x.mjs" ]; then echo " [run] node scripts/pull-x.mjs" - HANDLES="steipete,openclaw" SURF_AI_API_KEY="$SURF_AI_API_KEY" \ + HANDLES="steipete,openclaw" SURF_API_KEY="$SURF_API_KEY" \ node "$ROOT/scripts/pull-x.mjs" || echo " ⚠ pull-x.mjs failed" else - echo " [skip] set SURF_AI_API_KEY and ensure scripts/pull-x.mjs exists" + echo " [skip] set SURF_API_KEY and ensure scripts/pull-x.mjs exists" fi echo "" diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs index 87a4d09..9f66706 100644 --- a/scripts/check-links.mjs +++ b/scripts/check-links.mjs @@ -5,9 +5,10 @@ // exists). External URLs and same-page anchors are out of scope here — // markdownlint handles link *syntax*; this handles link *targets*. // -// Scope: top-level markdown only. The examples/ subtrees carry raw, third-party -// source material whose internal links point at their original repo layouts, so -// they are intentionally excluded. +// Scope: hand-maintained docs only. examples/*/data/ holds raw, third-party +// source material whose internal links point at their original repo layouts, +// and the persona files (SOUL/STYLE/MEMORY, examples/*.md) contain sample links +// inside illustrative output — neither is ours to keep resolvable. // // Usage: node scripts/check-links.mjs @@ -16,8 +17,29 @@ import { dirname, resolve, join } from "node:path"; const ROOT = resolve(process.cwd()); -// Maintained docs live at the repo root. -const docs = readdirSync(ROOT).filter((f) => f.endsWith(".md")); +const mdIn = (dir) => { + const abs = join(ROOT, dir); + if (!existsSync(abs)) return []; + return readdirSync(abs) + .filter((f) => f.endsWith(".md")) + .map((f) => (dir ? `${dir}/${f}` : f)); +}; + +// Root templates/specs, community docs, the folder guides, and the per-example +// entry points a reader actually navigates from. +const exampleDirs = readdirSync(join(ROOT, "examples"), { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name); + +const docs = [ + ...mdIn(""), + ...mdIn(".github"), + "examples/_GUIDE.md", + "data/_GUIDE.md", + ...exampleDirs.flatMap((name) => + ["README.md", "QUICKSTART.md"].map((f) => `examples/${name}/${f}`), + ), +].filter((rel) => existsSync(join(ROOT, rel))); // Markdown inline links: [text](target) and HTML attrs: src="..." href="..." const MD_LINK = /\[[^\]]*\]\(\s*(<[^>]+>|[^)\s]+)/g; From fed6e6063814f862515924256198592fc2f65c6e Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:47:16 -0400 Subject: [PATCH 7/9] optimize: apply approved MEDIUM items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Strip "task brief / task spec / per task requirement" leaks (10 sites). These referenced a brief that only ever existed in the prompt of whatever agent authored the files. steipete's harness emitted the phrase into its generated output, so both the generators (weak-model-test.mjs:286,340) and the already-emitted artifact (tests/weak-model-results.md:21,267) are fixed — otherwise the next run would put it straight back. - README trees now match disk: garry-tan documented data/x, data/writing and data/yt (none exist; they are created by scripts/ on first run) and listed 2 of the 6 files in tests/; vivian-balakrishnan documented a data/speeches/ that does not exist; vitalik-buterin omitted tests/ while linking into it. Also corrected garry-tan's tree root label (garrytan/ -> garry-tan/). - BUILD.md and data/_GUIDE.md had zero inbound references repo-wide. Linked both from CONTRIBUTING.md. They are product docs that fell out of the nav, not stale files. - run_weak_model.py: stop catching HTTPError and returning "[ERROR 401: ...]" as if it were the model's answer — that wrote a complete-looking results file and exited 0, for an artifact published as proof the soul file works. The error now propagates and the transcript truncates honestly. Dropped the ~/.hermes/.env fallback (a personal-machine path) and the now-unused urllib.error import; documented the certifi dependency instead. --- .github/CONTRIBUTING.md | 6 ++++-- examples/garry-tan/README.md | 16 +++++++++------- examples/garry-tan/tests/run_weak_model.py | 17 +++++------------ examples/garry-tan/tests/weak_model_test.md | 2 +- examples/karpathy/README.md | 2 +- examples/karpathy/scripts/weak-model-test.mjs | 4 ++-- examples/steipete/README.md | 2 +- examples/steipete/scripts/weak-model-test.mjs | 6 +++--- examples/steipete/tests/weak-model-results.md | 4 ++-- examples/vitalik-buterin/README.md | 8 +++++--- examples/vivian-balakrishnan/README.md | 3 +-- 11 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5f42319..68d8221 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -13,8 +13,10 @@ reading it could predict its subject's take on a new topic.** factual error or broken link. Build your soul first with the [soul-builder skill](../BUILD.md) (`/soul-builder`) -or by hand from the templates. See [`examples/_GUIDE.md`](../examples/_GUIDE.md) -for what goes in the calibration files. +or by hand from the templates — [`BUILD.md`](../BUILD.md) is the full spec for +what a soul folder must contain. See [`examples/_GUIDE.md`](../examples/_GUIDE.md) +for what goes in the calibration files, and [`data/_GUIDE.md`](../data/_GUIDE.md) +for what belongs in `data/`. ## Before you start diff --git a/examples/garry-tan/README.md b/examples/garry-tan/README.md index 5180788..5d8756c 100644 --- a/examples/garry-tan/README.md +++ b/examples/garry-tan/README.md @@ -7,15 +7,13 @@ This folder is structured to be dropped into `aeonfun/soul.md` at `examples/garr ## What's in here ``` -garrytan/ +garry-tan/ ├── SOUL.md ← Identity, worldview, opinions ├── STYLE.md ← Voice mechanics ├── MEMORY.md ← Empty session log ├── data/ -│ ├── influences.md -│ ├── x/ ← Twitter archive target (see scripts/) -│ ├── writing/ ← YC blog + personal essays -│ └── yt/ ← YouTube transcripts +│ └── influences.md +│ ← x/, writing/, yt/ are created by scripts/ on first run ├── examples/ │ ├── good-outputs.md ← 12 calibration samples │ └── bad-outputs.md ← 10 anti-patterns @@ -24,8 +22,12 @@ garrytan/ │ ├── fetch_writing.py ← YC blog + personal posts → data/writing/ │ └── fetch_yt.py ← YouTube transcripts → data/yt/*.md └── tests/ - ├── prediction_test.md ← 10 unseen prompts + expected takes - └── weak_model_test.md ← gpt-4o-mini run + verdict + ├── prediction_test.md ← 10 unseen prompts + expected takes + ├── platform_tests.md ← 3 platform-shape prompts + ├── weak_model_test.md ← protocol + verdict + ├── run_weak_model.py ← the harness + ├── results_gpt-4o-mini.md ← full transcript + └── scores_gpt-4o-mini.md ← per-prompt grades (38.5/48 PASS) ``` ## How to rebuild the corpus diff --git a/examples/garry-tan/tests/run_weak_model.py b/examples/garry-tan/tests/run_weak_model.py index 12b93f2..731d1ec 100644 --- a/examples/garry-tan/tests/run_weak_model.py +++ b/examples/garry-tan/tests/run_weak_model.py @@ -2,9 +2,11 @@ """ Run the soul file against a weak model via OpenRouter. -Reads OPENROUTER_API_KEY from env (or from ~/.hermes/.env as a fallback). +Reads OPENROUTER_API_KEY from env. Writes a markdown transcript with one section per prompt. +Requires `certifi` (pip install certifi). + Usage: python tests/run_weak_model.py \ --model openai/gpt-4o-mini \ @@ -21,7 +23,6 @@ import pathlib import sys import ssl -import urllib.error import urllib.request import certifi @@ -89,11 +90,6 @@ def load_api_key() -> str: key = os.environ.get("OPENROUTER_API_KEY") if key: return key - fallback = pathlib.Path.home() / ".hermes" / ".env" - if fallback.exists(): - for line in fallback.read_text().splitlines(): - if line.startswith("OPENROUTER_API_KEY="): - return line.split("=", 1)[1].strip() sys.exit("OPENROUTER_API_KEY not found") @@ -119,11 +115,8 @@ def call_model(api_key: str, model: str, system: str, user: str) -> str: "X-Title": "soul-garrytan weak-model test", }, ) - try: - with urllib.request.urlopen(req, timeout=90, context=_SSL_CTX) as resp: - data = json.loads(resp.read()) - except urllib.error.HTTPError as e: - return f"[ERROR {e.code}: {e.read().decode(errors='replace')}]" + with urllib.request.urlopen(req, timeout=90, context=_SSL_CTX) as resp: + data = json.loads(resp.read()) return data["choices"][0]["message"]["content"] diff --git a/examples/garry-tan/tests/weak_model_test.md b/examples/garry-tan/tests/weak_model_test.md index 30c986e..19889cb 100644 --- a/examples/garry-tan/tests/weak_model_test.md +++ b/examples/garry-tan/tests/weak_model_test.md @@ -147,7 +147,7 @@ with open(args.out, "w") as out: print(f"wrote → {args.out}", file=sys.stderr) ``` -Run it, grade it, commit the result. A passing run is part of the deliverable. +Run it, grade it, commit the result. --- diff --git a/examples/karpathy/README.md b/examples/karpathy/README.md index 9534360..10b68e4 100644 --- a/examples/karpathy/README.md +++ b/examples/karpathy/README.md @@ -68,7 +68,7 @@ Karpathy writes like a teacher who builds. Short sentences. Specific technical a - **Identity** — who he is, the throughline (build from scratch to understand, teach by building) - **12 worldview items** — using his own language and phrasing, with tweet-style direct quotes - **5 modes** — The Teacher, The Hacker/Builder, The ML Philosopher, The Industry Insider, The Nerd -- **Tensions & contradictions (8)** — the self-aware paradoxes: safety-concerned but anti-pause, YC-world-adjacent but not a founder personality, demystifier who calls neural nets "magical," etc. (This is the "range" the task brief asked for.) +- **Tensions & contradictions (8)** — the self-aware paradoxes: safety-concerned but anti-pause, YC-world-adjacent but not a founder personality, demystifier who calls neural nets "magical," etc. - **Boundaries** — what he won't do (partisan politics, doomer takes, hype) - **Pet peeves** — frameworks that hide the code, "it's just X" dismissals, confident timelines diff --git a/examples/karpathy/scripts/weak-model-test.mjs b/examples/karpathy/scripts/weak-model-test.mjs index 9dc1aee..c52299c 100644 --- a/examples/karpathy/scripts/weak-model-test.mjs +++ b/examples/karpathy/scripts/weak-model-test.mjs @@ -3,13 +3,13 @@ * Weak Model Test — Karpathy Soul File * * Tests whether the SOUL.md + STYLE.md stack can hold Karpathy's voice - * on gpt-4o-mini (per task requirement). + * on gpt-4o-mini. * * Usage: * OPENROUTER_API_KEY=sk-or-... node scripts/weak-model-test.mjs * OPENAI_API_KEY=sk-... node scripts/weak-model-test.mjs (native OpenAI) * - * Default model: openai/gpt-4o-mini (as task specifies) + * Default model: openai/gpt-4o-mini * Override: MODEL=anthropic/claude-haiku-4-5 node scripts/weak-model-test.mjs */ diff --git a/examples/steipete/README.md b/examples/steipete/README.md index 019ac55..6d27ae7 100644 --- a/examples/steipete/README.md +++ b/examples/steipete/README.md @@ -50,7 +50,7 @@ steipete-soul/ ## Voice validation -The soul stack is validated by an automated weak-model test on the model the task brief specifies (`gpt-4o-mini`): +The soul stack is validated by an automated weak-model test on `gpt-4o-mini`: ```bash OPENROUTER_API_KEY=sk-or-... MODEL=openai/gpt-4o-mini node scripts/weak-model-test.mjs diff --git a/examples/steipete/scripts/weak-model-test.mjs b/examples/steipete/scripts/weak-model-test.mjs index 93a16a8..568cd77 100644 --- a/examples/steipete/scripts/weak-model-test.mjs +++ b/examples/steipete/scripts/weak-model-test.mjs @@ -3,7 +3,7 @@ * Weak Model Test — steipete Soul File * * Tests whether the SOUL.md + STYLE.md + examples/ stack can hold Peter - * Steinberger's voice on gpt-4o-mini (per task spec). + * Steinberger's voice on gpt-4o-mini. * * Usage: * OPENROUTER_API_KEY=sk-or-... node scripts/weak-model-test.mjs @@ -283,7 +283,7 @@ md += `- **Voice (0-2)**: signal density (named tools, lobster register, tl;dr o md += `- **Stance (0-2)**: not-hedging + has-opinion + has-anchor (number / version / named tool).\n`; md += `- **Anti-pattern penalty (up to -2)**: corporate vocab (leverage, unlock, revolutionize, synergy, stakeholders, "thrilled," game-changer, paradigm shift, "both have strengths," etc.) detected per prompt.\n\n`; md += `Max per prompt: 4. Pass threshold: 3.0/4 average — equivalent to "most prompts pass cleanly with at most 1-2 minor failures."\n\n`; -md += `The model used is \`gpt-4o-mini\` per the task spec (or \`openai/gpt-4o-mini\` via OpenRouter, identical model). The intent of the weak-model test: if the soul file holds voice on a smaller, less personality-aware model, it will hold on stronger ones too.\n\n---\n\n`; +md += `The model used is \`gpt-4o-mini\` (or \`openai/gpt-4o-mini\` via OpenRouter, identical model). The intent of the weak-model test: if the soul file holds voice on a smaller, less personality-aware model, it will hold on stronger ones too.\n\n---\n\n`; md += `## Score Summary\n\n`; md += `| # | Test | Voice | Stance | Anti | Score |\n`; @@ -337,7 +337,7 @@ if (lowScores.length === 0) { md += `## Reproducibility\n\n`; md += `\`\`\`bash\n`; -md += `# OpenAI direct (recommended for the gpt-4o-mini per task spec)\n`; +md += `# OpenAI direct (recommended for gpt-4o-mini)\n`; md += `OPENAI_API_KEY=sk-... node scripts/weak-model-test.mjs\n\n`; md += `# OpenRouter (same model, different vendor)\n`; md += `OPENROUTER_API_KEY=sk-or-... node scripts/weak-model-test.mjs\n\n`; diff --git a/examples/steipete/tests/weak-model-results.md b/examples/steipete/tests/weak-model-results.md index 50e421e..bd0f60d 100644 --- a/examples/steipete/tests/weak-model-results.md +++ b/examples/steipete/tests/weak-model-results.md @@ -18,7 +18,7 @@ Each of 12 test prompts (mirroring `tests/prediction-test.md`) is scored on: Max per prompt: 4. Pass threshold: 3.0/4 average — equivalent to "most prompts pass cleanly with at most 1-2 minor failures." -The model used is `gpt-4o-mini` per the task spec (or `openai/gpt-4o-mini` via OpenRouter, identical model). The intent of the weak-model test: if the soul file holds voice on a smaller, less personality-aware model, it will hold on stronger ones too. +The model used is `gpt-4o-mini` (or `openai/gpt-4o-mini` via OpenRouter, identical model). The intent of the weak-model test: if the soul file holds voice on a smaller, less personality-aware model, it will hold on stronger ones too. --- @@ -264,7 +264,7 @@ The model used is `gpt-4o-mini` per the task spec (or `openai/gpt-4o-mini` via O ## Reproducibility ```bash -# OpenAI direct (recommended for the gpt-4o-mini per task spec) +# OpenAI direct (recommended for gpt-4o-mini) OPENAI_API_KEY=sk-... node scripts/weak-model-test.mjs # OpenRouter (same model, different vendor) diff --git a/examples/vitalik-buterin/README.md b/examples/vitalik-buterin/README.md index 83efc0e..a7a9e97 100644 --- a/examples/vitalik-buterin/README.md +++ b/examples/vitalik-buterin/README.md @@ -17,9 +17,11 @@ vitalik-buterin/ ├── MEMORY.md ← running log across sessions ├── data/ │ └── influences.md ← people, papers, concepts, his own essays -└── examples/ - ├── good-outputs.md ← 14 calibration samples + 4 verbatim verified quotes - └── bad-outputs.md ← 12 anti-patterns with diagnosis + 7-point checklist +├── examples/ +│ ├── good-outputs.md ← 14 calibration samples + 4 verbatim verified quotes +│ └── bad-outputs.md ← 12 anti-patterns with diagnosis + 7-point checklist +└── tests/ + └── prediction-test.md ← unseen prompts + predicted takes ``` --- diff --git a/examples/vivian-balakrishnan/README.md b/examples/vivian-balakrishnan/README.md index bcd7bcd..40bdb56 100644 --- a/examples/vivian-balakrishnan/README.md +++ b/examples/vivian-balakrishnan/README.md @@ -16,8 +16,7 @@ vivian-balakrishnan/ ├── STYLE.md ← voice rules: vocabulary, rhetorical moves, register ├── MEMORY.md ← running log across sessions ├── data/ -│ ├── influences.md ← Singapore lineage, books, frameworks -│ └── speeches/ ← (placeholder for added MFA transcripts) +│ └── influences.md ← Singapore lineage, books, frameworks └── examples/ ├── good-outputs.md ← 14 calibration samples + 12 verbatim verified quotes └── bad-outputs.md ← 12 anti-patterns with diagnosis + 7-point checklist From 1d24fffe9dddc2e676e38236891c405cdf64a82f Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:52:47 -0400 Subject: [PATCH 8/9] optimize: repair the corrupted corpus files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of the three files the silent-curl bug poisoned are now real data. Fixed the sources first so a re-run stays correct, then re-fetched: - data/writing/software-2.0.html: was a 9379-byte GitHub Pages 404. The essay is on Medium, not karpathy.github.io — the URL in BLOG_URLS was simply wrong. Now 240KB of the real essay ("I sometimes see people refer to neural networks as just another tool in your machine learning toolbox", Nov 2017). This is what SOUL.md:20 calls "my core thesis" and what weak-model-test.mjs scores against, so it was the load-bearing one. - data/github/karpathy_char-rnn_README.md: was 14 bytes of "404: Not Found". That repo ships Readme.md, not README.md. Now the real 12.8KB README. The fetch loop now tries README.md/Readme.md/readme.md across master and main rather than assuming one spelling. Verified BLOG_SLUGS and BLOG_URLS still pair 1:1 (13/13) after the URL swap — they are positionally matched by `set -- $BLOG_URLS`, so a miscount would silently mislabel every file after it. Not repaired: data/x/steipete.replies.json is still [] and needs a SURF_API_KEY to re-pull. Annotated it in the README so it can't be mistaken for "steipete has no replies" — its sibling openclaw.replies.json has 100. Swept all of examples/*/data for other empty or error-body files: none. --- .../data/github/karpathy_char-rnn_README.md | 142 ++++- .../karpathy/data/writing/software-2.0.html | 580 +++++++++++++++--- examples/karpathy/scripts/fetch-data.sh | 18 +- examples/steipete/README.md | 3 +- 4 files changed, 650 insertions(+), 93 deletions(-) diff --git a/examples/karpathy/data/github/karpathy_char-rnn_README.md b/examples/karpathy/data/github/karpathy_char-rnn_README.md index 1becba2..a14aa2d 100644 --- a/examples/karpathy/data/github/karpathy_char-rnn_README.md +++ b/examples/karpathy/data/github/karpathy_char-rnn_README.md @@ -1 +1,141 @@ -404: Not Found \ No newline at end of file + +# char-rnn + +This code implements **multi-layer Recurrent Neural Network** (RNN, LSTM, and GRU) for training/sampling from character-level language models. In other words the model takes one text file as input and trains a Recurrent Neural Network that learns to predict the next character in a sequence. The RNN can then be used to generate text character by character that will look like the original training data. The context of this code base is described in detail in my [blog post](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). + +If you are new to Torch/Lua/Neural Nets, it might be helpful to know that this code is really just a slightly more fancy version of this [100-line gist](https://gist.github.com/karpathy/d4dee566867f8291f086) that I wrote in Python/numpy. The code in this repo additionally: allows for multiple layers, uses an LSTM instead of a vanilla RNN, has more supporting code for model checkpointing, and is of course much more efficient since it uses mini-batches and can run on a GPU. + +## Update: torch-rnn + +[Justin Johnson](http://cs.stanford.edu/people/jcjohns/) (@jcjohnson) recently re-implemented char-rnn from scratch with a much nicer/smaller/cleaner/faster Torch code base. It's under the name [torch-rnn](https://github.com/jcjohnson/torch-rnn). It uses Adam for optimization and hard-codes the RNN/LSTM forward/backward passes for space/time efficiency. This also avoids headaches with cloning models in this repo. In other words, torch-rnn should be the default char-rnn implemention to use now instead of the one in this code base. + +## Requirements + +This code is written in Lua and requires [Torch](http://torch.ch/). If you're on Ubuntu, installing Torch in your home directory may look something like: + +```bash +$ curl -s https://raw.githubusercontent.com/torch/ezinstall/master/install-deps | bash +$ git clone https://github.com/torch/distro.git ~/torch --recursive +$ cd ~/torch; +$ ./install.sh # and enter "yes" at the end to modify your bashrc +$ source ~/.bashrc +``` + +See the Torch installation documentation for more details. After Torch is installed we need to get a few more packages using [LuaRocks](https://luarocks.org/) (which already came with the Torch install). In particular: + +```bash +$ luarocks install nngraph +$ luarocks install optim +$ luarocks install nn +``` + +If you'd like to train on an NVIDIA GPU using CUDA (this can be to about 15x faster), you'll of course need the GPU, and you will have to install the [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit). Then get the `cutorch` and `cunn` packages: + +```bash +$ luarocks install cutorch +$ luarocks install cunn +``` + +If you'd like to use OpenCL GPU instead (e.g. ATI cards), you will instead need to install the `cltorch` and `clnn` packages, and then use the option `-opencl 1` during training ([cltorch issues](https://github.com/hughperkins/cltorch/issues)): + +```bash +$ luarocks install cltorch +$ luarocks install clnn +``` + +## Usage + +### Data + +All input data is stored inside the `data/` directory. You'll notice that there is an example dataset included in the repo (in folder `data/tinyshakespeare`) which consists of a subset of works of Shakespeare. I'm providing a few more datasets on [this page](http://cs.stanford.edu/people/karpathy/char-rnn/). + +**Your own data**: If you'd like to use your own data then create a single file `input.txt` and place it into a folder in the `data/` directory. For example, `data/some_folder/input.txt`. The first time you run the training script it will do some preprocessing and write two more convenience cache files into `data/some_folder`. + +**Dataset sizes**: Note that if your data is too small (1MB is already considered very small) the RNN won't learn very effectively. Remember that it has to learn everything completely from scratch. Conversely if your data is large (more than about 2MB), feel confident to increase `rnn_size` and train a bigger model (see details of training below). It will work *significantly better*. For example with 6MB you can easily go up to `rnn_size` 300 or even more. The biggest that fits on my GPU and that I've trained with this code is `rnn_size` 700 with `num_layers` 3 (2 is default). + +### Training + +Start training the model using `train.lua`. As a sanity check, to run on the included example dataset simply try: + +``` +$ th train.lua -gpuid -1 +``` + +Notice that here we are setting the flag `gpuid` to -1, which tells the code to train using CPU, otherwise it defaults to GPU 0. There are many other flags for various options. Consult `$ th train.lua -help` for comprehensive settings. Here's another example that trains a bigger network and also shows how you can run on your own custom dataset (this already assumes that `data/some_folder/input.txt` exists): + +``` +$ th train.lua -data_dir data/some_folder -rnn_size 512 -num_layers 2 -dropout 0.5 +``` + +**Checkpoints.** While the model is training it will periodically write checkpoint files to the `cv` folder. The frequency with which these checkpoints are written is controlled with number of iterations, as specified with the `eval_val_every` option (e.g. if this is 1 then a checkpoint is written every iteration). The filename of these checkpoints contains a very important number: the **loss**. For example, a checkpoint with filename `lm_lstm_epoch0.95_2.0681.t7` indicates that at this point the model was on epoch 0.95 (i.e. it has almost done one full pass over the training data), and the loss on validation data was 2.0681. This number is very important because the lower it is, the better the checkpoint works. Once you start to generate data (discussed below), you will want to use the model checkpoint that reports the lowest validation loss. Notice that this might not necessarily be the last checkpoint at the end of training (due to possible overfitting). + +Another important quantities to be aware of are `batch_size` (call it B), `seq_length` (call it S), and the `train_frac` and `val_frac` settings. The batch size specifies how many streams of data are processed in parallel at one time. The sequence length specifies the length of each stream, which is also the limit at which the gradients can propagate backwards in time. For example, if `seq_length` is 20, then the gradient signal will never backpropagate more than 20 time steps, and the model might not *find* dependencies longer than this length in number of characters. Thus, if you have a very difficult dataset where there are a lot of long-term dependencies you will want to increase this setting. Now, if at runtime your input text file has N characters, these first all get split into chunks of size `BxS`. These chunks then get allocated across three splits: train/val/test according to the `frac` settings. By default `train_frac` is 0.95 and `val_frac` is 0.05, which means that 95% of our data chunks will be trained on and 5% of the chunks will be used to estimate the validation loss (and hence the generalization). If your data is small, it's possible that with the default settings you'll only have very few chunks in total (for example 100). This is bad: In these cases you may want to decrease batch size or sequence length. + +Note that you can also initialize parameters from a previously saved checkpoint using `init_from`. + +### Sampling + +Given a checkpoint file (such as those written to `cv`) we can generate new text. For example: + +``` +$ th sample.lua cv/some_checkpoint.t7 -gpuid -1 +``` + +Make sure that if your checkpoint was trained with GPU it is also sampled from with GPU, or vice versa. Otherwise the code will (currently) complain. As with the train script, see `$ th sample.lua -help` for full options. One important one is (for example) `-length 10000` which would generate 10,000 characters (default = 2000). + +**Temperature**. An important parameter you may want to play with is `-temperature`, which takes a number in range \(0, 1\] (0 not included), default = 1. The temperature is dividing the predicted log probabilities before the Softmax, so lower temperature will cause the model to make more likely, but also more boring and conservative predictions. Higher temperatures cause the model to take more chances and increase diversity of results, but at a cost of more mistakes. + +**Priming**. It's also possible to prime the model with some starting text using `-primetext`. This starts out the RNN with some hardcoded characters to *warm* it up with some context before it starts generating text. E.g. a fun primetext might be `-primetext "the meaning of life is "`. + +**Training with GPU but sampling on CPU**. Right now the solution is to use the `convert_gpu_cpu_checkpoint.lua` script to convert your GPU checkpoint to a CPU checkpoint. In near future you will not have to do this explicitly. E.g.: + +``` +$ th convert_gpu_cpu_checkpoint.lua cv/lm_lstm_epoch30.00_1.3950.t7 +``` + +will create a new file `cv/lm_lstm_epoch30.00_1.3950.t7_cpu.t7` that you can use with the sample script and with `-gpuid -1` for CPU mode. + +Happy sampling! + +## Tips and Tricks + +### Monitoring Validation Loss vs. Training Loss +If you're somewhat new to Machine Learning or Neural Networks it can take a bit of expertise to get good models. The most important quantity to keep track of is the difference between your training loss (printed during training) and the validation loss (printed once in a while when the RNN is run on the validation data (by default every 1000 iterations)). In particular: + +- If your training loss is much lower than validation loss then this means the network might be **overfitting**. Solutions to this are to decrease your network size, or to increase dropout. For example you could try dropout of 0.5 and so on. +- If your training/validation loss are about equal then your model is **underfitting**. Increase the size of your model (either number of layers or the raw number of neurons per layer) + +### Approximate number of parameters + +The two most important parameters that control the model are `rnn_size` and `num_layers`. I would advise that you always use `num_layers` of either 2/3. The `rnn_size` can be adjusted based on how much data you have. The two important quantities to keep track of here are: + +- The number of parameters in your model. This is printed when you start training. +- The size of your dataset. 1MB file is approximately 1 million characters. + +These two should be about the same order of magnitude. It's a little tricky to tell. Here are some examples: + +- I have a 100MB dataset and I'm using the default parameter settings (which currently print 150K parameters). My data size is significantly larger (100 mil >> 0.15 mil), so I expect to heavily underfit. I am thinking I can comfortably afford to make `rnn_size` larger. +- I have a 10MB dataset and running a 10 million parameter model. I'm slightly nervous and I'm carefully monitoring my validation loss. If it's larger than my training loss then I may want to try to increase dropout a bit and see if that heps the validation loss. + +### Best models strategy + +The winning strategy to obtaining very good models (if you have the compute time) is to always err on making the network larger (as large as you're willing to wait for it to compute) and then try different dropout values (between 0,1). Whatever model has the best validation performance (the loss, written in the checkpoint filename, low is good) is the one you should use in the end. + +It is very common in deep learning to run many different models with many different hyperparameter settings, and in the end take whatever checkpoint gave the best validation performance. + +By the way, the size of your training and validation splits are also parameters. Make sure you have a decent amount of data in your validation set or otherwise the validation performance will be noisy and not very informative. + +## Additional Pointers and Acknowledgements + +This code was originally based on Oxford University Machine Learning class [practical 6](https://github.com/oxford-cs-ml-2015/practical6), which is in turn based on [learning to execute](https://github.com/wojciechz/learning_to_execute) code from Wojciech Zaremba. Chunks of it were also developed in collaboration with my labmate [Justin Johnson](http://cs.stanford.edu/people/jcjohns/). + +To learn more about RNN language models I recommend looking at: + +- [My recent talk](https://skillsmatter.com/skillscasts/6611-visualizing-and-understanding-recurrent-networks) on char-rnn +- [Generating Sequences With Recurrent Neural Networks](http://arxiv.org/abs/1308.0850) by Alex Graves +- [Generating Text with Recurrent Neural Networks](http://www.cs.utoronto.ca/~ilya/pubs/2011/LANG-RNN.pdf) by Ilya Sutskever +- [Tomas Mikolov's Thesis](http://www.fit.vutbr.cz/~imikolov/rnnlm/thesis.pdf) + +## License + +MIT diff --git a/examples/karpathy/data/writing/software-2.0.html b/examples/karpathy/data/writing/software-2.0.html index 5b10050..7cddd12 100644 --- a/examples/karpathy/data/writing/software-2.0.html +++ b/examples/karpathy/data/writing/software-2.0.html @@ -1,89 +1,495 @@ - - - - - - Page not found · GitHub Pages - MediumSoftware 2.0. I sometimes see people refer to neural… | by Andrej Karpathy | Medium
Sitemap
Open in app

Sign in

Medium Logo
Write
Search

Sign in

Software 2.0

Andrej Karpathy
9 min readNov 11, 2017

I sometimes see people refer to neural networks as just “another tool in your machine learning toolbox”. They have some pros and cons, they work here or there, and sometimes you can use them to win Kaggle competitions. Unfortunately, this interpretation completely misses the forest for the trees. Neural networks are not just another classifier, they represent the beginning of a fundamental shift in how we develop software. They are Software 2.0.

The “classical stack” of Software 1.0 is what we’re all familiar with — it is written in languages such as Python, C++, etc. It consists of explicit instructions to the computer written by a programmer. By writing each line of code, the programmer identifies a specific point in program space with some desirable behavior.

Press enter or click to view image in full size

In contrast, Software 2.0 is written in much more abstract, human unfriendly language, such as the weights of a neural network. No human is involved in writing this code because there are a lot of weights (typical networks might have millions), and coding directly in weights is kind of hard (I tried).

Press enter or click to view image in full size

Instead, our approach is to specify some goal on the behavior of a desirable program (e.g., “satisfy a dataset of input output pairs of examples”, or “win a game of Go”), write a rough skeleton of the code (i.e. a neural net architecture) that identifies a subset of program space to search, and use the computational resources at our disposal to search this space for a program that works. In the case of neural networks, we restrict the search to a continuous subset of the program space where the search process can be made (somewhat surprisingly) efficient with backpropagation and stochastic gradient descent.

Press enter or click to view image in full size

To make the analogy explicit, in Software 1.0, human-engineered source code (e.g. some .cpp files) is compiled into a binary that does useful work. In Software 2.0 most often the source code comprises 1) the dataset that defines the desirable behavior and 2) the neural net architecture that gives the rough skeleton of the code, but with many details (the weights) to be filled in. The process of training the neural network compiles the dataset into the binary — the final neural network. In most practical applications today, the neural net architectures and the training systems are increasingly standardized into a commodity, so most of the active “software development” takes the form of curating, growing, massaging and cleaning labeled datasets. This is fundamentally altering the programming paradigm by which we iterate on our software, as the teams split in two: the 2.0 programmers (data labelers) edit and grow the datasets, while a few 1.0 programmers maintain and iterate on the surrounding training code infrastructure, analytics, visualizations and labeling interfaces.

It turns out that a large portion of real-world problems have the property that it is significantly easier to collect the data (or more generally, identify a desirable behavior) than to explicitly write the program. Because of this and many other benefits of Software 2.0 programs that I will go into below, we are witnessing a massive transition across the industry where of a lot of 1.0 code is being ported into 2.0 code. Software (1.0) is eating the world, and now AI (Software 2.0) is eating software.

Ongoing transition

Let’s briefly examine some concrete examples of this ongoing transition. In each of these areas we’ve seen improvements over the last few years when we give up on trying to address a complex problem by writing explicit code and instead transition the code into the 2.0 stack.

Visual Recognition used to consist of engineered features with a bit of machine learning sprinkled on top at the end (e.g., an SVM). Since then, we discovered much more powerful visual features by obtaining large datasets (e.g. ImageNet) and searching in the space of Convolutional Neural Network architectures. More recently, we don’t even trust ourselves to hand-code the architectures and we’ve begun searching over those as well.

Speech recognition used to involve a lot of preprocessing, gaussian mixture models and hidden markov models, but today consist almost entirely of neural net stuff. A very related, often cited humorous quote attributed to Fred Jelinek from 1985 reads “Every time I fire a linguist, the performance of our speech recognition system goes up”.

Speech synthesis has historically been approached with various stitching mechanisms, but today the state of the art models are large ConvNets (e.g. WaveNet) that produce raw audio signal outputs.

Machine Translation has usually been approaches with phrase-based statistical techniques, but neural networks are quickly becoming dominant. My favorite architectures are trained in the multilingual setting, where a single model translates from any source language to any target language, and in weakly supervised (or entirely unsupervised) settings.

Games. Explicitly hand-coded Go playing programs have been developed for a long while, but AlphaGo Zero (a ConvNet that looks at the raw state of the board and plays a move) has now become by far the strongest player of the game. I expect we’re going to see very similar results in other areas, e.g. DOTA 2, or StarCraft.

Databases. More traditional systems outside of Artificial Intelligence are also seeing early hints of a transition. For instance, “The Case for Learned Index Structures” replaces core components of a data management system with a neural network, outperforming cache-optimized B-Trees by up to 70% in speed while saving an order-of-magnitude in memory.

You’ll notice that many of my links above involve work done at Google. This is because Google is currently at the forefront of re-writing large chunks of itself into Software 2.0 code. “One model to rule them all” provides an early sketch of what this might look like, where the statistical strength of the individual domains is amalgamated into one consistent understanding of the world.

The benefits of Software 2.0

Why should we prefer to port complex programs into Software 2.0? Clearly, one easy answer is that they work better in practice. However, there are a lot of other convenient reasons to prefer this stack. Let’s take a look at some of the benefits of Software 2.0 (think: a ConvNet) compared to Software 1.0 (think: a production-level C++ code base). Software 2.0 is:

Computationally homogeneous. A typical neural network is, to the first order, made up of a sandwich of only two operations: matrix multiplication and thresholding at zero (ReLU). Compare that with the instruction set of classical software, which is significantly more heterogenous and complex. Because you only have to provide Software 1.0 implementation for a small number of the core computational primitives (e.g. matrix multiply), it is much easier to make various correctness/performance guarantees.

Simple to bake into silicon. As a corollary, since the instruction set of a neural network is relatively small, it is significantly easier to implement these networks much closer to silicon, e.g. with custom ASICs, neuromorphic chips, and so on. The world will change when low-powered intelligence becomes pervasive around us. E.g., small, inexpensive chips could come with a pretrained ConvNet, a speech recognizer, and a WaveNet speech synthesis network all integrated in a small protobrain that you can attach to stuff.

Constant running time. Every iteration of a typical neural net forward pass takes exactly the same amount of FLOPS. There is zero variability based on the different execution paths your code could take through some sprawling C++ code base. Of course, you could have dynamic compute graphs but the execution flow is normally still significantly constrained. This way we are also almost guaranteed to never find ourselves in unintended infinite loops.

Constant memory use. Related to the above, there is no dynamically allocated memory anywhere so there is also little possibility of swapping to disk, or memory leaks that you have to hunt down in your code.

It is highly portable. A sequence of matrix multiplies is significantly easier to run on arbitrary computational configurations compared to classical binaries or scripts.

It is very agile. If you had a C++ code and someone wanted you to make it twice as fast (at cost of performance if needed), it would be highly non-trivial to tune the system for the new spec. However, in Software 2.0 we can take our network, remove half of the channels, retrain, and there — it runs exactly at twice the speed and works a bit worse. It’s magic. Conversely, if you happen to get more data/compute, you can immediately make your program work better just by adding more channels and retraining.

Modules can meld into an optimal whole. Our software is often decomposed into modules that communicate through public functions, APIs, or endpoints. However, if two Software 2.0 modules that were originally trained separately interact, we can easily backpropagate through the whole. Think about how amazing it could be if your web browser could automatically re-design the low-level system instructions 10 stacks down to achieve a higher efficiency in loading web pages. Or if the computer vision library (e.g. OpenCV) you imported could be auto-tuned on your specific data. With 2.0, this is the default behavior.

It is better than you. Finally, and most importantly, a neural network is a better piece of code than anything you or I can come up with in a large fraction of valuable verticals, which currently at the very least involve anything to do with images/video and sound/speech.

The limitations of Software 2.0

The 2.0 stack also has some of its own disadvantages. At the end of the optimization we’re left with large networks that work well, but it’s very hard to tell how. Across many applications areas, we’ll be left with a choice of using a 90% accurate model we understand, or 99% accurate model we don’t.

The 2.0 stack can fail in unintuitive and embarrassing ways ,or worse, they can “silently fail”, e.g., by silently adopting biases in their training data, which are very difficult to properly analyze and examine when their sizes are easily in the millions in most cases.

Finally, we’re still discovering some of the peculiar properties of this stack. For instance, the existence of adversarial examples and attacks highlights the unintuitive nature of this stack.

Programming in the 2.0 stack

Software 1.0 is code we write. Software 2.0 is code written by the optimization based on an evaluation criterion (such as “classify this training data correctly”). It is likely that any setting where the program is not obvious but one can repeatedly evaluate the performance of it (e.g. — did you classify some images correctly? do you win games of Go?) will be subject to this transition, because the optimization can find much better code than what a human can write.

The lens through which we view trends matters. If you recognize Software 2.0 as a new and emerging programming paradigm instead of simply treating neural networks as a pretty good classifier in the class of machine learning techniques, the extrapolations become more obvious, and it’s clear that there is much more work to do.

In particular, we’ve built up a vast amount of tooling that assists humans in writing 1.0 code, such as powerful IDEs with features like syntax highlighting, debuggers, profilers, go to def, git integration, etc. In the 2.0 stack, the programming is done by accumulating, massaging and cleaning datasets. For example, when the network fails in some hard or rare cases, we do not fix those predictions by writing code, but by including more labeled examples of those cases. Who is going to develop the first Software 2.0 IDEs, which help with all of the workflows in accumulating, visualizing, cleaning, labeling, and sourcing datasets? Perhaps the IDE bubbles up images that the network suspects are mislabeled based on the per-example loss, or assists in labeling by seeding labels with predictions, or suggests useful examples to label based on the uncertainty of the network’s predictions.

Similarly, Github is a very successful home for Software 1.0 code. Is there space for a Software 2.0 Github? In this case repositories are datasets and commits are made up of additions and edits of the labels.

Traditional package managers and related serving infrastructure like pip, conda, docker, etc. help us more easily deploy and compose binaries. How do we effectively deploy, share, import and work with Software 2.0 binaries? What is the conda equivalent for neural networks?

In the short term, Software 2.0 will become increasingly prevalent in any domain where repeated evaluation is possible and cheap, and where the algorithm itself is difficult to design explicitly. There are many exciting opportunities to consider the entire software development ecosystem and how it can be adapted to this new programming paradigm. And in the long run, the future of this paradigm is bright because it is increasingly clear that when we develop AGI, it will certainly be written in Software 2.0.

Machine Learning
Artificial Intelligence
Programming
Software Development
Future

--

--

Andrej Karpathy
Andrej Karpathy

Written by Andrej Karpathy

61K followers
186 following

I like to train deep neural nets on large datasets.

Help

Status

About

Careers

Press

Blog

Store

Privacy

Rules

Terms

Text to speech

\ No newline at end of file diff --git a/examples/karpathy/scripts/fetch-data.sh b/examples/karpathy/scripts/fetch-data.sh index 4cc6136..25dc15c 100644 --- a/examples/karpathy/scripts/fetch-data.sh +++ b/examples/karpathy/scripts/fetch-data.sh @@ -25,7 +25,7 @@ echo "" echo "--- Fetching blog posts ---" BLOG_SLUGS="state-of-cv rnn-effectiveness ai-short-peek rl phd medium software-2.0 recipe biohacking-lite forward-pass blockchain lecun1989 microgpt" -BLOG_URLS="https://karpathy.github.io/2012/10/22/state-of-computer-vision/ https://karpathy.github.io/2015/05/21/rnn-effectiveness/ https://karpathy.github.io/2015/11/14/ai/ https://karpathy.github.io/2016/05/31/rl/ https://karpathy.github.io/2016/09/07/phd/ https://karpathy.github.io/2018/01/20/medium/ https://karpathy.github.io/2017/01/25/software-2.0/ https://karpathy.github.io/2019/04/25/recipe/ https://karpathy.github.io/2020/06/11/biohacking-lite/ https://karpathy.github.io/2021/03/27/forward-pass/ https://karpathy.github.io/2021/06/21/blockchain/ https://karpathy.github.io/2022/03/14/lecun1989/ https://karpathy.github.io/2026/02/12/microgpt/" +BLOG_URLS="https://karpathy.github.io/2012/10/22/state-of-computer-vision/ https://karpathy.github.io/2015/05/21/rnn-effectiveness/ https://karpathy.github.io/2015/11/14/ai/ https://karpathy.github.io/2016/05/31/rl/ https://karpathy.github.io/2016/09/07/phd/ https://karpathy.github.io/2018/01/20/medium/ https://karpathy.medium.com/software-2-0-a64152b37c35 https://karpathy.github.io/2019/04/25/recipe/ https://karpathy.github.io/2020/06/11/biohacking-lite/ https://karpathy.github.io/2021/03/27/forward-pass/ https://karpathy.github.io/2021/06/21/blockchain/ https://karpathy.github.io/2022/03/14/lecun1989/ https://karpathy.github.io/2026/02/12/microgpt/" set -- $BLOG_URLS for slug in $BLOG_SLUGS; do @@ -118,9 +118,19 @@ for repo in "${REPOS[@]}"; do echo " [cached] $repo" else echo " [fetch] $repo" - curl -fsSL --max-time 30 "https://raw.githubusercontent.com/${repo}/master/README.md" -o "$outfile" \ - || curl -fsSL --max-time 30 "https://raw.githubusercontent.com/${repo}/main/README.md" -o "$outfile" \ - || echo " [WARN] Failed to fetch README for $repo" + # Branch and capitalisation both vary across these repos (char-rnn ships + # Readme.md on master), so try each combination before giving up. + fetched="" + for branch in master main; do + for name in README.md Readme.md readme.md; do + if curl -fsSL --max-time 30 \ + "https://raw.githubusercontent.com/${repo}/${branch}/${name}" -o "$outfile"; then + fetched="$branch/$name" + break 2 + fi + done + done + [ -n "$fetched" ] || echo " [WARN] Failed to fetch README for $repo" sleep 0.3 fi done diff --git a/examples/steipete/README.md b/examples/steipete/README.md index 6d27ae7..9b85c71 100644 --- a/examples/steipete/README.md +++ b/examples/steipete/README.md @@ -41,7 +41,8 @@ steipete-soul/ └── x/ ├── steipete.profile.json ← @steipete profile (496k followers) ├── steipete.posts-original.json ← 100 timeline posts (filtered to 36 self-authored) - ├── steipete.replies.json + ├── steipete.replies.json ← EMPTY ([]) — this pull failed silently and + │ needs a re-run: SURF_API_KEY=... node scripts/pull-x.mjs ├── openclaw.profile.json ← @openclaw profile ├── openclaw.posts-original.json ← 100 posts (59 openclaw-authored) ├── openclaw.replies.json ← 100 replies From 3788d93b191bd1363eea8d0d251fd4d189de4b19 Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars <61592645+aaronjmars@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:57:37 -0400 Subject: [PATCH 9/9] =?UTF-8?q?optimize:=20quick=20wins=20=E2=80=94=20stal?= =?UTF-8?q?e=20facts=20and=20a=20self-contradicting=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .github/README.md: every star count in the frameworks table was stale and UNDERSTATED. Verified all 9 repos against the GitHub API: OpenClaw 322k -> 383k, Nanobot 34.6k -> 46k, ZeroClaw 27.8k -> 32.3k, NanoClaw 24k -> 30.3k, PicoClaw 25.3k -> 29.7k, OpenFang 14.9k -> 18k, IronClaw 10.4k -> 12.5k, and Hermes Agent 8.7k -> 218k (checked full_name for a redirect; there is none). Re-sorted, since the table is ordered by stars. - garry-tan/weak_model_test.md embedded a "reference implementation" using the openai SDK, OPENAI_API_KEY and a --prompts flag. The real run_weak_model.py uses urllib + OpenRouter + certifi and has no such flag. After the earlier commit corrected the invocation above it, the file contradicted itself. Replaced the 40-line fake with a pointer to the actual script. - karpathy/fetch-data.sh: the transcript cache checked for .vtt files, but only the derived .en.txt is committed (0 .vtt in the repo) — so every transcript re-downloaded on every run despite the script advertising "Idempotent". Now checks the .txt that actually exists. - karpathy/fetch-data.sh: "YouTube video IDs saved to SOURCES.md" claimed an action the script never performs; SOURCES.md is hand-maintained. - garry-tan/README.md: `claude --skill soul` -> `/soul` to match every other doc; dropped the pre-merge "structured to be dropped into" line. - .markdownlint-cli2.jsonc: dropped the vestigial node_modules ignore (no package.json in this repo). --- .github/README.md | 16 ++++---- .markdownlint-cli2.jsonc | 3 +- examples/garry-tan/README.md | 3 +- examples/garry-tan/tests/weak_model_test.md | 43 ++------------------- examples/karpathy/scripts/fetch-data.sh | 4 +- 5 files changed, 15 insertions(+), 54 deletions(-) diff --git a/.github/README.md b/.github/README.md index a8efe34..cf2eba2 100644 --- a/.github/README.md +++ b/.github/README.md @@ -110,14 +110,14 @@ Soul files are plain markdown — if an agent can read files, it can embody you. | Framework | Language | Stars | |-----------|----------|-------| | [Aeon](https://github.com/aeonfun/aeon) | YAML/Markdown | — | -| [OpenClaw](https://github.com/openclaw/openclaw) | TypeScript | 322k | -| [Nanobot](https://github.com/HKUDS/nanobot) | Python | 34.6k | -| [ZeroClaw](https://github.com/zeroclaw-labs/zeroclaw) | Rust | 27.8k | -| [PicoClaw](https://github.com/sipeed/picoclaw) | Go | 25.3k | -| [NanoClaw](https://github.com/qwibitai/nanoclaw) | TypeScript | 24k | -| [OpenFang](https://github.com/RightNow-AI/openfang) | Rust | 14.9k | -| [IronClaw](https://github.com/nearai/ironclaw) | Rust | 10.4k | -| [Hermes Agent](https://github.com/NousResearch/hermes-agent) | Python | 8.7k | +| [OpenClaw](https://github.com/openclaw/openclaw) | TypeScript | 383k | +| [Hermes Agent](https://github.com/NousResearch/hermes-agent) | Python | 218k | +| [Nanobot](https://github.com/HKUDS/nanobot) | Python | 46k | +| [ZeroClaw](https://github.com/zeroclaw-labs/zeroclaw) | Rust | 32.3k | +| [NanoClaw](https://github.com/qwibitai/nanoclaw) | TypeScript | 30.3k | +| [PicoClaw](https://github.com/sipeed/picoclaw) | Go | 29.7k | +| [OpenFang](https://github.com/RightNow-AI/openfang) | Rust | 18k | +| [IronClaw](https://github.com/nearai/ironclaw) | Rust | 12.5k | | Claude Code · OpenCode · Codex · Goose | various | — | Also works with any model via system prompt — see [Using With Other Tools](#using-with-other-tools). diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index ae00c37..f1affb7 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -14,6 +14,5 @@ "MD053": true, // every link/image reference definition must be used "MD054": true // disallowed malformed link/image styles }, - "globs": ["**/*.md"], - "ignores": ["node_modules"] + "globs": ["**/*.md"] } diff --git a/examples/garry-tan/README.md b/examples/garry-tan/README.md index 5d8756c..841a5ea 100644 --- a/examples/garry-tan/README.md +++ b/examples/garry-tan/README.md @@ -2,7 +2,6 @@ A `soul.md` identity distillation for Garry Tan — co-founder of Posterous, founding partner of Initialized Capital, president and CEO of Y Combinator. -This folder is structured to be dropped into `aeonfun/soul.md` at `examples/garry-tan/`. ## What's in here @@ -48,7 +47,7 @@ python scripts/fetch_yt.py ```bash # From a project root, copy into a /soul directory and invoke -claude --skill soul "write a tweet reacting to the latest OpenAI announcement" +/soul write a tweet reacting to the latest OpenAI announcement ``` The SKILL.md from the parent `soul.md` repo handles the embodiment logic. diff --git a/examples/garry-tan/tests/weak_model_test.md b/examples/garry-tan/tests/weak_model_test.md index 19889cb..5b30591 100644 --- a/examples/garry-tan/tests/weak_model_test.md +++ b/examples/garry-tan/tests/weak_model_test.md @@ -107,47 +107,10 @@ The three things that usually break weak-model adherence: --- -## Run script (reference implementation) - -```python -# tests/run_weak_model.py -import argparse, json, os, re, sys -from openai import OpenAI - -ap = argparse.ArgumentParser() -ap.add_argument("--model", default="gpt-4o-mini") -ap.add_argument("--soul", required=True) -ap.add_argument("--prompts", required=True) # comma-sep .md files -ap.add_argument("--out", required=True) -args = ap.parse_args() - -client = OpenAI(api_key=os.environ["OPENAI_API_KEY"]) -soul = open(args.soul).read() -prompts = [] -for f in args.prompts.split(","): - prompts += re.findall(r"^## \d+\..*?\n(.*?)(?=\n---|\Z)", open(f).read(), re.S|re.M) - -system = ( - "You are Garry Tan. Embody the identity below. Never break character. " - "Match the voice in good-outputs.md, avoid bad-outputs.md.\n\n" - f"\n{soul}\n" -) - -with open(args.out, "w") as out: - for i, p in enumerate(prompts, 1): - resp = client.chat.completions.create( - model=args.model, - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": p.strip()}, - ], - temperature=0.7, - ) - out.write(f"## Prompt {i}\n\n{p.strip()}\n\n**Response:**\n\n{resp.choices[0].message.content}\n\n---\n\n") -print(f"wrote → {args.out}", file=sys.stderr) -``` +## Run script -Run it, grade it, commit the result. +The harness is [`run_weak_model.py`](run_weak_model.py) — argparse CLI, OpenRouter +via `urllib`, prompts defined in the script. Run it, grade it, commit the result. --- diff --git a/examples/karpathy/scripts/fetch-data.sh b/examples/karpathy/scripts/fetch-data.sh index 25dc15c..c75dab5 100644 --- a/examples/karpathy/scripts/fetch-data.sh +++ b/examples/karpathy/scripts/fetch-data.sh @@ -67,7 +67,7 @@ YOUTUBE_VIDEOS=( if command -v yt-dlp &>/dev/null; then for vid in "${YOUTUBE_VIDEOS[@]}"; do outfile="$DATA/transcripts/${vid}.vtt" - if [ -f "$outfile" ] || [ -f "$DATA/transcripts/${vid}.en.vtt" ]; then + if [ -s "$DATA/transcripts/${vid}.en.txt" ]; then echo " [cached] $vid" else echo " [fetch] $vid" @@ -90,7 +90,7 @@ if command -v yt-dlp &>/dev/null; then done else echo " [SKIP] yt-dlp not found. Install: pip install yt-dlp" - echo " YouTube video IDs saved to $DATA/transcripts/SOURCES.md" + echo " YouTube video IDs are listed in $DATA/transcripts/SOURCES.md" fi echo ""