Update run-evals.py

Author: RangeKing

scripts/run-evals.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import json
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -54,17 +55,111 @@
     "evals/evals.json",
 ]
 
+MAX_SKILL_NAME_LENGTH = 64
+
+
+def parse_frontmatter_minimal(frontmatter_text: str) -> tuple[dict | None, str | None]:
+    parsed: dict[str, object] = {}
+
+    for raw_line in frontmatter_text.splitlines():
+        if not raw_line.strip():
+            continue
+        if raw_line.startswith(" ") or raw_line.startswith("\t"):
+            continue
+        if ":" not in raw_line:
+            return None, f"Unsupported frontmatter line: {raw_line}"
+
+        key, value = raw_line.split(":", 1)
+        key = key.strip()
+        value = value.strip()
+
+        if not key:
+            return None, f"Invalid frontmatter key in line: {raw_line}"
+
+        if value.startswith('"') and value.endswith('"') and len(value) >= 2:
+            parsed[key] = value[1:-1]
+        elif value.startswith("'") and value.endswith("'") and len(value) >= 2:
+            parsed[key] = value[1:-1]
+        elif value == "":
+            parsed[key] = None
+        else:
+            parsed[key] = value
+
+    return parsed, None
+
+
+def local_quick_validate(skill_dir: Path) -> tuple[bool, str]:
+    skill_md = skill_dir / "SKILL.md"
+    if not skill_md.exists():
+        return False, "SKILL.md not found"
+
+    content = skill_md.read_text()
+    if not content.startswith("---"):
+        return False, "No YAML frontmatter found"
+
+    match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
+    if not match:
+        return False, "Invalid frontmatter format"
+
+    frontmatter_text = match.group(1)
+
+    frontmatter, parse_error = parse_frontmatter_minimal(frontmatter_text)
+    if parse_error:
+        return False, parse_error
+    if not isinstance(frontmatter, dict):
+        return False, "Frontmatter must be a YAML dictionary"
+
+    allowed_properties = {"name", "description", "license", "allowed-tools", "metadata"}
+    unexpected_keys = set(frontmatter.keys()) - allowed_properties
+    if unexpected_keys:
+        unexpected = ", ".join(sorted(unexpected_keys))
+        allowed = ", ".join(sorted(allowed_properties))
+        return (
+            False,
+            f"Unexpected key(s) in SKILL.md frontmatter: {unexpected}. Allowed properties are: {allowed}",
+        )
+
+    if "name" not in frontmatter:
+        return False, "Missing 'name' in frontmatter"
+    if "description" not in frontmatter:
+        return False, "Missing 'description' in frontmatter"
+
+    name = frontmatter.get("name", "")
+    if not isinstance(name, str):
+        return False, f"Name must be a string, got {type(name).__name__}"
+    name = name.strip()
+    if not re.match(r"^[a-z0-9-]+$", name):
+        return False, f"Name '{name}' should be hyphen-case"
+    if name.startswith("-") or name.endswith("-") or "--" in name:
+        return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens"
+    if len(name) > MAX_SKILL_NAME_LENGTH:
+        return False, f"Name is too long ({len(name)} characters). Maximum is {MAX_SKILL_NAME_LENGTH}."
+
+    description = frontmatter.get("description", "")
+    if not isinstance(description, str):
+        return False, f"Description must be a string, got {type(description).__name__}"
+    description = description.strip()
+    if "<" in description or ">" in description:
+        return False, "Description cannot contain angle brackets (< or >)"
+    if len(description) > 1024:
+        return False, f"Description is too long ({len(description)} characters). Maximum is 1024."
+
+    return True, "Local fallback validation passed."
+
 
 def run_quick_validate(skill_dir: Path) -> tuple[bool, str]:
     validator = Path.home() / ".codex/skills/.system/skill-creator/scripts/quick_validate.py"
-    proc = subprocess.run(
-        [sys.executable, str(validator), str(skill_dir)],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    output = (proc.stdout + proc.stderr).strip()
-    return proc.returncode == 0, output
+    if validator.exists():
+        proc = subprocess.run(
+            [sys.executable, str(validator), str(skill_dir)],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        output = (proc.stdout + proc.stderr).strip()
+        return proc.returncode == 0, output
+
+    return local_quick_validate(skill_dir)
 
 
 def check_file_exists(skill_dir: Path) -> tuple[bool, list[str]]: