|
| 1 | +"""codebase_mapper.languages.clojure — Tier-1 Clojure / ClojureScript support. |
| 2 | +
|
| 3 | +There is no maintained PyPI tree-sitter-clojure grammar, so — like the Python |
| 4 | +analyzer (stdlib ``ast``) — this is a self-contained reader. Clojure is |
| 5 | +homoiconic: code is data made of s-expressions, so a small string/char/ |
| 6 | +comment-aware tokenizer plus an iterative (stack-based, no recursion ceiling) |
| 7 | +reader recovers the structural surface we index: |
| 8 | +
|
| 9 | + * ``namespace`` — the ``(ns my.app.core ...)`` name. |
| 10 | + * ``imports`` — namespaces pulled in via ``(:require ...)`` / ``(:use ...)`` |
| 11 | + inside the ns form (kind ``"require"``). ``:import`` (Java |
| 12 | + interop classes) is out of scope — those aren't repo files. |
| 13 | + * ``items`` — one record per top-level ``def``-form (``defn``/``defn-``/ |
| 14 | + ``defmacro``/``defmulti``/``defmethod``/``defonce`` -> |
| 15 | + function; ``def`` -> var; ``defrecord`` -> record; |
| 16 | + ``deftype`` -> type; ``defprotocol``/``definterface`` -> |
| 17 | + protocol; ``ns`` -> namespace), each with line/byte spans |
| 18 | + (powers L2 chunking + the symbol surface). |
| 19 | +
|
| 20 | +Public surface mirrors the other analyzers: |
| 21 | +
|
| 22 | + * ``extract_clojure_ast_summary(content, path) -> (summary, errors)`` |
| 23 | + * ``resolve_clojure_imports(src_path, summary, paths_set) -> (in_repo, external)`` |
| 24 | +
|
| 25 | +Known limits (documented, not silent): ``#_`` form-discard is not honored, and |
| 26 | +reader metadata (``^...``) is dropped — both affect only rare item-naming edge |
| 27 | +cases, never the namespace/require surface. |
| 28 | +""" |
| 29 | +from __future__ import annotations |
| 30 | + |
| 31 | +CLOJURE_EXTENSIONS = (".clj", ".cljs", ".cljc", ".cljr") |
| 32 | + |
| 33 | +# Top-level def-form head -> item kind. |
| 34 | +_DEF_KINDS: dict[str, str] = { |
| 35 | + "defn": "function", "defn-": "function", "defmacro": "function", |
| 36 | + "defmulti": "function", "defmethod": "function", "defonce": "function", |
| 37 | + "def": "var", |
| 38 | + "defrecord": "record", "deftype": "type", |
| 39 | + "defprotocol": "protocol", "definterface": "protocol", |
| 40 | + "ns": "namespace", |
| 41 | +} |
| 42 | + |
| 43 | +_WS = " \t\r\n," |
| 44 | +_DELIMS = '()[]{}' |
| 45 | + |
| 46 | + |
| 47 | +def _tokenize(text: str) -> list[tuple]: |
| 48 | + """Yield (type, value, start, end, line) tokens. String/char-literal/comment |
| 49 | + aware so delimiters inside them never miscount. Types: open, close, sym, kw, |
| 50 | + str, char, meta, discard.""" |
| 51 | + toks: list[tuple] = [] |
| 52 | + i, n, line = 0, len(text), 1 |
| 53 | + while i < n: |
| 54 | + c = text[i] |
| 55 | + if c == "\n": |
| 56 | + line += 1 |
| 57 | + i += 1 |
| 58 | + continue |
| 59 | + if c in " \t\r,": |
| 60 | + i += 1 |
| 61 | + continue |
| 62 | + if c == ";": # line comment |
| 63 | + while i < n and text[i] != "\n": |
| 64 | + i += 1 |
| 65 | + continue |
| 66 | + if c == '"': # string literal |
| 67 | + start, sline = i, line |
| 68 | + i += 1 |
| 69 | + while i < n: |
| 70 | + if text[i] == "\\": |
| 71 | + i += 2 |
| 72 | + continue |
| 73 | + if text[i] == "\n": |
| 74 | + line += 1 |
| 75 | + if text[i] == '"': |
| 76 | + i += 1 |
| 77 | + break |
| 78 | + i += 1 |
| 79 | + toks.append(("str", text[start:i], start, i, sline)) |
| 80 | + continue |
| 81 | + if c == "\\": # character literal: \x, \newline, A ... |
| 82 | + start = i |
| 83 | + i += 1 |
| 84 | + if i < n: |
| 85 | + if text[i].isalpha(): |
| 86 | + i += 1 |
| 87 | + while i < n and (text[i].isalnum() or text[i] == "-"): |
| 88 | + i += 1 |
| 89 | + else: |
| 90 | + i += 1 |
| 91 | + toks.append(("char", text[start:i], start, i, line)) |
| 92 | + continue |
| 93 | + if c == "#" and i + 1 < n: |
| 94 | + nxt = text[i + 1] |
| 95 | + if nxt == "_": # discard next form |
| 96 | + toks.append(("discard", "#_", i, i + 2, line)) |
| 97 | + i += 2 |
| 98 | + continue |
| 99 | + if nxt == '"': # regex literal #"..." |
| 100 | + start, sline = i, line |
| 101 | + i += 2 |
| 102 | + while i < n: |
| 103 | + if text[i] == "\\": |
| 104 | + i += 2 |
| 105 | + continue |
| 106 | + if text[i] == "\n": |
| 107 | + line += 1 |
| 108 | + if text[i] == '"': |
| 109 | + i += 1 |
| 110 | + break |
| 111 | + i += 1 |
| 112 | + toks.append(("str", text[start:i], start, i, sline)) |
| 113 | + continue |
| 114 | + if nxt in "({": # set #{...} or anon-fn #(...) |
| 115 | + toks.append(("open", text[i:i + 2], i, i + 2, line)) |
| 116 | + i += 2 |
| 117 | + continue |
| 118 | + # #' var-quote, #= etc — fall through; `#` joins the next atom. |
| 119 | + if c == "^": # metadata marker |
| 120 | + toks.append(("meta", "^", i, i + 1, line)) |
| 121 | + i += 1 |
| 122 | + continue |
| 123 | + if c in "'`~@": # quote / syntax-quote / unquote / deref — structural no-ops |
| 124 | + i += 1 |
| 125 | + continue |
| 126 | + if c in "([{": |
| 127 | + toks.append(("open", c, i, i + 1, line)) |
| 128 | + i += 1 |
| 129 | + continue |
| 130 | + if c in ")]}": |
| 131 | + toks.append(("close", c, i, i + 1, line)) |
| 132 | + i += 1 |
| 133 | + continue |
| 134 | + # atom: symbol / keyword / number |
| 135 | + start = i |
| 136 | + while i < n and text[i] not in _WS and text[i] not in _DELIMS and text[i] != ";": |
| 137 | + i += 1 |
| 138 | + val = text[start:i] |
| 139 | + if not val: # safety: never stall |
| 140 | + i += 1 |
| 141 | + continue |
| 142 | + toks.append(("kw" if val.startswith(":") else "sym", val, start, i, line)) |
| 143 | + return toks |
| 144 | + |
| 145 | + |
| 146 | +def _parse(toks: list[tuple]) -> list[dict]: |
| 147 | + """Iterative (stack-based) reader -> list of top-level form nodes. |
| 148 | +
|
| 149 | + A list/vector/map node is ``{"kind": "list", "delim", "children", "start", |
| 150 | + "end", "line"}``; leaves are ``{"kind": "sym"|"kw"|"str"|"char", "value", |
| 151 | + "start", "end", "line"}``. ``meta`` / ``discard`` tokens are dropped (see |
| 152 | + module docstring). Stack-based so a deeply-nested file cannot overflow. |
| 153 | + """ |
| 154 | + root: list[dict] = [] |
| 155 | + stack: list[list[dict]] = [root] |
| 156 | + open_nodes: list[dict] = [] |
| 157 | + for ttype, val, start, end, line in toks: |
| 158 | + if ttype == "open": |
| 159 | + node = {"kind": "list", "delim": val[-1], "children": [], |
| 160 | + "start": start, "end": end, "line": line} |
| 161 | + stack[-1].append(node) |
| 162 | + stack.append(node["children"]) |
| 163 | + open_nodes.append(node) |
| 164 | + elif ttype == "close": |
| 165 | + if len(stack) > 1: |
| 166 | + stack.pop() |
| 167 | + open_nodes.pop()["end"] = end |
| 168 | + elif ttype in ("sym", "kw", "str", "char"): |
| 169 | + stack[-1].append({"kind": ttype, "value": val, |
| 170 | + "start": start, "end": end, "line": line}) |
| 171 | + # meta / discard: ignored structurally |
| 172 | + return root |
| 173 | + |
| 174 | + |
| 175 | +def _first_name_sym(children: list[dict]) -> str | None: |
| 176 | + """The def-form's name: the first ``sym`` after the head (index 0), skipping |
| 177 | + dropped metadata maps / keywords.""" |
| 178 | + for child in children[1:]: |
| 179 | + if child.get("kind") == "sym": |
| 180 | + return child["value"] |
| 181 | + return None |
| 182 | + |
| 183 | + |
| 184 | +def _ns_requires(ns_node: dict) -> list[str]: |
| 185 | + """Required namespaces from an ns form's ``(:require ...)`` / ``(:use ...)`` |
| 186 | + clauses. Each spec is a bare symbol (``foo.bar``) or a vector whose first |
| 187 | + element is the namespace (``[foo.bar :as fb :refer [x]]``).""" |
| 188 | + out: list[str] = [] |
| 189 | + for clause in ns_node.get("children", []): |
| 190 | + if clause.get("kind") != "list" or not clause.get("children"): |
| 191 | + continue |
| 192 | + head = clause["children"][0] |
| 193 | + if head.get("kind") == "kw" and head["value"] in (":require", ":use"): |
| 194 | + for spec in clause["children"][1:]: |
| 195 | + if spec.get("kind") == "sym": |
| 196 | + out.append(spec["value"]) |
| 197 | + elif (spec.get("kind") == "list" and spec.get("delim") == "[" |
| 198 | + and spec.get("children")): |
| 199 | + first = spec["children"][0] |
| 200 | + if first.get("kind") == "sym": |
| 201 | + out.append(first["value"]) |
| 202 | + return out |
| 203 | + |
| 204 | + |
| 205 | +def extract_clojure_ast_summary(content: bytes, path: str) -> tuple[dict | None, list[str]]: |
| 206 | + try: |
| 207 | + text = content.decode("utf-8") |
| 208 | + except UnicodeDecodeError: |
| 209 | + return None, ["clojure_decode_error"] |
| 210 | + |
| 211 | + forms = _parse(_tokenize(text)) |
| 212 | + |
| 213 | + namespace: str | None = None |
| 214 | + items: list[dict] = [] |
| 215 | + funcs: list[str] = [] |
| 216 | + types: list[str] = [] |
| 217 | + requires: list[tuple[str, int]] = [] |
| 218 | + |
| 219 | + for form in forms: |
| 220 | + if form.get("kind") != "list" or not form.get("children"): |
| 221 | + continue |
| 222 | + head = form["children"][0] |
| 223 | + if head.get("kind") != "sym": |
| 224 | + continue |
| 225 | + h = head["value"] |
| 226 | + kind = _DEF_KINDS.get(h) |
| 227 | + if kind is None: |
| 228 | + continue |
| 229 | + name = _first_name_sym(form["children"]) |
| 230 | + if name is None: |
| 231 | + continue |
| 232 | + line_start = form["line"] |
| 233 | + line_end = line_start + text[form["start"]:form["end"]].count("\n") |
| 234 | + items.append({ |
| 235 | + "kind": kind, |
| 236 | + "name": name, |
| 237 | + "parent": None, |
| 238 | + "line_start": line_start, |
| 239 | + "line_end": line_end, |
| 240 | + "byte_start": form["start"], |
| 241 | + "byte_end": form["end"], |
| 242 | + }) |
| 243 | + if h == "ns": |
| 244 | + namespace = name |
| 245 | + for ns in _ns_requires(form): |
| 246 | + requires.append((ns, line_start)) |
| 247 | + elif kind == "function": |
| 248 | + funcs.append(name) |
| 249 | + elif kind in ("record", "type", "protocol"): |
| 250 | + types.append(name) |
| 251 | + |
| 252 | + # Dedupe imports on the namespace, keep first line, sort. |
| 253 | + seen: set[str] = set() |
| 254 | + imports: list[dict] = [] |
| 255 | + for ns, lineno in requires: |
| 256 | + if ns in seen: |
| 257 | + continue |
| 258 | + seen.add(ns) |
| 259 | + imports.append({"kind": "require", "source": ns, "lineno": lineno}) |
| 260 | + imports.sort(key=lambda x: (x["lineno"], x["source"])) |
| 261 | + |
| 262 | + return { |
| 263 | + "language": "clojure", |
| 264 | + "namespace": namespace, |
| 265 | + "imports": imports, |
| 266 | + "top_level_functions": sorted(set(funcs)), |
| 267 | + "top_level_classes": sorted(set(types)), |
| 268 | + "items": items, |
| 269 | + }, [] |
| 270 | + |
| 271 | + |
| 272 | +def _ns_to_relpath(ns: str) -> str: |
| 273 | + """Clojure namespace -> source-relative path stem: dots become slashes and |
| 274 | + dashes become underscores (the Clojure file-naming convention).""" |
| 275 | + return ns.replace("-", "_").replace(".", "/") |
| 276 | + |
| 277 | + |
| 278 | +def resolve_clojure_imports( |
| 279 | + src_path: str, summary: dict, paths_set: set[str], |
| 280 | +) -> tuple[list[str], list[str]]: |
| 281 | + """Resolve required namespaces to in-repo files; everything else (stdlib |
| 282 | + ``clojure.*``, third-party) is external. A namespace ``a.b-c`` maps to |
| 283 | + ``a/b_c.clj[cs|c|r]`` under any source root (matched as a path suffix).""" |
| 284 | + in_repo: set[str] = set() |
| 285 | + external: set[str] = set() |
| 286 | + for imp in summary.get("imports", []): |
| 287 | + ns = imp.get("source") |
| 288 | + if not ns: |
| 289 | + continue |
| 290 | + rel = _ns_to_relpath(ns) |
| 291 | + match: str | None = None |
| 292 | + for ext in CLOJURE_EXTENSIONS: |
| 293 | + cand = rel + ext |
| 294 | + hits = [p for p in paths_set if p == cand or p.endswith("/" + cand)] |
| 295 | + if hits: |
| 296 | + match = sorted(hits)[0] |
| 297 | + break |
| 298 | + if match is not None: |
| 299 | + in_repo.add(match) |
| 300 | + else: |
| 301 | + external.add(ns) |
| 302 | + return sorted(in_repo), sorted(external) |
0 commit comments