diff --git a/mcp/README.md b/mcp/README.md index 4dfeb2d..c4013f5 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -117,6 +117,30 @@ gate writes. Every tool is timeout- and SSRF-guarded, caps its output, and retur structured error instead of hanging. The browser is pre-warmed at startup, so the first call takes about 100 ms. +## Token cost + +Handing an agent a raw web page is expensive. The "Web scraping" Wikipedia article is +68,240 tokens of HTML; Nike's homepage is 353,000 (tiktoken cl100k_base). An agent that +drops a page into context pays that before it reads a word. + +A built-in web fetch handles the easy case well. It summarizes readable pages cheaply and +clears anti-bot on many sites. It returns nothing on JavaScript-rendered pages and some hard +walls, and the agent then falls back to the raw HTML and still fails. That gap is where the +Fortress MCP earns its place: one call, bounded clean output, on pages the built-in tool +cannot read. Measured on one residential IP: + +| Page | Raw HTML | Built-in web fetch | Fortress MCP | +|---|---|---|---| +| Wikipedia (Web scraping) | 68,240 tok | ~950 tok summary | full clean markdown | +| Nike homepage | 353,241 tok | HTTP 403 | ~700 tok clean text | +| quotes.toscrape.com/js | empty shell | "NO QUOTES FOUND" | 285 tok, all 10 quotes | +| Ticketmaster | 151,946 tok | cleared | cleared | +| Indeed | 403 to a naive client | cleared | cleared | +| Hacker News | 11,765 tok | cleared | cleared | + +Reproduce it with [`benchmark_tokens.py`](benchmark_tokens.py): it counts the tokens each path +returns for a list of URLs and flags which ones a naive client is blocked on. + ## Benchmarks The same tasks, run once with an agent's built-in web fetch and again through the Fortress MCP: diff --git a/mcp/benchmark_tokens.py b/mcp/benchmark_tokens.py new file mode 100644 index 0000000..ed8418f --- /dev/null +++ b/mcp/benchmark_tokens.py @@ -0,0 +1,91 @@ +"""Token cost of giving an AI agent a web page: raw HTML vs the Fortress MCP. + +For each URL this prints: + - raw HTML tokens (what a naive agent dumps into context), or the block status + a plain HTTP client hits + - Fortress clean tokens (what the MCP returns), when `tilion` is installed + +A built-in web fetch (e.g. Claude Code's WebFetch) sits between these: it summarizes +readable pages cheaply and clears anti-bot on many sites, but it returns nothing on +JavaScript-rendered pages and some hard walls. This script measures the two ends you +can reproduce anywhere; the middle column depends on your client. + +Tokens are counted with tiktoken cl100k_base (an approximation for other tokenizers). + + pip install tiktoken + pip install "tilion[mcp]" # optional, enables the Fortress column + python benchmark_tokens.py + python benchmark_tokens.py --no-fortress # raw-HTML column only +""" +from __future__ import annotations +import sys + +URLS = [ + "https://en.wikipedia.org/wiki/Web_scraping", + "https://www.nike.com/", + "https://quotes.toscrape.com/js/", + "https://www.ticketmaster.com/", + "https://news.ycombinator.com/", +] + +_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/125.0 Safari/537.36") + +try: + import tiktoken + _enc = tiktoken.get_encoding("cl100k_base") + def ntok(s: str) -> int: + return len(_enc.encode(s)) +except Exception: + def ntok(s: str) -> int: + return len(s) // 4 # rough fallback if tiktoken is missing + + +def raw_html(url: str): + import urllib.request, ssl + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + req = urllib.request.Request(url, headers={"User-Agent": _UA}) + try: + with urllib.request.urlopen(req, timeout=25, context=ctx) as r: + return r.read().decode("utf-8", "replace"), None + except Exception as e: + return None, type(e).__name__ + + +def fortress_text(url: str): + """What the Fortress MCP returns for the page. Needs `pip install tilion`.""" + try: + import asyncio + from tilion import Tilion + except Exception: + return None, "tilion not installed" + + async def run(): + async with Tilion(headless=True) as t: + r = await t.fetch(url) + return r.get("text") or r.get("markdown") or "" + + try: + return asyncio.run(run()), None + except Exception as e: + return None, type(e).__name__ + + +def main(): + want_fortress = "--no-fortress" not in sys.argv + print(f"{'url':46s} {'raw HTML':>18s} {'Fortress MCP':>18s}") + print("-" * 84) + for url in URLS: + html, err = raw_html(url) + raw_col = f"{ntok(html):,} tok" if html else f"blocked ({err})" + f_col = "" + if want_fortress: + text, ferr = fortress_text(url) + f_col = f"{ntok(text):,} tok" if text else (ferr or "-") + print(f"{url:46s} {raw_col:>18s} {f_col:>18s}") + + +if __name__ == "__main__": + main()