feat(scraper): SCRAPE_CACHE_MODE and --no-fetch-cache for crawl4ai (#50)

.env.example

@@ -14,6 +14,11 @@ FIRECRAWL_API_KEY=
 # SCRAPE_DISCOVER_PROVIDER=crawl4ai
 # SCRAPE_FETCH_PROVIDER=firecrawl
 
+# Provider cache override (king-scrape, currently honoured by crawl4ai only).
+# Allowed values: default (or unset), enabled, bypass, disabled, read_only,
+# write_only. The CLI flag --no-fetch-cache is shorthand for SCRAPE_CACHE_MODE=bypass.
+# SCRAPE_CACHE_MODE=default
+
 # Optional. OpenRouter API key for OpenRouter LLM stages or Ollama fallback.
 # Get yours at https://openrouter.ai
 OPENROUTER_API_KEY=

CHANGELOG.md

@@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `--no-fetch-cache` flag on `king-scrape` and `SCRAPE_CACHE_MODE` env
+  var (#50). Bypasses the Crawl4AI provider's local cache (`~/.crawl4ai/`)
+  for the duration of the run without wiping the cache directory by hand.
+  `SCRAPE_CACHE_MODE` accepts `bypass`, `disabled`, `read_only`,
+  `write_only`, or `default`/unset. The CLI flag is shorthand for
+  `SCRAPE_CACHE_MODE=bypass` and uses `setdefault` semantics so an
+  explicit pre-existing env value wins (mirrors `--provider`'s
+  precedence). `main()` restores the prior env value on exit so the
+  flag does not leak into an embedding application or test session.
+  Honoured by the crawl4ai provider; firecrawl ignores it (its API
+  defaults to fresh-fetch). Knob is global today; per-stage variants
+  (`SCRAPE_DISCOVER_CACHE_MODE` / `SCRAPE_FETCH_CACHE_MODE`) deferred
+  to a follow-up if real configurations need them. Lays the primitive
+  `king-scrape update <name>` needs to make `force_refresh=True`
+  actually fetch from the network.
 - Content-hash provenance through every layer of the scraper pipeline
   (ADR-0012). `Chunk` and `EnrichedChunk` carry a `content_hash` field
   populated by `sha256(content)`. Each fetched page now writes a sidecar

docs/CLI_GUIDE.md

@@ -367,6 +367,13 @@ Useful flags:
   `SCRAPE_PROVIDER` for the process. Stage-specific environment variables
   take precedence over the flag. See [Scraper providers](#scraper-providers)
   for the full table.
+- `--no-fetch-cache`: bypass the scraper provider's local cache for this
+  run. Sets `SCRAPE_CACHE_MODE=bypass`. Useful when upstream content has
+  changed and you need a fresh fetch without wiping the cache directory
+  by hand (Crawl4AI keeps a local cache under `~/.crawl4ai/`). Honoured
+  by the crawl4ai provider; firecrawl's API defaults to fresh-fetch and
+  ignores this flag. Set `SCRAPE_CACHE_MODE` directly to `bypass`,
+  `disabled`, `read_only`, or `write_only` for finer-grained control.
 
 `king-scrape` writes exported documentation JSON to `.king-context/data/`.
 Use `kctx index` to build or rebuild the file-based CLI store from that JSON.

installer/templates/env.example

@@ -14,6 +14,11 @@ FIRECRAWL_API_KEY=
 # SCRAPE_DISCOVER_PROVIDER=crawl4ai
 # SCRAPE_FETCH_PROVIDER=firecrawl
 
+# Provider cache override (king-scrape, currently honoured by crawl4ai only).
+# Allowed values: default (or unset), enabled, bypass, disabled, read_only,
+# write_only. The CLI flag --no-fetch-cache is shorthand for SCRAPE_CACHE_MODE=bypass.
+# SCRAPE_CACHE_MODE=default
+
 # Optional. OpenRouter API key for OpenRouter LLM stages or Ollama fallback.
 # Get yours at https://openrouter.ai
 OPENROUTER_API_KEY=

src/king_context/scraper/cli.py

@@ -346,6 +346,18 @@ def _build_parser() -> argparse.ArgumentParser:
             "have precedence."
         ),
     )
+    parser.add_argument(
+        "--no-fetch-cache",
+        dest="no_fetch_cache",
+        action="store_true",
+        help=(
+            "Bypass the scraper provider's local cache for this run. "
+            "Sets SCRAPE_CACHE_MODE=bypass. Useful when upstream content "
+            "has changed and you need a fresh fetch without wiping the "
+            "cache directory by hand. Honoured by the crawl4ai provider; "
+            "other providers may ignore it."
+        ),
+    )
     return parser
 
 
@@ -356,6 +368,14 @@ def main() -> None:
 
     parser = _build_parser()
     args = parser.parse_args()
+    cache_mode_was_set = "SCRAPE_CACHE_MODE" in os.environ
+    cache_mode_prior = os.environ.get("SCRAPE_CACHE_MODE")
+    if getattr(args, "no_fetch_cache", False):
+        # ``setdefault`` mirrors the precedence of ``--provider`` (cli.py
+        # above): an existing env wins. A contributor who set
+        # SCRAPE_CACHE_MODE=read_only and ALSO passes --no-fetch-cache keeps
+        # the explicit env value rather than being silently downgraded.
+        os.environ.setdefault("SCRAPE_CACHE_MODE", "bypass")
     config = load_config(
         enrichment_model=args.model,
         chunk_max_tokens=args.chunk_max_tokens,
@@ -371,6 +391,16 @@ def main() -> None:
     except ValueError as exc:
         print(f"error: {exc}", file=sys.stderr)
         sys.exit(2)
+    finally:
+        # Restore the env so calling ``main()`` from tests or from an
+        # embedding application does not leak the bypass flag into the
+        # next caller's process state.
+        if cache_mode_was_set:
+            # Type narrowing: was-set guarantees ``.get`` returned ``str``.
+            assert cache_mode_prior is not None
+            os.environ["SCRAPE_CACHE_MODE"] = cache_mode_prior
+        else:
+            os.environ.pop("SCRAPE_CACHE_MODE", None)
 
 
 if __name__ == "__main__":

src/scraper_providers/crawl4ai_provider.py

@@ -1,6 +1,8 @@
 """Crawl4AI scraper provider — local Playwright-based backend."""
 from __future__ import annotations
 
+import os
+import sys
 from datetime import datetime, timezone
 from typing import Any, Iterable
 
@@ -14,6 +16,7 @@
     from crawl4ai import (  # type: ignore[import-not-found]
         AsyncWebCrawler,
         BrowserConfig,
+        CacheMode,
         CrawlerRunConfig,
     )
     from crawl4ai.deep_crawling import (  # type: ignore[import-not-found]
@@ -23,11 +26,76 @@
 except ImportError:
     AsyncWebCrawler = None  # type: ignore[assignment,misc]
     BrowserConfig = None  # type: ignore[assignment,misc]
+    CacheMode = None  # type: ignore[assignment,misc]
     CrawlerRunConfig = None  # type: ignore[assignment,misc]
     BFSDeepCrawlStrategy = None  # type: ignore[assignment,misc]
     _CRAWL4AI_AVAILABLE = False
 
 
+# Maps the public ``SCRAPE_CACHE_MODE`` env value to the attribute name on
+# Crawl4AI's ``CacheMode`` enum. ``"default"`` is handled by an early return
+# in ``_resolve_cache_mode`` and is intentionally absent here.
+_CACHE_MODE_MAP = {
+    "enabled": "ENABLED",
+    "bypass": "BYPASS",
+    "disabled": "DISABLED",
+    "read_only": "READ_ONLY",
+    "write_only": "WRITE_ONLY",
+}
+
+# One-shot warning state so a typo in SCRAPE_CACHE_MODE does not produce
+# hundreds of duplicate stderr lines during a deep crawl. Keyed by the raw
+# value so different typos still each warn once. Tests reset this set via
+# ``monkeypatch.setattr(...)`` to keep warn-once assertions independent.
+_WARNED_CACHE_VALUES: set[str] = set()
+
+
+def _warn_unknown_cache_mode(raw: str) -> None:
+    if raw in _WARNED_CACHE_VALUES:
+        return
+    _WARNED_CACHE_VALUES.add(raw)
+    print(
+        f"warning: SCRAPE_CACHE_MODE='{raw}' not recognised; "
+        "using crawl4ai default",
+        file=sys.stderr,
+    )
+
+
+def _resolve_cache_mode() -> Any:
+    """Resolve ``SCRAPE_CACHE_MODE`` env to a ``CacheMode`` value or ``None``.
+
+    ``None`` means "use the library default" — callers omit ``cache_mode``
+    from ``CrawlerRunConfig`` rather than passing ``None``, so existing
+    behaviour is preserved when the env is unset.
+
+    Two distinct paths return ``None``: (1) the value is missing, ``default``,
+    or unrecognised — emits a stderr warning the first time per raw value;
+    (2) the ``crawl4ai`` library is not installed — silent because the
+    caller's ``_ensure_available`` will surface a clearer error before any
+    real fetch happens.
+    """
+    raw = (os.environ.get("SCRAPE_CACHE_MODE") or "").strip().lower()
+    if not raw or raw == "default":
+        return None
+    if CacheMode is None:
+        # Library missing: stay silent. The caller path won't reach a real
+        # crawl4ai call (the provider's _ensure_available raises a clearer
+        # ProviderUnavailableError), so a "not recognised" warning here would
+        # only confuse a user whose actual problem is a missing dependency.
+        return None
+    mapped = _CACHE_MODE_MAP.get(raw)
+    if mapped is None:
+        _warn_unknown_cache_mode(raw)
+        return None
+    try:
+        return getattr(CacheMode, mapped)
+    except AttributeError:
+        # Future crawl4ai version may rename the enum value. Fall back to
+        # default behaviour with a one-shot warning so the run continues.
+        _warn_unknown_cache_mode(raw)
+        return None
+
+
 _INSTALL_HINT = (
     "crawl4ai not installed in the active Python environment. "
     "If you installed via npx @king-context/cli, run "
@@ -103,10 +171,15 @@ class Crawl4AIDiscoveryProvider:
     async def discover_urls(self, base_url: str) -> list[str]:
         _ensure_available()
         _ensure_browser_present()
+        cache_kwargs: dict[str, Any] = {}
+        cache_mode = _resolve_cache_mode()
+        if cache_mode is not None:
+            cache_kwargs["cache_mode"] = cache_mode
         run_config = CrawlerRunConfig(  # type: ignore[misc]
             deep_crawl_strategy=BFSDeepCrawlStrategy(  # type: ignore[misc]
                 max_depth=2, include_external=False
             ),
+            **cache_kwargs,
         )
         try:
             async with AsyncWebCrawler(  # type: ignore[misc]
@@ -131,11 +204,23 @@ class Crawl4AIFetchProvider:
     async def fetch_one(self, url: str) -> PageContent:
         _ensure_available()
         _ensure_browser_present()
+        cache_mode = _resolve_cache_mode()
+        run_config = (
+            CrawlerRunConfig(cache_mode=cache_mode)  # type: ignore[misc]
+            if cache_mode is not None
+            else None
+        )
         try:
             async with AsyncWebCrawler(  # type: ignore[misc]
                 config=BrowserConfig(headless=True),  # type: ignore[misc]
             ) as crawler:
-                result = await crawler.arun(url=url)
+                if run_config is not None:
+                    result = await crawler.arun(url=url, config=run_config)
+                else:
+                    # Preserve the pre-fix call shape when no cache override is
+                    # set, so existing behaviour and any provider-side defaults
+                    # remain unchanged for users who have not opted in.
+                    result = await crawler.arun(url=url)
         except Exception as exc:
             if _is_browser_missing(exc):
                 raise ProviderUnavailableError("crawl4ai", _SETUP_HINT) from exc

tests/test_scraper/test_cli.py

@@ -1,6 +1,7 @@
 import argparse
 import asyncio
 import json
+import os
 from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -107,6 +108,7 @@ def test_cli_parse_args_basic():
     assert args.no_llm_filter is False
     assert args.no_auto_seed is False
     assert args.include_maybe is False
+    assert args.no_fetch_cache is False
 
 
 def test_cli_parse_args_with_flags():
@@ -142,6 +144,100 @@ def test_cli_parse_args_invalid_step():
         parser.parse_args(["https://docs.example.com", "--step", "invalid"])
 
 
+def test_cli_no_fetch_cache_flag_sets_env(monkeypatch):
+    """`--no-fetch-cache` exports SCRAPE_CACHE_MODE=bypass for the run."""
+    import sys
+    from king_context.scraper.cli import main
+    from king_context.scraper.config import ScraperConfig
+
+    captured: dict = {}
+
+    async def fake_run_pipeline(args, config):
+        captured["env"] = os.environ.get("SCRAPE_CACHE_MODE")
+
+    monkeypatch.delenv("SCRAPE_CACHE_MODE", raising=False)
+    monkeypatch.setattr(
+        "king_context.scraper.cli.run_pipeline", fake_run_pipeline
+    )
+    monkeypatch.setattr(
+        "king_context.scraper.cli.load_config",
+        lambda **_: ScraperConfig(),
+    )
+    monkeypatch.setattr(
+        sys, "argv",
+        ["king-scrape", "https://docs.example.com", "--no-fetch-cache"],
+    )
+
+    main()
+
+    assert captured["env"] == "bypass"
+    # main() must restore env on exit so the flag does not leak into other
+    # tests or into an embedding application that calls main() repeatedly.
+    assert "SCRAPE_CACHE_MODE" not in os.environ
+
+
+def test_cli_no_fetch_cache_respects_pre_existing_env(monkeypatch):
+    """A user who explicitly set SCRAPE_CACHE_MODE keeps that value; the
+    flag does not silently downgrade it. Mirrors --provider's setdefault
+    behaviour."""
+    import sys
+    from king_context.scraper.cli import main
+    from king_context.scraper.config import ScraperConfig
+
+    captured: dict = {}
+
+    async def fake_run_pipeline(args, config):
+        captured["env"] = os.environ.get("SCRAPE_CACHE_MODE")
+
+    monkeypatch.setenv("SCRAPE_CACHE_MODE", "disabled")
+    monkeypatch.setattr(
+        "king_context.scraper.cli.run_pipeline", fake_run_pipeline
+    )
+    monkeypatch.setattr(
+        "king_context.scraper.cli.load_config",
+        lambda **_: ScraperConfig(),
+    )
+    monkeypatch.setattr(
+        sys, "argv",
+        ["king-scrape", "https://docs.example.com", "--no-fetch-cache"],
+    )
+
+    main()
+
+    assert captured["env"] == "disabled"
+    # Restored to the pre-existing value, not popped.
+    assert os.environ.get("SCRAPE_CACHE_MODE") == "disabled"
+
+
+def test_cli_without_no_fetch_cache_does_not_set_env(monkeypatch):
+    import sys
+    from king_context.scraper.cli import main
+    from king_context.scraper.config import ScraperConfig
+
+    captured: dict = {}
+
+    async def fake_run_pipeline(args, config):
+        captured["env"] = os.environ.get("SCRAPE_CACHE_MODE")
+
+    monkeypatch.delenv("SCRAPE_CACHE_MODE", raising=False)
+    monkeypatch.setattr(
+        "king_context.scraper.cli.run_pipeline", fake_run_pipeline
+    )
+    monkeypatch.setattr(
+        "king_context.scraper.cli.load_config",
+        lambda **_: ScraperConfig(),
+    )
+    monkeypatch.setattr(
+        sys, "argv",
+        ["king-scrape", "https://docs.example.com"],
+    )
+
+    main()
+
+    assert captured["env"] is None
+    assert "SCRAPE_CACHE_MODE" not in os.environ
+
+
 # ---------------------------------------------------------------------------
 # Full pipeline
 # ---------------------------------------------------------------------------