From f5595f1e7833944358dc2d7946767d0628ee8b27 Mon Sep 17 00:00:00 2001 From: Zachary Roth <100426704+zacharyr0th@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:56:34 -0700 Subject: [PATCH] chore: apply docpull workspace snapshot --- .gitignore | 9 + README.md | 20 +- src/docpull/cli.py | 5 + src/docpull/conversion/special_cases.py | 42 +- src/docpull/discovery/contracts.py | 62 +- src/docpull/graph.py | 38 +- src/docpull/local_workflows.py | 49 +- src/docpull/pack_tools.py | 172 +- src/docpull/project.py | 938 +++++- src/docpull/security/download_policy.py | 67 +- src/docpull/source_scoring.py | 6 +- tests/pack_fixtures.py | 5 +- tests/test_discovery_contracts.py | 47 + tests/test_graph.py | 39 +- tests/test_pack_tools.py | 57 + tests/test_project.py | 209 ++ tests/test_real_site_regressions.py | 3 + tests/test_security_hardening.py | 65 + tests/test_special_cases.py | 24 + web/.gitignore | 38 - web/README.md | 31 - web/app/docs/page.tsx | 20 - web/app/globals.css | 282 -- web/app/layout.tsx | 100 - web/app/not-found.tsx | 34 - web/app/page.tsx | 35 - web/app/robots.ts | 15 - web/app/sitemap.ts | 22 - web/components/AsciiBackground.tsx | 122 - web/components/AsciiBackgroundLoader.tsx | 11 - web/components/CodeExamples.tsx | 154 - web/components/Comparison.tsx | 224 -- web/components/FAQ.tsx | 61 - web/components/Features.tsx | 275 -- web/components/Footer.tsx | 198 -- web/components/GithubIcon.tsx | 15 - web/components/Header.tsx | 84 - web/components/Hero.tsx | 87 - web/components/HowItWorks.tsx | 430 --- web/components/Install.tsx | 64 - web/components/ParallelPacks.tsx | 121 - web/components/Profiles.tsx | 50 - web/components/StructuredData.tsx | 86 - web/components/ThemeProvider.tsx | 17 - web/components/ThemeToggle.tsx | 54 - web/components/docs/DocsArticle.tsx | 391 --- web/components/docs/DocsCodeBlock.tsx | 85 - web/components/docs/DocsPrimitives.tsx | 144 - web/components/docs/DocsShell.tsx | 186 - web/components/docs/docs-data.ts | 75 - web/components/faq-content.tsx | 230 -- web/components/landing/BrandMark.tsx | 19 - web/components/landing/CodeTabs.tsx | 153 - web/components/landing/CommandCopy.tsx | 90 - web/components/landing/GlassPanel.tsx | 9 - web/components/landing/InfoCard.tsx | 49 - web/components/landing/LandingSection.tsx | 52 - web/components/landing/SectionHeader.tsx | 49 - web/components/landing/StatGrid.tsx | 33 - web/components/landing/TerminalPanel.tsx | 63 - web/components/landing/index.ts | 11 - web/eslint.config.mjs | 26 - web/lib/use-reduced-motion.ts | 18 - web/lib/utils.ts | 6 - web/next.config.ts | 9 - web/package-lock.json | 3744 --------------------- web/package.json | 40 - web/postcss.config.mjs | 7 - web/public/logo.svg | 4 - web/public/og-image.png | Bin 602053 -> 0 bytes web/public/parallel-wordmark.svg | 1 - web/tsconfig.json | 41 - 72 files changed, 1782 insertions(+), 8240 deletions(-) delete mode 100644 web/.gitignore delete mode 100644 web/README.md delete mode 100644 web/app/docs/page.tsx delete mode 100644 web/app/globals.css delete mode 100644 web/app/layout.tsx delete mode 100644 web/app/not-found.tsx delete mode 100644 web/app/page.tsx delete mode 100644 web/app/robots.ts delete mode 100644 web/app/sitemap.ts delete mode 100644 web/components/AsciiBackground.tsx delete mode 100644 web/components/AsciiBackgroundLoader.tsx delete mode 100644 web/components/CodeExamples.tsx delete mode 100644 web/components/Comparison.tsx delete mode 100644 web/components/FAQ.tsx delete mode 100644 web/components/Features.tsx delete mode 100644 web/components/Footer.tsx delete mode 100644 web/components/GithubIcon.tsx delete mode 100644 web/components/Header.tsx delete mode 100644 web/components/Hero.tsx delete mode 100644 web/components/HowItWorks.tsx delete mode 100644 web/components/Install.tsx delete mode 100644 web/components/ParallelPacks.tsx delete mode 100644 web/components/Profiles.tsx delete mode 100644 web/components/StructuredData.tsx delete mode 100644 web/components/ThemeProvider.tsx delete mode 100644 web/components/ThemeToggle.tsx delete mode 100644 web/components/docs/DocsArticle.tsx delete mode 100644 web/components/docs/DocsCodeBlock.tsx delete mode 100644 web/components/docs/DocsPrimitives.tsx delete mode 100644 web/components/docs/DocsShell.tsx delete mode 100644 web/components/docs/docs-data.ts delete mode 100644 web/components/faq-content.tsx delete mode 100644 web/components/landing/BrandMark.tsx delete mode 100644 web/components/landing/CodeTabs.tsx delete mode 100644 web/components/landing/CommandCopy.tsx delete mode 100644 web/components/landing/GlassPanel.tsx delete mode 100644 web/components/landing/InfoCard.tsx delete mode 100644 web/components/landing/LandingSection.tsx delete mode 100644 web/components/landing/SectionHeader.tsx delete mode 100644 web/components/landing/StatGrid.tsx delete mode 100644 web/components/landing/TerminalPanel.tsx delete mode 100644 web/components/landing/index.ts delete mode 100644 web/eslint.config.mjs delete mode 100644 web/lib/use-reduced-motion.ts delete mode 100644 web/lib/utils.ts delete mode 100644 web/next.config.ts delete mode 100644 web/package-lock.json delete mode 100644 web/package.json delete mode 100644 web/postcss.config.mjs delete mode 100644 web/public/logo.svg delete mode 100644 web/public/og-image.png delete mode 100644 web/public/parallel-wordmark.svg delete mode 100644 web/tsconfig.json diff --git a/.gitignore b/.gitignore index f1fc19e..7433bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -185,6 +185,15 @@ downloads/ /packs/ run.accounting.json +# Private planning / competitive strategy artifacts +.agents/ +context-todo.txt +docs/competitive-roadmap.md +docs/competitive-*.md +docs/*-competitive-*.md +docs/*-competitor-*.md +docs/*-roadmap.md + # Generated reports and analysis files TEST_REPORT.md CODE_QUALITY_REPORT.md diff --git a/README.md b/README.md index 67b6836..e55fa78 100644 --- a/README.md +++ b/README.md @@ -201,9 +201,23 @@ docpull watch https://docs.stripe.com --export cursor --alert changes `docpull diff` is hash-based and deterministic locally. Optional BYOK semantic summaries are advisory and skip cleanly when no model key is configured. -Use `docpull add URL --discover` or `docpull sync --update-discovery` to -refresh and persist discovered source URLs in `docpull.yaml`; sync then uses -that stored URL set for repeatable exact refreshes. +Use `docpull add URL --discover` to discover and persist source URLs once. +Normal `docpull sync` uses that stored URL set without rediscovering; use +`docpull sync --update-discovery` when you explicitly want to refresh it. + +For larger vendor-doc pulls, plan the crawl frontier before fetching: + +```bash +# Build a balanced dry-run frontier under .docpull/plans/ +docpull plan --profile balanced + +# Fetch exactly the selected URLs from the latest plan +docpull sync --plan latest +``` + +Profiles include `broad`, `balanced`, `api-docs`, and `rag-clean`. Plans record +selected URLs, rejected URLs with reasons, category counts, source balance, and +warnings such as generated-directory traps or localized duplicates. For authenticated sources, store only environment variable references in `docpull.yaml`; DocPull resolves values in memory at sync time and writes only diff --git a/src/docpull/cli.py b/src/docpull/cli.py index dc4c9a6..ce90ed5 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -193,6 +193,7 @@ def create_parser() -> argparse.ArgumentParser: Subcommands: init Create a persistent DocPull project add Add a source to a project + plan Plan a balanced project crawl frontier sync Sync configured project sources diff Diff project runs status Show project status @@ -1381,6 +1382,10 @@ def main(argv: list[str] | None = None) -> int: from .project import run_add_cli return run_add_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] == "plan": + from .project import run_plan_cli + + return run_plan_cli(raw_argv[1:]) if raw_argv and raw_argv[0] == "sync": from .project import run_sync_cli diff --git a/src/docpull/conversion/special_cases.py b/src/docpull/conversion/special_cases.py index 228f4a1..33cbb7e 100644 --- a/src/docpull/conversion/special_cases.py +++ b/src/docpull/conversion/special_cases.py @@ -19,6 +19,7 @@ from typing import Any, Protocol from urllib.parse import urlparse +import yaml from bs4 import BeautifulSoup, Tag logger = logging.getLogger(__name__) @@ -512,7 +513,7 @@ def _schema_properties( class OpenApiExtractor: - """Render OpenAPI / Swagger JSON specs directly to Markdown. + """Render OpenAPI / Swagger JSON or YAML specs directly to Markdown. Triggers only when the body parses as an OpenAPI document. Renders each operation with description, parameters (grouped by location), request body @@ -524,15 +525,10 @@ class OpenApiExtractor: def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None: text = _decode_html(html).lstrip() - if not text.startswith("{"): - return None - try: - data = json.loads(text) - except json.JSONDecodeError as err: - logger.debug("OpenAPI extractor skipped %s: JSON parse failed: %s", url, err) - return None - if not isinstance(data, dict): + data = self._parse_spec(text, url) + if data is None: return None + version = data.get("openapi") or data.get("swagger") if not isinstance(version, str): return None @@ -568,6 +564,34 @@ def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None: extra={"framework": "openapi", "openapi_version": version}, ) + @staticmethod + def _parse_spec(text: str, url: str) -> dict[str, Any] | None: + if not text: + return None + if text.startswith("{"): + try: + data = json.loads(text) + except json.JSONDecodeError as err: + logger.debug("OpenAPI extractor skipped %s: JSON parse failed: %s", url, err) + return None + return data if isinstance(data, dict) else None + if not OpenApiExtractor._looks_like_yaml_spec(text, url): + return None + try: + data = yaml.safe_load(text) + except yaml.YAMLError as err: + logger.debug("OpenAPI extractor skipped %s: YAML parse failed: %s", url, err) + return None + return data if isinstance(data, dict) else None + + @staticmethod + def _looks_like_yaml_spec(text: str, url: str) -> bool: + path = urlparse(url).path.lower() + if path.endswith((".yaml", ".yml")): + return True + head = text[:4000] + return re.search(r"(?m)^\s*(openapi|swagger)\s*:", head) is not None + def _render_operation( self, lines: list[str], diff --git a/src/docpull/discovery/contracts.py b/src/docpull/discovery/contracts.py index 1995cbc..bd9bfc3 100644 --- a/src/docpull/discovery/contracts.py +++ b/src/docpull/discovery/contracts.py @@ -503,7 +503,9 @@ def _append_site_scan_record( url = str(candidate.get("url") or "").strip() if not url: return - parsed = urlparse(url) + parsed = _safe_urlparse(url) + if parsed is None: + return if parsed.scheme not in {"http", "https"} or not parsed.netloc: return key = normalize_url(url) @@ -577,8 +579,10 @@ def _site_origin(url: str) -> str: def _same_origin(left: str, right: str) -> bool: - left_parsed = urlparse(left) - right_parsed = urlparse(right) + left_parsed = _safe_urlparse(left) + right_parsed = _safe_urlparse(right) + if left_parsed is None or right_parsed is None: + return False return ( left_parsed.scheme in {"http", "https"} and left_parsed.scheme == right_parsed.scheme @@ -590,7 +594,10 @@ def _dedupe_urls(urls: Iterable[str]) -> list[str]: output: list[str] = [] seen: set[str] = set() for url in urls: - key = normalize_url(url) + try: + key = normalize_url(url) + except ValueError: + continue if key in seen: continue seen.add(key) @@ -649,25 +656,31 @@ def _extract_text_links(text: str, base_url: str) -> list[tuple[str, str | None] title = match.group(1).strip() or None href = match.group(2).strip() resolved = _resolve_candidate_url(href, base_url) - if resolved and normalize_url(resolved) not in seen: - seen.add(normalize_url(resolved)) + key = _candidate_url_key(resolved) + if resolved and key and key not in seen: + seen.add(key) links.append((resolved, title)) for match in _URL_RE.finditer(text): - href = match.group(0).rstrip(".,;)]}") + href = _clean_text_link_href(match.group(0)) resolved = _resolve_candidate_url(href, base_url) - if resolved and normalize_url(resolved) not in seen: - seen.add(normalize_url(resolved)) + key = _candidate_url_key(resolved) + if resolved and key and key not in seen: + seen.add(key) links.append((resolved, None)) return links def _resolve_candidate_url(href: str, base_url: str) -> str | None: + href = _clean_text_link_href(href) if not href or href.startswith(("#", "mailto:", "tel:", "javascript:", "data:")): return None - resolved = urljoin(base_url, href) - parsed = urlparse(resolved) + try: + resolved = urljoin(base_url, href) + parsed = urlparse(resolved) + except ValueError: + return None if parsed.scheme not in {"http", "https"} or not parsed.netloc: return None clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path or '/'}" @@ -676,6 +689,22 @@ def _resolve_candidate_url(href: str, base_url: str) -> str | None: return clean +def _clean_text_link_href(href: str) -> str: + clean = href.strip() + if "](" in clean: + clean = clean.split("](", 1)[0] + return clean.rstrip(".,;)]}") + + +def _candidate_url_key(url: str | None) -> str | None: + if not url: + return None + try: + return normalize_url(url) + except ValueError: + return None + + async def _scan_feeds( start_url: str, *, @@ -1078,7 +1107,9 @@ async def _scan_github_docs_tree( def _github_repo(url: str) -> tuple[str, str] | None: - parsed = urlparse(url) + parsed = _safe_urlparse(url) + if parsed is None: + return None if parsed.hostname not in {"github.com", "www.github.com"}: return None parts = [part for part in parsed.path.split("/") if part] @@ -1110,6 +1141,13 @@ def _github_doc_file(path: str, *, root: bool) -> bool: return True +def _safe_urlparse(url: str) -> Any | None: + try: + return urlparse(url) + except ValueError: + return None + + def _xml_local_name(tag: str) -> str: return tag.rsplit("}", 1)[-1] if "}" in tag else tag diff --git a/src/docpull/graph.py b/src/docpull/graph.py index a270c67..0611a59 100644 --- a/src/docpull/graph.py +++ b/src/docpull/graph.py @@ -8,6 +8,7 @@ import json import re from collections import Counter +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -22,6 +23,8 @@ DEFAULT_ENTITY_LIMIT = 500 DEFAULT_QUERY_LIMIT = 10 DEFAULT_NEIGHBOR_LIMIT = 20 +MAX_GRAPH_ENTITIES_PER_CHUNK = 24 +MAX_GRAPH_ENTITIES_PER_SENTENCE = 8 _WORD_RE = re.compile(r"[a-z0-9][a-z0-9-]{1,}", re.IGNORECASE) _SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+") @@ -515,6 +518,7 @@ def _build_nodes_and_edges( ) entity_nodes = _entity_nodes(entities_payload) + entity_rank = {str(node.get("id")): index for index, node in enumerate(entity_nodes)} nodes.extend(entity_nodes) for record in pack.documents: chunk_id = record.chunk_id or _stable_id("chunk", record.document_id, record.content_hash) @@ -538,10 +542,15 @@ def _build_nodes_and_edges( for record in pack.documents: chunk_id = record.chunk_id or _stable_id("chunk", record.document_id, record.content_hash) - entity_ids = sorted(set(chunk_entity_ids.get(chunk_id, []))) + entity_ids = _limited_entity_ids( + chunk_entity_ids.get(chunk_id, []), + entity_rank=entity_rank, + limit=MAX_GRAPH_ENTITIES_PER_CHUNK, + ) if len(entity_ids) < 2: continue citation_id = pack.citation_by_url.get(record.url) + chunk_excerpt = _excerpt(record.content) for left, right in itertools.combinations(entity_ids, 2): edges.append( _edge( @@ -551,6 +560,7 @@ def _build_nodes_and_edges( "co_occurs_in_chunk", record=record, citation_id=citation_id, + excerpt=chunk_excerpt, ) ) edges.extend( @@ -558,6 +568,7 @@ def _build_nodes_and_edges( record, entity_nodes=entity_nodes, entity_ids=entity_ids, + entity_rank=entity_rank, citation_id=citation_id, ) ) @@ -633,15 +644,20 @@ def _entity_relation_edges( *, entity_nodes: list[dict[str, Any]], entity_ids: list[str], + entity_rank: dict[str, int], citation_id: str | None, ) -> list[dict[str, Any]]: entity_nodes_by_id = {str(node["id"]): node for node in entity_nodes if node.get("id") in entity_ids} edges: list[dict[str, Any]] = [] for sentence in _sentences(record.content): - sentence_entity_ids = sorted( - entity_id - for entity_id, entity_node in entity_nodes_by_id.items() - if _entity_in_content(entity_node, sentence) + sentence_entity_ids = _limited_entity_ids( + ( + entity_id + for entity_id, entity_node in entity_nodes_by_id.items() + if _entity_in_content(entity_node, sentence) + ), + entity_rank=entity_rank, + limit=MAX_GRAPH_ENTITIES_PER_SENTENCE, ) if len(sentence_entity_ids) < 2: continue @@ -661,6 +677,18 @@ def _entity_relation_edges( return edges +def _limited_entity_ids( + entity_ids: Iterable[str], + *, + entity_rank: dict[str, int], + limit: int, +) -> list[str]: + unique_ids = {entity_id for entity_id in entity_ids if entity_id} + return sorted(unique_ids, key=lambda entity_id: (entity_rank.get(entity_id, 1_000_000), entity_id))[ + :limit + ] + + def _pack_fingerprint(pack: LocalPack) -> dict[str, Any]: manifest = pack.manifest if isinstance(pack.manifest, dict) else {} records = [ diff --git a/src/docpull/local_workflows.py b/src/docpull/local_workflows.py index 012a53c..52c2abf 100644 --- a/src/docpull/local_workflows.py +++ b/src/docpull/local_workflows.py @@ -24,10 +24,12 @@ _diff_markdown, _domain, _expected_domains, + _pack_token_counts, _read_ndjson, _read_pack_metadata, _safe_int, _search_markdown, + _url_category_counts, _write_json, build_citation_map, build_research_brief, @@ -356,7 +358,7 @@ def audit_pack( records = _read_ndjson(pack_dir / "documents.ndjson") score_payload = score_pack(pack_dir, required_domains=required_domains) citation_payload = build_citation_map(pack_dir, required_domains=required_domains) - dimensions = _audit_dimensions(records, score_payload, citation_payload) + dimensions = _audit_dimensions(pack_dir, records, score_payload, citation_payload) weighted_score = _weighted_audit_score(dimensions, score_payload["score"]) payload = { "schema_version": AUDIT_SCHEMA_VERSION, @@ -512,6 +514,7 @@ def _sources_markdown(sources: list[dict[str, Any]]) -> str: def _audit_dimensions( + pack_dir: Path, records: list[dict[str, Any]], score_payload: dict[str, Any], citation_payload: dict[str, Any], @@ -523,9 +526,11 @@ def _audit_dimensions( ] duplicate_count = len(content_hashes) - len(set(content_hashes)) duplicate_rate = duplicate_count / len(content_hashes) if content_hashes else 0.0 - token_counts = [ - _safe_int(record.get("token_count")) for record in records if _safe_int(record.get("token_count")) - ] + token_counts, token_source = _pack_token_counts(pack_dir, records) + category_counts = _url_category_counts(sorted(set(urls))) + localized_count = category_counts.get("localized", 0) + largest_domain_share = max(domains.values()) / len(urls) if urls and domains else 0.0 + balance_score = int(max(0, 100 - max(0.0, largest_domain_share - 0.5) * 120)) if urls else 0 citation_urls = { str(source.get("url")) for source in citation_payload.get("sources", []) if source.get("url") } @@ -563,8 +568,27 @@ def _audit_dimensions( "count": len(token_counts), "min": min(token_counts) if token_counts else None, "max": max(token_counts) if token_counts else None, + "source": token_source, }, - "summary": "Token counts are evaluated when chunk metadata exists.", + "summary": "Token counts are evaluated from document or chunk metadata.", + }, + "source_balance": { + "score": balance_score, + "value": { + "largest_domain_share": round(largest_domain_share, 4), + "domains": dict(domains.most_common()), + }, + "summary": "Flags packs dominated by one source domain.", + }, + "localized_content": { + "score": max(0, 100 - localized_count * 10), + "value": localized_count, + "summary": f"{localized_count} localized URL(s)", + }, + "content_categories": { + "score": 100, + "value": dict(category_counts.most_common()), + "summary": "URL-category mix for source planning review.", }, "required_domain_coverage": { "score": 100 if not score_payload.get("issues") else max(0, score_payload["score"]), @@ -599,7 +623,7 @@ def _freshness_score(records: list[dict[str, Any]]) -> int: def _chunk_size_score(token_counts: list[int]) -> int: if not token_counts: - return 85 + return 70 too_small = sum(1 for value in token_counts if value < 100) too_large = sum(1 for value in token_counts if value > 8000) penalty = (too_small + too_large) * 8 @@ -608,12 +632,15 @@ def _chunk_size_score(token_counts: list[int]) -> int: def _weighted_audit_score(dimensions: dict[str, Any], base_score: int) -> int: weights = { - "source_diversity": 0.12, - "freshness": 0.16, - "duplicate_rate": 0.18, - "citation_coverage": 0.20, + "source_diversity": 0.10, + "freshness": 0.14, + "duplicate_rate": 0.16, + "citation_coverage": 0.18, "chunk_size_distribution": 0.12, - "required_domain_coverage": 0.12, + "required_domain_coverage": 0.10, + "source_balance": 0.06, + "localized_content": 0.03, + "content_categories": 0.01, } dimension_score = sum( _safe_int(dimensions[name].get("score")) * weight for name, weight in weights.items() diff --git a/src/docpull/pack_tools.py b/src/docpull/pack_tools.py index c3b24e6..b798528 100644 --- a/src/docpull/pack_tools.py +++ b/src/docpull/pack_tools.py @@ -8,7 +8,7 @@ from collections import Counter from pathlib import Path from typing import Any -from urllib.parse import urlparse +from urllib.parse import unquote, urlparse from rich.console import Console from rich.markup import escape @@ -401,7 +401,7 @@ def score_pack(pack_dir: Path, *, required_domains: list[str] | None = None) -> domains = Counter(_domain(url) for url in urls if _domain(url)) content_hashes = [str(record.get("content_hash", "")) for record in records if record.get("content_hash")] duplicate_chunks = len(content_hashes) - len(set(content_hashes)) - token_counts = [int(record.get("token_count", 0) or 0) for record in records] + token_counts, token_source = _pack_token_counts(pack_dir, records) empty_records = [record for record in records if not str(record.get("content", "")).strip()] if record_count == 0: @@ -441,6 +441,14 @@ def score_pack(pack_dir: Path, *, required_domains: list[str] | None = None) -> warnings.append( _issue("duplicate_chunks", f"{duplicate_chunks} records share duplicate content hashes.") ) + if record_count and not token_counts: + score -= 5 + warnings.append( + _issue( + "missing_token_counts", + "Pack has records but no token_count metadata on documents or chunks.", + ) + ) if not (pack_dir / "corpus.manifest.json").exists(): score -= 15 issues.append(_issue("missing_manifest", "corpus.manifest.json is missing.")) @@ -509,6 +517,19 @@ def score_pack(pack_dir: Path, *, required_domains: list[str] | None = None) -> ) ) + category_counts = _url_category_counts(unique_urls) + localized_count = category_counts.get("localized", 0) + largest_domain_share = 0.0 + if urls and domains: + largest_domain_share = max(domains.values()) / len(urls) + if len(domains) > 1 and largest_domain_share >= 0.85: + warnings.append( + _issue( + "source_imbalance", + "One source domain accounts for at least 85% of pack records.", + ) + ) + score = max(0, min(100, score)) return { "schema_version": SCORE_SCHEMA_VERSION, @@ -523,6 +544,10 @@ def score_pack(pack_dir: Path, *, required_domains: list[str] | None = None) -> "domain_count": len(domains), "duplicate_chunk_count": duplicate_chunks, "total_tokens": sum(token_counts), + "token_count_source": token_source, + "largest_domain_share": round(largest_domain_share, 4), + "localized_url_count": localized_count, + "category_counts": dict(category_counts.most_common()), }, "domains": dict(domains.most_common()), "expected_domains": expected, @@ -548,6 +573,86 @@ def score_pack_sources(pack_dir: Path, *, required_domains: list[str] | None = N } +def _pack_token_counts(pack_dir: Path, records: list[dict[str, Any]]) -> tuple[list[int], str]: + document_counts = [_safe_int(record.get("token_count")) for record in records] + document_counts = [count for count in document_counts if count > 0] + if document_counts: + return document_counts, "documents" + + chunk_path = pack_dir / "chunks.jsonl" + if not chunk_path.exists(): + chunk_path = pack_dir / "chunks.ndjson" + if chunk_path.exists(): + chunk_counts = [_safe_int(record.get("token_count")) for record in _read_ndjson(chunk_path)] + chunk_counts = [count for count in chunk_counts if count > 0] + if chunk_counts: + return chunk_counts, chunk_path.name + return [], "missing" + + +def _url_category_counts(urls: list[str]) -> Counter[str]: + counts: Counter[str] = Counter() + for url in urls: + counts[_pack_url_category(url)] += 1 + return counts + + +def _pack_url_category(url: str) -> str: + parsed = urlparse(url) + path = parsed.path.lower() + segments = [segment for segment in path.split("/") if segment] + filename = segments[-1] if segments else "" + if any( + segment + in { + "es", + "fr", + "de", + "it", + "ja", + "ko", + "pt", + "pt-br", + "zh", + "zh-cn", + "zh-tw", + "ru", + "pl", + "nl", + "tr", + "id", + "vi", + "ar", + } + for segment in segments + ): + return "localized" + if filename in {"llms.txt", "llms-full.txt"}: + return "llms" + if filename.endswith(("openapi.json", "openapi.yaml", "openapi.yml", "swagger.json")): + return "openapi" + if filename.endswith(".pdf"): + return "pdf" + if any(segment in {"api", "reference", "api-reference"} for segment in segments): + return "api_reference" + if any( + segment in {"docs", "documentation", "guide", "guides", "quickstart", "sdk", "sdks"} + for segment in segments + ): + return "docs" + if any(segment in {"changelog", "changes", "release-notes", "releases"} for segment in segments): + return "changelog" + if any(segment in {"blog", "news", "press", "articles"} for segment in segments): + return "blog" + if any(segment in {"glossary", "learn", "resources"} for segment in segments): + return "glossary" + if any(segment in {"legal", "terms", "privacy"} for segment in segments): + return "legal" + if any(segment in {"pricing", "plans"} for segment in segments): + return "pricing" + return "marketing" + + def build_citation_map( pack_dir: Path, *, @@ -611,6 +716,7 @@ def build_research_brief( sources, citation_by_url, expected = _citation_sources(pack_dir, pack, records, required_domains) brief_objective = objective or str(pack.get("objective") or "Review local DocPull context pack") entities = _extract_entities(records, citation_by_url, limit=max(1, entity_limit)) if entity_limit else [] + token_counts, token_source = _pack_token_counts(pack_dir, records) return { "schema_version": BRIEF_SCHEMA_VERSION, "generated_at": utc_now_iso(), @@ -621,7 +727,8 @@ def build_research_brief( "source_count": len(sources), "record_count": len(records), "entity_count": len(entities), - "total_tokens": sum(_safe_int(record.get("token_count")) for record in records), + "total_tokens": sum(token_counts), + "token_count_source": token_source, }, "load_plan": sources[: min(len(sources), 12)], "key_excerpts": _select_key_excerpts( @@ -1764,11 +1871,68 @@ def _off_domain_urls(urls: list[str], expected_domains: list[str]) -> list[str]: off_domain: list[str] = [] for url in urls: domain = _domain(url) - if not any(domain == expected or domain.endswith(f".{expected}") for expected in expected_domains): + if _domain_matches_expected(domain, expected_domains): + continue + if _is_machine_readable_doc_url(url) and _url_mentions_expected_domain(url, expected_domains): + continue + if domain: off_domain.append(url) return off_domain +def _domain_matches_expected(domain: str, expected_domains: list[str]) -> bool: + if not domain: + return False + domain_family = _domain_family(domain) + for expected in expected_domains: + expected = expected.lower().removeprefix("www.").rstrip(".") + if domain == expected or domain.endswith(f".{expected}"): + return True + expected_family = _domain_family(expected) + if domain_family and expected_family and domain_family == expected_family: + return True + return False + + +def _domain_family(domain: str) -> str: + labels = [label for label in domain.lower().removeprefix("www.").split(".") if label] + if len(labels) < 2: + return ".".join(labels) + if len(labels) >= 3 and len(labels[-1]) == 2 and labels[-2] in { + "ac", + "co", + "com", + "edu", + "gov", + "net", + "org", + }: + return ".".join(labels[-3:]) + return ".".join(labels[-2:]) + + +def _is_machine_readable_doc_url(url: str) -> bool: + path = urlparse(url).path.lower() + filename = Path(path).name + if filename in {"llms.txt", "llms-full.txt"}: + return True + return any(token in filename for token in ("openapi", "swagger")) and filename.endswith( + (".json", ".yaml", ".yml") + ) + + +def _url_mentions_expected_domain(url: str, expected_domains: list[str]) -> bool: + normalized_url = unquote(url).lower() + for expected in expected_domains: + expected = expected.lower().removeprefix("www.").rstrip(".") + expected_family = _domain_family(expected) + if expected and expected in normalized_url: + return True + if expected_family and expected_family in normalized_url: + return True + return False + + def _issue( code: str, message: str, diff --git a/src/docpull/project.py b/src/docpull/project.py index 6686985..f343aaf 100644 --- a/src/docpull/project.py +++ b/src/docpull/project.py @@ -20,11 +20,12 @@ import time import urllib.error import urllib.request +from collections import Counter from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Any, Literal -from urllib.parse import urlparse +from urllib.parse import unquote, urlparse import yaml from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator @@ -34,7 +35,16 @@ from .accounting import RunAccounting from .conversion.chunking import TokenCounter, chunk_markdown from .core.fetcher import Fetcher +from .discovery.contracts import ( + CANDIDATE_SOURCES_FILENAME, + SELECTED_SOURCES_FILENAME, + SELECTED_URLS_FILENAME, + CandidateSourceRecord, + records_from_site_scan, +) from .exports import export_pack +from .http.client import AsyncHttpClient +from .http.rate_limiter import PerHostRateLimiter from .models.config import ( AuthConfig, AuthType, @@ -52,6 +62,8 @@ build_citation_map, diff_packs, ) +from .security.url_validator import UrlValidator +from .source_scoring import score_source from .time_utils import utc_now, utc_now_iso PROJECT_SCHEMA_VERSION = 1 @@ -72,10 +84,12 @@ SemanticMode = Literal["auto", "off", "on"] ContextTarget = Literal["cursor", "claude", "codex", "openai", "llamaindex", "langchain"] SourceAuthType = Literal["bearer_env", "basic_env", "cookie_env", "header_env"] +ProjectPlanProfile = Literal["broad", "balanced", "api-docs", "rag-clean"] SOURCE_TYPES: tuple[str, ...] = ("auto", "html", "pdf", "markdown", "openapi", "github", "sitemap") CONTEXT_TARGETS: tuple[str, ...] = ("cursor", "claude", "codex", "openai", "llamaindex", "langchain") OUTPUT_FORMATS: tuple[str, ...] = ("markdown", "ndjson", "sqlite", "context-pack") +PROJECT_PLAN_PROFILES: tuple[str, ...] = ("broad", "balanced", "api-docs", "rag-clean") SemanticClient = Callable[[str], str] @@ -122,6 +136,7 @@ class ProjectSource(BaseModel): url: str type: SourceType = "auto" discover: bool = False + refresh_discovery_on_sync: bool = False discovered_urls: list[str] = Field(default_factory=list) discovered_at: str | None = None auth: ProjectSourceAuth | None = None @@ -175,7 +190,10 @@ class ProjectCrawlConfig(BaseModel): rate_limit: float = Field(0.5, ge=0) include_paths: list[str] = Field(default_factory=list) exclude_paths: list[str] = Field(default_factory=list) + adaptive_rate_limit: bool = False streaming_discovery: bool = True + plan_profile: ProjectPlanProfile = "balanced" + max_pages_per_source: int | None = Field(None, ge=1) model_config = {"extra": "forbid"} @@ -270,6 +288,7 @@ class ProjectPaths: runs: Path cache: Path manifests: Path + plans: Path exports: Path evals: Path releases: Path @@ -278,6 +297,15 @@ class ProjectPaths: remote_config: Path +@dataclass(frozen=True) +class ResolvedProjectPlan: + plan_id: str + plan_dir: Path + profile: ProjectPlanProfile + urls_by_source: dict[str, list[str]] + payload: dict[str, Any] + + def run_init_cli(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="docpull init", description="Create a DocPull project") parser.add_argument("name", nargs="?", help="Project name") @@ -339,12 +367,17 @@ def run_sync_cli(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="docpull sync", description="Sync configured project sources") parser.add_argument("--source", help="Only sync one source name") parser.add_argument("--run-id", help="Explicit run ID") + parser.add_argument( + "--plan", + dest="plan_ref", + help="Use a project plan ID, plan directory, or 'latest' as the sync frontier", + ) parser.add_argument("--dry-run", action="store_true") parser.add_argument("--json", action="store_true", dest="json_output") parser.add_argument( "--update-discovery", action="store_true", - help="Record that source discovery should be refreshed before future syncs", + help="Refresh and persist source discovery before this sync", ) args = parser.parse_args(argv) console = Console() @@ -352,6 +385,7 @@ def run_sync_cli(argv: list[str] | None = None) -> int: payload = sync_project( source_name=args.source, run_id=args.run_id, + plan=args.plan_ref, dry_run=args.dry_run, update_discovery=args.update_discovery, ) @@ -370,6 +404,50 @@ def run_sync_cli(argv: list[str] | None = None) -> int: return 0 +def run_plan_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="docpull plan", + description="Plan a balanced project crawl frontier without fetching page content", + ) + parser.add_argument("--source", help="Only plan one source name") + parser.add_argument("--plan-id", help="Explicit plan ID") + parser.add_argument("--profile", choices=PROJECT_PLAN_PROFILES, help="Corpus profile") + parser.add_argument("--max-pages", type=int, help="Maximum selected URLs across all sources") + parser.add_argument("--max-pages-per-source", type=int, help="Maximum selected URLs per source") + parser.add_argument( + "--no-site-scan", + action="store_true", + help="Skip llms.txt, sitemap, feed, OpenAPI, and GitHub hint scans", + ) + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + console = Console() + try: + payload = plan_project( + source_name=args.source, + plan_id=args.plan_id, + profile=args.profile, + max_pages=args.max_pages, + max_pages_per_source=args.max_pages_per_source, + scan_site_hints=not args.no_site_scan, + ) + except ProjectError as err: + console.print("[red]Project plan error:[/red] " + escape(str(err))) + return 1 + if args.json_output: + console.print_json(data=payload) + else: + summary = payload["summary"] + console.print( + "[green]Project plan:[/green] " + f"{payload['plan_id']} profile={payload['profile']} " + f"selected={summary['selected_count']} rejected={summary['rejected_count']} " + f"warnings={len(payload['warnings'])}" + ) + console.print(f"Plan: {payload['artifacts']['plan']}") + return 0 + + def run_diff_cli(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="docpull diff", description="Diff project runs") parser.add_argument("--from", dest="from_run_id", help="Older run ID") @@ -668,6 +746,7 @@ def sync_project( *, source_name: str | None = None, run_id: str | None = None, + plan: str | Path | None = None, dry_run: bool = False, update_discovery: bool = False, root: Path | None = None, @@ -677,7 +756,7 @@ def sync_project( if not config.sources: raise ProjectError("Project has no sources. Add one with `docpull add URL`.") selected = _selected_sources(config, source_name) - discovery_refresh_names = [source.name for source in selected if source.discover] + discovery_refresh_names = [source.name for source in selected if source.refresh_discovery_on_sync] if update_discovery or discovery_refresh_names: refresh_names = [source.name for source in selected] if update_discovery else discovery_refresh_names config = _refresh_project_discovery(project_root, config, source_names=refresh_names) @@ -686,6 +765,7 @@ def sync_project( _validate_auth_ready(selected) paths = ensure_project_dirs(project_root) ensure_project_index(project_root, config) + resolved_plan = _resolve_project_plan(project_root, plan, selected) if plan else None current_run_id = _safe_run_id(run_id or _new_run_id()) run_dir = paths.runs / current_run_id if run_dir.exists(): @@ -721,6 +801,7 @@ def sync_project( source_health=source_health, fetch_stats=fetch_stats, update_discovery=update_discovery, + plan=resolved_plan, ) for source in selected: @@ -733,6 +814,11 @@ def sync_project( project_root=project_root, config=config, source=source, + plan_urls=( + resolved_plan.urls_by_source.get(source.name) + if resolved_plan is not None + else None + ), output_dir=source_output_dir, ) ) @@ -794,9 +880,144 @@ def sync_project( source_health=source_health, fetch_stats=fetch_stats, update_discovery=update_discovery, + plan=resolved_plan, ) +def plan_project( + *, + source_name: str | None = None, + plan_id: str | None = None, + profile: str | None = None, + max_pages: int | None = None, + max_pages_per_source: int | None = None, + scan_site_hints: bool = True, + root: Path | None = None, +) -> dict[str, Any]: + """Write a project crawl-frontier plan without fetching page content.""" + project_root = find_project_root(root or Path.cwd()) + config = load_project_config(project_root) + if not config.sources: + raise ProjectError("Project has no sources. Add one with `docpull add URL`.") + selected_sources = _selected_sources(config, source_name) + plan_profile = _project_plan_profile(profile or config.crawl.plan_profile) + if max_pages is not None and max_pages < 1: + raise ProjectError("max_pages must be at least 1") + if max_pages_per_source is not None and max_pages_per_source < 1: + raise ProjectError("max_pages_per_source must be at least 1") + + paths = ensure_project_dirs(project_root) + current_plan_id = _safe_run_id(plan_id or _new_plan_id()) + plan_dir = paths.plans / current_plan_id + if plan_dir.exists(): + raise ProjectError(f"Plan already exists: {current_plan_id}") + plan_dir.mkdir(parents=True) + + effective_max_pages = max_pages if max_pages is not None else config.crawl.max_pages + per_source_limit = max_pages_per_source or config.crawl.max_pages_per_source + if per_source_limit is None: + per_source_limit = _default_plan_source_limit( + profile=plan_profile, + max_pages=effective_max_pages, + source_count=len(selected_sources), + ) + + generated_at = utc_now_iso() + all_candidates: list[dict[str, Any]] = [] + selected_candidates: list[dict[str, Any]] = [] + rejected_candidates: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] + + for source in selected_sources: + records = _stored_project_candidate_records(source, generated_at=generated_at) + if scan_site_hints: + try: + records.extend( + asyncio.run( + _scan_project_source_hints( + config=config, + source=source, + max_results_per_source=per_source_limit, + ) + ) + ) + except Exception as err: # noqa: BLE001 + warnings.append( + { + "source_name": source.name, + "code": "site_scan_failed", + "message": str(err), + } + ) + + planned = _plan_source_candidates( + source=source, + records=records, + profile=plan_profile, + limit=per_source_limit, + ) + all_candidates.extend(planned["candidates"]) + selected_candidates.extend(planned["selected"]) + rejected_candidates.extend(planned["rejected"]) + warnings.extend(planned["warnings"]) + + if effective_max_pages is not None and len(selected_candidates) > effective_max_pages: + selected_candidates, overflow = _balance_project_plan_selection( + selected_candidates, + limit=effective_max_pages, + source_order=[source.name for source in selected_sources], + ) + for item in overflow: + rejected_candidates.append( + { + **item, + "reject_reason": "global_max_pages", + } + ) + warnings.append( + { + "code": "global_max_pages_applied", + "message": f"Selected frontier was capped at {effective_max_pages} URLs.", + } + ) + + candidate_records = [item["record"] for item in all_candidates] + selected_records = [item["record"] for item in selected_candidates] + rejected_records = [_rejected_plan_record(item) for item in rejected_candidates] + _write_jsonl( + plan_dir / CANDIDATE_SOURCES_FILENAME, + [record.model_dump(mode="json", exclude_none=True) for record in candidate_records], + ) + _write_jsonl( + plan_dir / SELECTED_SOURCES_FILENAME, + [record.model_dump(mode="json", exclude_none=True) for record in selected_records], + ) + (plan_dir / SELECTED_URLS_FILENAME).write_text( + "".join(f"{record.url}\n" for record in selected_records), + encoding="utf-8", + ) + _write_jsonl(plan_dir / "rejected_sources.ndjson", rejected_records) + + payload = _project_plan_payload( + config=config, + plan_id=current_plan_id, + plan_dir=plan_dir, + profile=plan_profile, + generated_at=generated_at, + selected=selected_candidates, + rejected=rejected_candidates, + all_candidates=all_candidates, + warnings=warnings, + max_pages=effective_max_pages, + max_pages_per_source=per_source_limit, + scan_site_hints=scan_site_hints, + ) + _write_json(plan_dir / "frontier.plan.json", payload) + (plan_dir / "PLAN.md").write_text(_project_plan_markdown(payload), encoding="utf-8") + (paths.plans / "latest-plan").write_text(current_plan_id + "\n", encoding="utf-8") + return payload + + def diff_project( *, from_run_id: str | None = None, @@ -1288,6 +1509,7 @@ async def _sync_source( project_root: Path, config: ProjectConfig, source: ProjectSource, + plan_urls: list[str] | None = None, output_dir: Path, ) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) @@ -1295,19 +1517,35 @@ async def _sync_source( errors: list[dict[str, Any]] = [] skips: list[dict[str, Any]] = [] robots_blocked = 0 + quarantined_prefixes: set[str] = set() async with Fetcher(fetch_config) as fetcher: - if source.discovered_urls: - urls = _unique_urls([source.url, *source.discovered_urls]) + if plan_urls is not None or source.discovered_urls: + urls = _unique_urls(plan_urls if plan_urls is not None else [source.url, *source.discovered_urls]) if config.crawl.max_pages is not None: urls = urls[: config.crawl.max_pages] for url in urls: + prefix = _rate_limit_prefix(url) + if prefix in quarantined_prefixes: + skips.append( + { + "source_name": source.name, + "url": url, + "reason": "rate_limited_prefix_quarantine", + "rate_limit_prefix": prefix, + "type": "fetch_skipped", + } + ) + continue ctx = await fetcher.fetch_one(url, save=True) if ctx.error: + if _looks_like_rate_limit_error(ctx.error): + quarantined_prefixes.add(prefix) errors.append( { "source_name": source.name, "url": url, "error": ctx.error, + "rate_limit_prefix": prefix if prefix in quarantined_prefixes else None, "type": "fetch_failed", } ) @@ -1408,6 +1646,656 @@ async def _discover_source_urls( return _unique_urls(urls) +async def _scan_project_source_hints( + *, + config: ProjectConfig, + source: ProjectSource, + max_results_per_source: int, +) -> list[CandidateSourceRecord]: + validator = UrlValidator() + validation = validator.validate(source.url) + if not validation.is_valid: + raise ProjectError(f"Source scan URL rejected: {validation.rejection_reason}") + expected_domains = _source_expected_domains(source) + async with AsyncHttpClient( + rate_limiter=PerHostRateLimiter( + default_delay=max(config.crawl.rate_limit, 0.2), + default_concurrent=min(config.crawl.per_host_concurrent, 2), + ), + url_validator=validator, + default_timeout=20.0, + max_retries=1, + ) as client: + records = await records_from_site_scan( + source.url, + client=client, + sources=None, + query=f"{config.name} {source.name} docs", + expected_domains=expected_domains, + max_results_per_source=max_results_per_source, + timeout_seconds=20.0, + ) + return [ + record.model_copy( + update={ + "metadata": { + **record.metadata, + "docpull_project_source": source.name, + "docpull_project_source_url": source.url, + "candidate_origin": "site_scan", + } + } + ) + for record in records + ] + + +def _stored_project_candidate_records( + source: ProjectSource, + *, + generated_at: str, +) -> list[CandidateSourceRecord]: + records: list[CandidateSourceRecord] = [] + for index, url in enumerate(_unique_urls([source.url, *source.discovered_urls]), start=1): + local_score = score_source(url=url, expected_domains=_source_expected_domains(source)) + records.append( + CandidateSourceRecord( + generated_at=generated_at, + url=url, + source=f"project-source:{source.name}", + provider="local", + score=float(local_score["score"]), + rank=index, + query=f"project:{source.name}", + discovered_at=source.discovered_at or generated_at, + raw_ref="docpull.yaml", + metadata={ + "docpull_project_source": source.name, + "docpull_project_source_url": source.url, + "candidate_origin": "stored_discovery" if url != source.url else "source_url", + "local_score": local_score["score"], + "score_grade": local_score["grade"], + "score_reasons": local_score["reasons"], + }, + ) + ) + return records + + +def _plan_source_candidates( + *, + source: ProjectSource, + records: list[CandidateSourceRecord], + profile: ProjectPlanProfile, + limit: int, +) -> dict[str, Any]: + seen: set[str] = set() + candidates: list[dict[str, Any]] = [] + rejected: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] + category_counts: Counter[str] = Counter() + + for record in records: + key = _normalized_plan_url(record.url) + if key in seen: + continue + seen.add(key) + classification = _classify_project_plan_url(record.url) + category_counts[str(classification["category"])] += 1 + score = _score_project_plan_candidate(record, classification, profile, source=source) + reject_reason = _project_plan_reject_reason( + record.url, + classification, + profile, + source=source, + ) + planned_record = record.model_copy( + update={ + "score": float(score), + "metadata": { + **record.metadata, + "docpull_project_source": source.name, + "plan_profile": profile, + "plan_score": score, + "plan_category": classification["category"], + "plan_tags": classification["tags"], + "language_hint": classification["language"], + "rate_limit_prefix": classification["rate_limit_prefix"], + "reject_reason": reject_reason, + }, + } + ) + item = { + "source_name": source.name, + "url": record.url, + "record": planned_record, + "score": score, + "category": classification["category"], + "tags": classification["tags"], + "language": classification["language"], + "rate_limit_prefix": classification["rate_limit_prefix"], + "reject_reason": reject_reason, + "rank": record.rank or 1_000_000, + } + candidates.append(item) + if reject_reason: + rejected.append(item) + + eligible = [item for item in candidates if not item["reject_reason"]] + eligible.sort(key=lambda item: (-int(item["score"]), int(item["rank"]), str(item["url"]))) + selected = eligible[:limit] + for item in eligible[limit:]: + rejected.append({**item, "reject_reason": "max_pages_per_source"}) + + if not selected: + warnings.append( + { + "source_name": source.name, + "code": "no_selected_urls", + "message": f"No URLs selected for source {source.name}.", + } + ) + if category_counts.get("generated_directory"): + warnings.append( + { + "source_name": source.name, + "code": "generated_directory_detected", + "message": f"{category_counts['generated_directory']} generated-directory URL(s) detected.", + } + ) + if category_counts.get("localized"): + warnings.append( + { + "source_name": source.name, + "code": "localized_urls_detected", + "message": f"{category_counts['localized']} localized URL(s) detected.", + } + ) + return { + "candidates": candidates, + "selected": selected, + "rejected": rejected, + "warnings": warnings, + } + + +def _classify_project_plan_url(url: str) -> dict[str, Any]: + parsed = urlparse(url) + path = parsed.path.lower() + segments = [segment for segment in path.split("/") if segment] + filename = segments[-1] if segments else "" + tags: set[str] = set() + category = "marketing" + language: str | None = None + + for segment in segments: + clean = segment.lower() + if clean in { + "es", + "fr", + "de", + "it", + "ja", + "jp", + "ko", + "pt", + "pt-br", + "zh", + "zh-cn", + "zh-tw", + "ru", + "pl", + "nl", + "tr", + "id", + "vi", + "ar", + }: + language = clean + tags.add("localized") + break + + if filename in {"llms.txt", "llms-full.txt"}: + category = "llms" + tags.add("llms") + elif filename.endswith((".pdf",)): + category = "pdf" + tags.add("pdf") + elif _looks_like_openapi_spec_filename(filename): + category = "openapi" + tags.add("openapi") + elif any(segment in {"changelog", "changes", "release-notes", "releases"} for segment in segments): + category = "changelog" + tags.add("changelog") + elif any(segment in {"api", "reference", "api-reference"} for segment in segments): + category = "api_reference" + tags.add("api_reference") + elif any( + segment in {"docs", "documentation", "guide", "guides", "quickstart", "sdk", "sdks", "integrations"} + for segment in segments + ): + category = "docs" + tags.add("docs") + elif any(segment in {"pricing", "plans"} for segment in segments): + category = "pricing" + tags.add("pricing") + elif any(segment in {"blog", "news", "press", "articles"} for segment in segments): + category = "blog" + tags.add("blog") + elif any(segment in {"glossary", "learn", "resources"} for segment in segments): + category = "glossary" + tags.add("glossary") + elif any(segment in {"legal", "terms", "privacy", "security", "trust"} for segment in segments): + category = "legal" + tags.add("legal") + + if tags.intersection({"localized"}) and category not in {"llms", "openapi"}: + category = "localized" + if _looks_like_generated_directory(segments): + category = "generated_directory" + tags.add("generated_directory") + tags.add("crawler_trap") + + return { + "category": category, + "tags": sorted(tags), + "language": language, + "rate_limit_prefix": _rate_limit_prefix(url), + } + + +def _looks_like_generated_directory(segments: list[str]) -> bool: + if len(segments) >= 2 and segments[0] == "websets" and segments[1] == "directory": + return True + return "directory" in segments and len(segments) >= 4 + + +def _looks_like_openapi_spec_filename(filename: str) -> bool: + return any(token in filename for token in ("openapi", "swagger")) and filename.endswith( + (".json", ".yaml", ".yml") + ) + + +def _score_project_plan_candidate( + record: CandidateSourceRecord, + classification: dict[str, Any], + profile: ProjectPlanProfile, + *, + source: ProjectSource, +) -> int: + base = score_source( + url=record.url, + title=record.title or "", + expected_domains=_source_expected_domains(source), + ) + score = int(base["score"]) + category = str(classification["category"]) + tags = set(classification["tags"]) + category_weights = { + "llms": 35, + "openapi": 32, + "api_reference": 26, + "docs": 20, + "changelog": 16, + "pricing": 8, + "blog": -14, + "glossary": -16, + "marketing": -10, + "legal": -24, + "localized": -18, + "pdf": -35, + "generated_directory": -55, + } + score += category_weights.get(category, 0) + if profile in {"api-docs", "rag-clean"}: + if category in {"llms", "openapi", "api_reference", "docs", "changelog"}: + score += 12 + if category in {"blog", "glossary", "marketing", "legal", "localized"}: + score -= 35 + elif profile == "balanced": + if category in {"llms", "openapi", "api_reference", "docs", "changelog", "pricing"}: + score += 8 + elif profile == "broad": + if "crawler_trap" not in tags: + score += 5 + return max(0, min(100, score)) + + +def _project_plan_reject_reason( + url: str, + classification: dict[str, Any], + profile: ProjectPlanProfile, + *, + source: ProjectSource, +) -> str | None: + category = str(classification["category"]) + tags = set(classification["tags"]) + if "pdf" in tags: + return "pdf_requires_opt_in" + if "generated_directory" in tags and profile != "broad": + return "generated_directory" + if "crawler_trap" in tags and profile != "broad": + return "crawler_trap" + if profile != "broad" and not _is_source_related_url(url, source): + if category in {"llms", "openapi"} and _url_mentions_source_domain(url, source): + return None + return "off_domain" + if category == "legal" and profile != "broad": + return "low_signal_category" + if category in {"blog", "glossary", "marketing"} and profile in {"api-docs", "rag-clean"}: + return "low_signal_category" + if "localized" in tags and profile in {"api-docs", "rag-clean"}: + return "localized_duplicate" + return None + + +def _balance_project_plan_selection( + selected: list[dict[str, Any]], + *, + limit: int, + source_order: list[str], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + by_source: dict[str, list[dict[str, Any]]] = {name: [] for name in source_order} + for item in selected: + by_source.setdefault(str(item["source_name"]), []).append(item) + for items in by_source.values(): + items.sort(key=lambda item: (-int(item["score"]), int(item["rank"]), str(item["url"]))) + + kept: list[dict[str, Any]] = [] + while len(kept) < limit and any(by_source.values()): + for source_name in source_order: + items = by_source.get(source_name) or [] + if not items: + continue + kept.append(items.pop(0)) + if len(kept) >= limit: + break + overflow = [item for items in by_source.values() for item in items] + return kept, overflow + + +def _project_plan_payload( + *, + config: ProjectConfig, + plan_id: str, + plan_dir: Path, + profile: ProjectPlanProfile, + generated_at: str, + selected: list[dict[str, Any]], + rejected: list[dict[str, Any]], + all_candidates: list[dict[str, Any]], + warnings: list[dict[str, Any]], + max_pages: int | None, + max_pages_per_source: int, + scan_site_hints: bool, +) -> dict[str, Any]: + selected_sources = Counter(str(item["source_name"]) for item in selected) + selected_categories = Counter(str(item["category"]) for item in selected) + candidate_categories = Counter(str(item["category"]) for item in all_candidates) + rejected_reasons = Counter(str(item.get("reject_reason") or "unknown") for item in rejected) + selected_payload = [_plan_item_payload(item) for item in selected] + rejected_payload = [_plan_item_payload(item, include_reject_reason=True) for item in rejected] + return { + "schema_version": PROJECT_SCHEMA_VERSION, + "generated_at": generated_at, + "project": config.name, + "plan_id": plan_id, + "plan_dir": str(plan_dir), + "profile": profile, + "workflow": "project-frontier-plan", + "options": { + "max_pages": max_pages, + "max_pages_per_source": max_pages_per_source, + "scan_site_hints": scan_site_hints, + }, + "summary": { + "candidate_count": len(all_candidates), + "selected_count": len(selected), + "rejected_count": len(rejected), + "source_count": len(selected_sources), + "category_counts": dict(selected_categories.most_common()), + "candidate_category_counts": dict(candidate_categories.most_common()), + "rejected_reason_counts": dict(rejected_reasons.most_common()), + "source_counts": dict(selected_sources.most_common()), + }, + "selected": selected_payload, + "rejected": rejected_payload[:1000], + "warnings": warnings, + "artifacts": { + "candidate_sources": str(plan_dir / CANDIDATE_SOURCES_FILENAME), + "selected_sources": str(plan_dir / SELECTED_SOURCES_FILENAME), + "selected_urls": str(plan_dir / SELECTED_URLS_FILENAME), + "rejected_sources": str(plan_dir / "rejected_sources.ndjson"), + "plan": str(plan_dir / "frontier.plan.json"), + "markdown": str(plan_dir / "PLAN.md"), + }, + } + + +def _project_plan_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + f"# Project Plan: {payload['plan_id']}", + "", + f"- Project: {payload['project']}", + f"- Profile: {payload['profile']}", + f"- Candidates: {summary['candidate_count']}", + f"- Selected: {summary['selected_count']}", + f"- Rejected: {summary['rejected_count']}", + "", + "## Sources", + "", + ] + for source_name, count in summary["source_counts"].items(): + lines.append(f"- {source_name}: {count}") + lines.extend(["", "## Categories", ""]) + for category, count in summary["category_counts"].items(): + lines.append(f"- {category}: {count}") + if payload["warnings"]: + lines.extend(["", "## Warnings", ""]) + for warning in payload["warnings"]: + label = warning.get("source_name") or "project" + lines.append(f"- {label}: {warning.get('code')} - {warning.get('message')}") + lines.extend(["", "## Selected URLs", ""]) + for item in payload["selected"][:100]: + lines.append(f"- [{item['source_name']}] {item['url']} ({item['category']}, score={item['score']})") + if len(payload["selected"]) > 100: + lines.append(f"- ... {len(payload['selected']) - 100} more") + return "\n".join(lines).rstrip() + "\n" + + +def _plan_item_payload(item: dict[str, Any], *, include_reject_reason: bool = False) -> dict[str, Any]: + payload = { + "source_name": item["source_name"], + "url": item["url"], + "score": item["score"], + "category": item["category"], + "tags": item["tags"], + "language": item["language"], + "rate_limit_prefix": item["rate_limit_prefix"], + } + if include_reject_reason: + payload["reject_reason"] = item.get("reject_reason") + return payload + + +def _rejected_plan_record(item: dict[str, Any]) -> dict[str, Any]: + candidate = item["record"] + if not isinstance(candidate, CandidateSourceRecord): + raise ProjectError("Invalid project plan candidate record") + record: dict[str, Any] = dict(candidate.model_dump(mode="json", exclude_none=True)) + metadata = record.get("metadata") + if not isinstance(metadata, dict): + metadata = {} + record["metadata"] = {**metadata, "reject_reason": item.get("reject_reason")} + return record + + +def _resolve_project_plan( + project_root: Path, + plan_ref: str | Path, + selected_sources: list[ProjectSource], +) -> ResolvedProjectPlan: + paths = ensure_project_dirs(project_root) + plan_dir = _resolve_project_plan_dir(paths, plan_ref) + payload = _read_json(plan_dir / "frontier.plan.json") + plan_id = str(payload.get("plan_id") or plan_dir.name) + profile = _project_plan_profile(str(payload.get("profile") or "balanced")) + source_names = {source.name for source in selected_sources} + urls_by_source: dict[str, list[str]] = {source.name: [] for source in selected_sources} + selected = payload.get("selected") + if not isinstance(selected, list): + raise ProjectError(f"Invalid project plan: {plan_dir}") + for item in selected: + if not isinstance(item, dict): + continue + source_name = str(item.get("source_name") or "") + url = str(item.get("url") or "") + if source_name not in source_names or not url: + continue + urls_by_source[source_name].append(url) + urls_by_source = {key: _unique_urls(value) for key, value in urls_by_source.items()} + if not any(urls_by_source.values()): + raise ProjectError("Project plan has no selected URLs for the requested source(s).") + return ResolvedProjectPlan( + plan_id=plan_id, + plan_dir=plan_dir, + profile=profile, + urls_by_source=urls_by_source, + payload=payload, + ) + + +def _resolve_project_plan_dir(paths: ProjectPaths, plan_ref: str | Path) -> Path: + ref = str(plan_ref) + if ref == "latest": + latest_path = paths.plans / "latest-plan" + if not latest_path.exists(): + raise ProjectError("No latest project plan exists. Run `docpull plan` first.") + ref = latest_path.read_text(encoding="utf-8").strip() + candidate = Path(ref) + plan_dir = candidate if candidate.is_absolute() or candidate.exists() else paths.plans / ref + if plan_dir.is_file(): + plan_dir = plan_dir.parent + if not (plan_dir / "frontier.plan.json").exists(): + raise ProjectError(f"Project plan does not exist: {plan_ref}") + return plan_dir.resolve() + + +def _run_plan_payload(plan: ResolvedProjectPlan | None) -> dict[str, Any] | None: + if plan is None: + return None + selected_count = sum(len(urls) for urls in plan.urls_by_source.values()) + return { + "plan_id": plan.plan_id, + "plan_dir": str(plan.plan_dir), + "profile": plan.profile, + "selected_count": selected_count, + "source_counts": {source: len(urls) for source, urls in plan.urls_by_source.items()}, + } + + +def _project_plan_profile(value: str) -> ProjectPlanProfile: + if value not in PROJECT_PLAN_PROFILES: + raise ProjectError(f"Unsupported plan profile: {value}") + return value # type: ignore[return-value] + + +def _new_plan_id() -> str: + return "plan_" + utc_now().strftime("%Y%m%dT%H%M%S%fZ") + + +def _default_plan_source_limit( + *, + profile: ProjectPlanProfile, + max_pages: int | None, + source_count: int, +) -> int: + profile_cap = { + "broad": 500, + "balanced": 150, + "api-docs": 200, + "rag-clean": 120, + }[profile] + if max_pages is None: + return profile_cap + per_source = max(1, (max_pages + max(source_count, 1) - 1) // max(source_count, 1)) + return max(1, min(profile_cap, per_source)) + + +def _source_host(source: ProjectSource) -> str: + return (urlparse(source.url).hostname or "").lower().rstrip(".") + + +def _source_expected_domains(source: ProjectSource) -> list[str]: + host = _source_host(source) + if not host: + return [] + domains = [host] + family = _domain_family(host) + if family and family != host: + domains.append(family) + return domains + + +def _domain_family(domain: str) -> str: + labels = [label for label in domain.lower().removeprefix("www.").split(".") if label] + if len(labels) < 2: + return ".".join(labels) + if len(labels) >= 3 and len(labels[-1]) == 2 and labels[-2] in { + "ac", + "co", + "com", + "edu", + "gov", + "net", + "org", + }: + return ".".join(labels[-3:]) + return ".".join(labels[-2:]) + + +def _is_source_related_url(url: str, source: ProjectSource) -> bool: + hostname = (urlparse(url).hostname or "").lower().removeprefix("www.").rstrip(".") + if not hostname: + return False + return any( + hostname == expected or hostname.endswith(f".{expected}") + for expected in _source_expected_domains(source) + ) + + +def _url_mentions_source_domain(url: str, source: ProjectSource) -> bool: + normalized_url = unquote(url).lower() + return any(expected and expected in normalized_url for expected in _source_expected_domains(source)) + + +def _normalized_plan_url(url: str) -> str: + parsed = urlparse(url) + path = parsed.path + for suffix in (".md", ".mdx"): + if path.lower().endswith(suffix): + path = path[: -len(suffix)] + break + return parsed._replace(path=path, fragment="").geturl().rstrip("/") + + +def _rate_limit_prefix(url: str) -> str: + parsed = urlparse(url) + segments = [segment for segment in parsed.path.split("/") if segment] + if len(segments) >= 2 and segments[0] == "websets" and segments[1] == "directory": + prefix_segments = segments[:2] + else: + prefix_segments = segments[:1] + path = "/" + "/".join(prefix_segments) if prefix_segments else "" + return f"{parsed.scheme}://{parsed.netloc}{path}" + + +def _looks_like_rate_limit_error(error: Any) -> bool: + text = str(error).lower() + return "429" in text or "too many requests" in text or "rate limit" in text + + def _unique_urls(values: list[str]) -> list[str]: seen: set[str] = set() output: list[str] = [] @@ -1437,6 +2325,7 @@ def _fetch_config( rate_limit=config.crawl.rate_limit, include_paths=config.crawl.include_paths, exclude_paths=config.crawl.exclude_paths, + adaptive_rate_limit=config.crawl.adaptive_rate_limit, streaming_discovery=config.crawl.streaming_discovery, ), output=OutputConfig( @@ -1469,6 +2358,7 @@ def _finalize_project_run( source_health: list[dict[str, Any]], fetch_stats: dict[str, int], update_discovery: bool, + plan: ResolvedProjectPlan | None = None, ) -> dict[str, Any]: paths = project_paths(project_root) finished_at = utc_now_iso() @@ -1478,6 +2368,7 @@ def _finalize_project_run( _write_jsonl(run_dir / "documents.ndjson", records) _write_jsonl(run_dir / "chunks.jsonl", chunks) _write_jsonl(run_dir / "errors.jsonl", errors) + _write_jsonl(run_dir / "skips.jsonl", skips) _write_json( run_dir / "source-health.json", {"schema_version": PROJECT_SCHEMA_VERSION, "sources": source_health}, @@ -1492,6 +2383,7 @@ def _finalize_project_run( started_at=started_at, finished_at=finished_at, update_discovery=update_discovery, + plan=plan, ) _write_json(run_dir / "manifest.json", manifest) _write_json(run_dir / "corpus.manifest.json", _corpus_manifest(manifest, records, source_entries)) @@ -1501,6 +2393,20 @@ def _finalize_project_run( _write_json(run_dir / "accounting.json", accounting) _write_json(paths.manifests / f"{run_id}.json", manifest) + artifacts: dict[str, str] = { + "run": "run.json", + "documents_jsonl": "documents.jsonl", + "chunks_jsonl": "chunks.jsonl", + "manifest": "manifest.json", + "errors": "errors.jsonl", + "skips": "skips.jsonl", + "accounting": "accounting.json", + "source_health": "source-health.json", + "documents_ndjson": "documents.ndjson", + "corpus_manifest": "corpus.manifest.json", + "sources": "sources.md", + "pack": "local.pack.json", + } run_payload = { "schema_version": PROJECT_SCHEMA_VERSION, "project": config.name, @@ -1517,20 +2423,12 @@ def _finalize_project_run( "robots_blocked": sum(_safe_int(item.get("robots_blocked_count")) for item in source_health), "paid_cloud_routes_used": 0, }, - "artifacts": { - "run": "run.json", - "documents_jsonl": "documents.jsonl", - "chunks_jsonl": "chunks.jsonl", - "manifest": "manifest.json", - "errors": "errors.jsonl", - "accounting": "accounting.json", - "source_health": "source-health.json", - "documents_ndjson": "documents.ndjson", - "corpus_manifest": "corpus.manifest.json", - "sources": "sources.md", - "pack": "local.pack.json", - }, + "plan": _run_plan_payload(plan), + "artifacts": artifacts, } + if plan is not None: + _write_json(run_dir / "plan.json", _run_plan_payload(plan)) + artifacts["plan"] = "plan.json" _write_json(run_dir / "run.json", run_payload) if status != "dry_run": paths.latest_run.write_text(run_id + "\n", encoding="utf-8") @@ -1548,6 +2446,7 @@ def project_paths(root: Path) -> ProjectPaths: runs=state / "runs", cache=state / "cache", manifests=state / "manifests", + plans=state / "plans", exports=state / "exports", evals=state / "evals", releases=state / "releases", @@ -1564,6 +2463,7 @@ def ensure_project_dirs(root: Path) -> ProjectPaths: paths.runs, paths.cache, paths.manifests, + paths.plans, paths.exports, paths.evals, paths.releases, @@ -2423,6 +3323,7 @@ def _run_manifest( started_at: str, finished_at: str, update_discovery: bool, + plan: ResolvedProjectPlan | None = None, ) -> dict[str, Any]: return { "schema_version": PROJECT_SCHEMA_VERSION, @@ -2433,6 +3334,7 @@ def _run_manifest( "started_at": started_at, "finished_at": finished_at, "update_discovery": update_discovery, + "plan": _run_plan_payload(plan), "sources": [_source_public_payload(source) for source in config.sources], "document_count": len(records), "chunk_count": len(chunks), diff --git a/src/docpull/security/download_policy.py b/src/docpull/security/download_policy.py index daccabc..6a000f4 100644 --- a/src/docpull/security/download_policy.py +++ b/src/docpull/security/download_policy.py @@ -23,9 +23,13 @@ class UnsafeDownloadError(ValueError): "application/rss+xml", "application/json", "application/ld+json", + "application/yaml", + "application/x-yaml", "text/plain", "text/markdown", "text/x-markdown", + "text/yaml", + "text/x-yaml", } ) @@ -197,17 +201,46 @@ class _MagicSignature: _ALLOWED_TEXT_CONTROL_BYTES = {9, 10, 12, 13} -def content_type_base(content_type: str | None) -> str: - """Return the lowercase media type without parameters.""" +_GENERIC_BINARY_CONTENT_TYPE = "application/octet-stream" + + +def content_type_media_types(content_type: str | None) -> tuple[str, ...]: + """Return lowercase media types without parameters. + + Some servers expose duplicate Content-Type headers as a comma-joined value + such as ``application/octet-stream, application/json``. Content-Type itself + is not a list-valued header, but accepting a later readable media type keeps + machine-readable specs fetchable while body sniffing still blocks binaries. + """ if not content_type: - return "" - return content_type.lower().split(";", 1)[0].strip() + return () + media_types: list[str] = [] + seen: set[str] = set() + for value in content_type.split(","): + media_type = value.lower().split(";", 1)[0].strip() + if media_type and media_type not in seen: + media_types.append(media_type) + seen.add(media_type) + return tuple(media_types) + + +def content_type_base(content_type: str | None) -> str: + """Return the first lowercase media type without parameters.""" + media_types = content_type_media_types(content_type) + return media_types[0] if media_types else "" def is_allowed_document_content_type(content_type: str | None) -> bool: """Whether a Content-Type is compatible with docpull's document pipeline.""" - base_type = content_type_base(content_type) - return not base_type or base_type in ALLOWED_DOCUMENT_CONTENT_TYPES + media_types = content_type_media_types(content_type) + if not media_types: + return True + if any( + media_type in _DANGEROUS_CONTENT_TYPES and media_type != _GENERIC_BINARY_CONTENT_TYPE + for media_type in media_types + ): + return False + return any(media_type in ALLOWED_DOCUMENT_CONTENT_TYPES for media_type in media_types) def _header_get(headers: dict[str, str], name: str) -> str | None: @@ -313,10 +346,24 @@ def validate_response_headers( f"uses disallowed extension '{extension}'." ) - base_type = content_type_base(content_type or _header_get(headers, "Content-Type")) - if base_type in _DANGEROUS_CONTENT_TYPES: - raise UnsafeDownloadError(f"Disallowed content type '{base_type}' for {url}.") - if base_type and base_type not in ALLOWED_DOCUMENT_CONTENT_TYPES: + media_types = content_type_media_types(content_type or _header_get(headers, "Content-Type")) + dangerous_type = next( + ( + media_type + for media_type in media_types + if media_type in _DANGEROUS_CONTENT_TYPES + and media_type != _GENERIC_BINARY_CONTENT_TYPE + ), + None, + ) + if dangerous_type is not None: + raise UnsafeDownloadError(f"Disallowed content type '{dangerous_type}' for {url}.") + if media_types and any(media_type in ALLOWED_DOCUMENT_CONTENT_TYPES for media_type in media_types): + return + if _GENERIC_BINARY_CONTENT_TYPE in media_types: + raise UnsafeDownloadError(f"Disallowed content type '{_GENERIC_BINARY_CONTENT_TYPE}' for {url}.") + base_type = media_types[0] if media_types else "" + if base_type: raise UnsafeDownloadError( f"Unsupported content type '{base_type}' for {url}; " "docpull only fetches readable web/text responses." diff --git a/src/docpull/source_scoring.py b/src/docpull/source_scoring.py index 75352fb..a64f540 100644 --- a/src/docpull/source_scoring.py +++ b/src/docpull/source_scoring.py @@ -49,7 +49,11 @@ def score_source( score += 10 reasons.append("docs_path") - if path.endswith(("llms.txt", "openapi.json", "swagger.json", "openapi.yaml", "openapi.yml")): + filename = path.rsplit("/", 1)[-1] + if filename in {"llms.txt", "llms-full.txt"} or ( + any(token in filename for token in ("openapi", "swagger")) + and filename.endswith((".json", ".yaml", ".yml")) + ): score += 15 reasons.append("machine_readable_spec") diff --git a/tests/pack_fixtures.py b/tests/pack_fixtures.py index 5cf779a..00ef87f 100644 --- a/tests/pack_fixtures.py +++ b/tests/pack_fixtures.py @@ -20,7 +20,7 @@ def write_context_pack( sources_dir = pack_dir / "sources" sources_dir.mkdir(exist_ok=True) domains = include_domains or ["docs.parallel.ai"] - pack_records = records or [ + pack_records = [dict(record) for record in records] if records is not None else [ { "document_id": "doc_1", "url": "https://docs.parallel.ai/api-reference/search/search", @@ -28,8 +28,11 @@ def write_context_pack( "content": "Parallel Search API returns cited JSON results for live agent search.", "content_hash": "hash_1", "source_type": "parallel_extract", + "token_count": 250, } ] + for record in pack_records: + record.setdefault("token_count", 250) sources = [] for index, record in enumerate(pack_records, start=1): diff --git a/tests/test_discovery_contracts.py b/tests/test_discovery_contracts.py index b71ad56..7ca6f79 100644 --- a/tests/test_discovery_contracts.py +++ b/tests/test_discovery_contracts.py @@ -375,6 +375,53 @@ def test_site_scan_collects_local_open_discovery_candidates() -> None: } +def test_site_scan_skips_malformed_text_links_without_failing() -> None: + routes = { + "https://docs.example.com/llms.txt": ( + "Bad absolute URL: https://[not-an-ipv6/docs\n[Guide](/guide)\n", + "text/plain", + ), + } + + records = asyncio.run( + records_from_site_scan( + "https://docs.example.com", + client=_FakeHttpClient(routes), + sources=["llms"], + expected_domains=["docs.example.com"], + ) + ) + + assert [record.url for record in records] == [ + "https://docs.example.com/llms.txt", + "https://docs.example.com/guide", + ] + + +def test_site_scan_trims_markdown_artifacts_from_bare_text_links() -> None: + routes = { + "https://docs.example.com/llms.txt": ( + "Duplicated markdown artifact: " + "https://docs.example.com/guides/enrich](https://docs.example.com/guides/enrich)\n", + "text/plain", + ), + } + + records = asyncio.run( + records_from_site_scan( + "https://docs.example.com", + client=_FakeHttpClient(routes), + sources=["llms"], + expected_domains=["docs.example.com"], + ) + ) + + assert [record.url for record in records] == [ + "https://docs.example.com/llms.txt", + "https://docs.example.com/guides/enrich", + ] + + def test_site_scan_collects_github_docs_tree_candidates() -> None: routes = { "https://api.github.com/repos/owner/project/contents?ref=HEAD": ( diff --git a/tests/test_graph.py b/tests/test_graph.py index 728986c..42dfa65 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -6,7 +6,16 @@ from pathlib import Path from docpull.cli import main -from docpull.graph import build_graph, graph_neighbors, graph_status, load_graph, query_graph, refresh_graph +from docpull.graph import ( + MAX_GRAPH_ENTITIES_PER_CHUNK, + MAX_GRAPH_ENTITIES_PER_SENTENCE, + build_graph, + graph_neighbors, + graph_status, + load_graph, + query_graph, + refresh_graph, +) from tests.pack_fixtures import write_context_pack @@ -94,6 +103,34 @@ def test_graph_query_and_neighbors_return_cited_evidence(tmp_path: Path) -> None assert any(item["type"] == "chunk" for item in neighbors["neighbors"]) +def test_build_graph_caps_dense_entity_pair_edges(tmp_path: Path) -> None: + pack_dir = tmp_path / "pack" + emails = " ".join(f"user{i}@example.com" for i in range(40)) + records = [ + { + "document_id": "doc_dense", + "chunk_id": "chunk_dense_1", + "url": "https://docs.parallel.ai/api-reference/search/search", + "title": "Dense entity page", + "content": f"Search API supports these contacts: {emails}.", + "content_hash": "hash_dense_1", + "source_type": "parallel_extract", + "chunk_index": 0, + "chunk_heading": "Dense", + "token_count": 100, + } + ] + write_context_pack(pack_dir, records=records) + + build_graph(pack_dir) + edges = _graph_records(pack_dir, "graph.edges.ndjson") + + cooccurs = [edge for edge in edges if edge["type"] == "entity_cooccurs"] + relations = [edge for edge in edges if edge["type"] == "entity_relation"] + assert len(cooccurs) <= MAX_GRAPH_ENTITIES_PER_CHUNK * (MAX_GRAPH_ENTITIES_PER_CHUNK - 1) // 2 + assert len(relations) <= MAX_GRAPH_ENTITIES_PER_SENTENCE * (MAX_GRAPH_ENTITIES_PER_SENTENCE - 1) // 2 + + def test_graph_status_detects_stale_pack_and_refresh_writes_diff(tmp_path: Path) -> None: pack_dir = tmp_path / "pack" records = write_context_pack(pack_dir, records=_records()) diff --git a/tests/test_pack_tools.py b/tests/test_pack_tools.py index 90e8488..6b2542b 100644 --- a/tests/test_pack_tools.py +++ b/tests/test_pack_tools.py @@ -86,6 +86,7 @@ def _record(url: str, content_hash: str, content: str = "content") -> dict[str, "content": content, "content_hash": content_hash, "source_type": "parallel_extract", + "token_count": 250, } @@ -107,6 +108,36 @@ def test_pack_score_flags_off_domain_sources(tmp_path: Path) -> None: assert result["expected_domains"] == ["parallel.ai"] +def test_pack_score_allows_related_api_subdomain(tmp_path: Path) -> None: + pack = tmp_path / "pack" + _write_pack( + pack, + [_record("https://api.parallel.ai/account/service/openapi.json", "aaa")], + include_domains=["docs.parallel.ai"], + ) + + result = score_pack(pack) + + assert result["score"] == 100 + assert result["issues"] == [] + + +def test_pack_score_allows_external_machine_readable_spec_with_expected_domain( + tmp_path: Path, +) -> None: + pack = tmp_path / "pack" + _write_pack( + pack, + [_record("https://app.stainless.com/api/spec/documented/context.dev/openapi.documented.yml", "aaa")], + include_domains=["docs.context.dev"], + ) + + result = score_pack(pack) + + assert result["score"] == 100 + assert result["issues"] == [] + + def test_pack_score_cli_writes_score_file(tmp_path: Path) -> None: pack = tmp_path / "pack" _write_pack( @@ -160,6 +191,32 @@ def test_pack_score_understands_search_pack_metadata(tmp_path: Path) -> None: assert result["score"] == 100 assert result["expected_domains"] == ["docs.parallel.ai"] assert result["warnings"] == [] + assert result["summary"]["total_tokens"] == 250 + assert result["summary"]["token_count_source"] == "documents" + + +def test_pack_score_warns_on_missing_token_counts_and_reads_chunks(tmp_path: Path) -> None: + pack = tmp_path / "pack" + record = _record("https://docs.parallel.ai/api-reference/search/search", "aaa") + record.pop("token_count") + _write_pack(pack, [record], include_domains=["docs.parallel.ai"]) + + missing = score_pack(pack) + + assert missing["summary"]["token_count_source"] == "missing" + assert {warning["code"] for warning in missing["warnings"]} == {"missing_token_counts"} + assert missing["score"] == 95 + + (pack / "chunks.jsonl").write_text( + json.dumps({"chunk_id": "chunk_1", "document_id": "doc_aaa", "token_count": 321}) + "\n", + encoding="utf-8", + ) + + with_chunks = score_pack(pack) + + assert with_chunks["summary"]["token_count_source"] == "chunks.jsonl" + assert with_chunks["summary"]["total_tokens"] == 321 + assert with_chunks["warnings"] == [] def test_pack_sources_ranks_expected_docs_sources(tmp_path: Path) -> None: diff --git a/tests/test_project.py b/tests/test_project.py index a544921..c389b61 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -22,6 +22,7 @@ export_context_pack, generate_eval_set, init_project, + plan_project, project_history, project_status, release_context_pack, @@ -320,6 +321,9 @@ def test_discovery_refresh_updates_config_and_syncs_exact_urls( monkeypatch: pytest.MonkeyPatch, ) -> None: class FakeFetcher: + discover_calls = 0 + fetched_urls: list[str] = [] + def __init__(self, config: Any) -> None: self.config = config self.stats = SimpleNamespace(pages_fetched=0, pages_failed=0, pages_skipped=0) @@ -331,6 +335,7 @@ async def __aexit__(self, *_exc: object) -> None: return None async def discover(self) -> list[str]: + FakeFetcher.discover_calls += 1 return [ "https://docs.example.com/api/a", "https://docs.example.com/api/b", @@ -338,6 +343,7 @@ async def discover(self) -> list[str]: ] async def fetch_one(self, url: str, *, save: bool = True) -> SimpleNamespace: + FakeFetcher.fetched_urls.append(url) self.stats.pages_fetched += 1 if save: output_dir = self.config.output.directory @@ -362,6 +368,7 @@ async def fetch_one(self, url: str, *, save: bool = True) -> SimpleNamespace: added = add_source("https://docs.example.com/api", name="docs", discover=True, root=tmp_path) assert added["source"]["discover"] is True + assert added["source"]["refresh_discovery_on_sync"] is False assert added["source"]["discovered_urls"] == [ "https://docs.example.com/api/a", "https://docs.example.com/api/b", @@ -377,9 +384,211 @@ async def fetch_one(self, url: str, *, save: bool = True) -> SimpleNamespace: "https://docs.example.com/api/a", "https://docs.example.com/api/b", ] + assert FakeFetcher.discover_calls == 1 health = json.loads((run_dir / "source-health.json").read_text(encoding="utf-8")) assert health["sources"][0]["discovered_url_count"] == 2 + sync_project(root=tmp_path, run_id="run_refresh", update_discovery=True) + assert FakeFetcher.discover_calls == 2 + + +def test_project_plan_filters_profiles_and_sync_uses_latest_plan( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + init_project(name="demo", source="https://exa.ai/docs", root=tmp_path) + config = project_module.load_project_config(tmp_path) + source = config.sources[0].model_copy( + update={ + "discover": True, + "discovered_urls": [ + "https://exa.ai/docs/api-reference/search", + "https://api.exa.ai/openapi.json", + "https://app.stainless.com/api/spec/documented/exa.ai/openapi.documented.yml", + "https://exa.ai/openapi.json", + "https://exa.ai/llms.txt", + "https://developer.monday.com/api-reference/reference/webhooks", + "https://exa.ai/blog/product", + "https://exa.ai/es/docs/api-reference/search", + "https://exa.ai/websets/directory/company/foo", + "https://exa.ai/legal/terms.pdf", + ], + "discovered_at": "2026-06-29T00:00:00+00:00", + } + ) + project_module.save_project_config(tmp_path, config.model_copy(update={"sources": [source]})) + + payload = plan_project( + root=tmp_path, + plan_id="plan_api", + profile="api-docs", + scan_site_hints=False, + max_pages_per_source=10, + ) + + selected_urls = {item["url"] for item in payload["selected"]} + rejected_urls = {item["url"] for item in payload["rejected"]} + plan_dir = tmp_path / ".docpull" / "plans" / "plan_api" + + assert "https://exa.ai/docs/api-reference/search" in selected_urls + assert "https://api.exa.ai/openapi.json" in selected_urls + assert "https://app.stainless.com/api/spec/documented/exa.ai/openapi.documented.yml" in selected_urls + assert "https://exa.ai/openapi.json" in selected_urls + assert "https://exa.ai/llms.txt" in selected_urls + assert "https://developer.monday.com/api-reference/reference/webhooks" in rejected_urls + assert "https://exa.ai/websets/directory/company/foo" in rejected_urls + assert "https://exa.ai/es/docs/api-reference/search" in rejected_urls + assert "https://exa.ai/blog/product" in rejected_urls + assert (plan_dir / "frontier.plan.json").exists() + assert (plan_dir / "selected_urls.txt").exists() + assert (plan_dir / "rejected_sources.ndjson").exists() + assert (tmp_path / ".docpull" / "plans" / "latest-plan").read_text(encoding="utf-8").strip() == "plan_api" + + class FakeFetcher: + fetched_urls: list[str] = [] + + def __init__(self, config: Any) -> None: + self.config = config + self.stats = SimpleNamespace(pages_fetched=0, pages_failed=0, pages_skipped=0) + + async def __aenter__(self) -> FakeFetcher: + return self + + async def __aexit__(self, *_exc: object) -> None: + return None + + async def fetch_one(self, url: str, *, save: bool = True) -> SimpleNamespace: + FakeFetcher.fetched_urls.append(url) + self.stats.pages_fetched += 1 + if save: + output_dir = self.config.output.directory + output_dir.mkdir(parents=True, exist_ok=True) + content = f"Fetched {url}" + record = { + "document_id": f"doc_{self.stats.pages_fetched}", + "url": url, + "title": url, + "content": content, + "content_hash": _hash(content), + "source_type": "test", + "metadata": {}, + "extraction": {}, + } + with (output_dir / "documents.ndjson").open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record) + "\n") + return SimpleNamespace(error=None, should_skip=False, skip_code=None, skip_reason=None) + + monkeypatch.setattr(project_module, "Fetcher", FakeFetcher) + sync_payload = sync_project(root=tmp_path, run_id="run_plan", plan="latest") + run_dir = tmp_path / ".docpull" / "runs" / "run_plan" + + assert sync_payload["plan"]["plan_id"] == "plan_api" + assert set(FakeFetcher.fetched_urls) == selected_urls + assert "https://exa.ai/websets/directory/company/foo" not in FakeFetcher.fetched_urls + manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) + assert manifest["plan"]["plan_id"] == "plan_api" + + +def test_project_plan_dedupes_markdown_mirror_urls( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + init_project(name="demo", source="https://docs.example.com", root=tmp_path) + config = project_module.load_project_config(tmp_path) + source = config.sources[0].model_copy( + update={ + "discovered_urls": ["https://docs.example.com/documentation/about"], + "discovered_at": "2026-06-29T00:00:00+00:00", + } + ) + project_module.save_project_config(tmp_path, config.model_copy(update={"sources": [source]})) + + async def fake_scan(**_kwargs: Any) -> list[Any]: + return [ + project_module.CandidateSourceRecord( + url="https://docs.example.com/documentation/about.md", + source="local-site-scan:llms", + provider="local", + rank=1, + metadata={"candidate_origin": "site_scan"}, + ) + ] + + monkeypatch.setattr(project_module, "_scan_project_source_hints", fake_scan) + + payload = plan_project( + root=tmp_path, + plan_id="plan_dedupe", + profile="api-docs", + scan_site_hints=True, + max_pages_per_source=10, + ) + + selected_urls = [item["url"] for item in payload["selected"]] + assert selected_urls.count("https://docs.example.com/documentation/about") == 1 + assert "https://docs.example.com/documentation/about.md" not in selected_urls + + +def test_sync_quarantines_repeated_rate_limited_prefixes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeFetcher: + fetched_urls: list[str] = [] + + def __init__(self, config: Any) -> None: + self.config = config + self.stats = SimpleNamespace(pages_fetched=0, pages_failed=0, pages_skipped=0) + + async def __aenter__(self) -> FakeFetcher: + return self + + async def __aexit__(self, *_exc: object) -> None: + return None + + async def fetch_one(self, url: str, *, save: bool = True) -> SimpleNamespace: + FakeFetcher.fetched_urls.append(url) + if url.endswith("/api/a"): + self.stats.pages_failed += 1 + return SimpleNamespace(error="HTTP 429 Too Many Requests", should_skip=False) + self.stats.pages_fetched += 1 + if save: + output_dir = self.config.output.directory + output_dir.mkdir(parents=True, exist_ok=True) + content = f"Fetched {url}" + record = { + "document_id": f"doc_{self.stats.pages_fetched}", + "url": url, + "title": url, + "content": content, + "content_hash": _hash(content), + "source_type": "test", + "metadata": {}, + "extraction": {}, + } + with (output_dir / "documents.ndjson").open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record) + "\n") + return SimpleNamespace(error=None, should_skip=False, skip_code=None, skip_reason=None) + + monkeypatch.setattr(project_module, "Fetcher", FakeFetcher) + init_project(name="demo", source="https://docs.example.com/root", root=tmp_path) + config = project_module.load_project_config(tmp_path) + source = config.sources[0].model_copy( + update={ + "discovered_urls": [ + "https://docs.example.com/api/a", + "https://docs.example.com/api/b", + ] + } + ) + project_module.save_project_config(tmp_path, config.model_copy(update={"sources": [source]})) + + sync_project(root=tmp_path, run_id="run_429") + skips = _jsonl(tmp_path / ".docpull" / "runs" / "run_429" / "skips.jsonl") + + assert "https://docs.example.com/api/b" not in FakeFetcher.fetched_urls + assert skips[0]["reason"] == "rate_limited_prefix_quarantine" + def test_diff_status_export_and_eval_set( tmp_path: Path, diff --git a/tests/test_real_site_regressions.py b/tests/test_real_site_regressions.py index 6090855..996cd63 100644 --- a/tests/test_real_site_regressions.py +++ b/tests/test_real_site_regressions.py @@ -166,6 +166,9 @@ def test_markdown_allowed(self): def test_plain_text_allowed(self): assert "text/plain" in ALLOWED_CONTENT_TYPES + def test_yaml_allowed(self): + assert "application/yaml" in ALLOWED_CONTENT_TYPES + class TestSpaOutputDetection: def test_loading_shell_flagged(self): diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index e6fbbb9..66eff2a 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -15,6 +15,7 @@ from docpull.http.client import AsyncHttpClient, _ValidatedResolver from docpull.pipeline.base import PageContext from docpull.pipeline.steps.save import SaveStep +from docpull.security.download_policy import is_allowed_document_content_type from docpull.security.robots import RobotsChecker, _RobotsResponse from docpull.security.url_validator import UrlValidationResult, UrlValidator @@ -640,6 +641,70 @@ async def test_http_client_allows_safe_markdown_body(self) -> None: assert response.content == b"# Safe docs\n\nBody" assert response.content_type == "text/markdown; charset=utf-8" + @pytest.mark.asyncio + async def test_http_client_allows_combined_octet_stream_json_header(self) -> None: + client = AsyncHttpClient( + rate_limiter=_DummyRateLimiter(), + max_retries=0, + ) + client._session = _FakeSession( + [ + _FakeResponse( + 200, + headers={"Content-Type": "application/octet-stream, application/json"}, + chunks=[b'{"openapi":"3.1.0","paths":{}}'], + url="https://public.example/openapi.json", + ) + ] + ) + + response = await client.get("https://public.example/openapi.json") + + assert response.content == b'{"openapi":"3.1.0","paths":{}}' + assert response.content_type == "application/octet-stream, application/json" + assert is_allowed_document_content_type(response.content_type) is True + + @pytest.mark.asyncio + async def test_http_client_allows_yaml_spec_content_type(self) -> None: + client = AsyncHttpClient( + rate_limiter=_DummyRateLimiter(), + max_retries=0, + ) + client._session = _FakeSession( + [ + _FakeResponse( + 200, + headers={"Content-Type": "application/yaml"}, + chunks=[b"openapi: 3.1.0\npaths: {}\n"], + url="https://public.example/openapi.yml", + ) + ] + ) + + response = await client.get("https://public.example/openapi.yml") + + assert response.content == b"openapi: 3.1.0\npaths: {}\n" + assert is_allowed_document_content_type(response.content_type) is True + + @pytest.mark.asyncio + async def test_http_client_rejects_dangerous_type_in_combined_header(self) -> None: + response = _FakeResponse( + 200, + headers={"Content-Type": "text/plain, application/x-msdownload"}, + chunks=[b"plain text"], + url="https://public.example/download", + ) + client = AsyncHttpClient( + rate_limiter=_DummyRateLimiter(), + max_retries=0, + ) + client._session = _FakeSession([response]) + + with pytest.raises(ValueError, match="Disallowed content type 'application/x-msdownload'"): + await client.get("https://public.example/download") + + assert response.content.iterated is False + @pytest.mark.asyncio async def test_http_client_does_not_buffer_4xx_body(self) -> None: response = _FakeResponse( diff --git a/tests/test_special_cases.py b/tests/test_special_cases.py index cfe64ee..aa82f53 100644 --- a/tests/test_special_cases.py +++ b/tests/test_special_cases.py @@ -237,6 +237,30 @@ def test_rejects_json_without_openapi_key(self): html = json.dumps({"foo": "bar"}).encode() assert OpenApiExtractor().try_extract(html, "https://example.com/") is None + def test_renders_yaml_openapi_to_markdown(self): + spec = b""" +openapi: 3.1.0 +info: + title: YAML API + description: YAML specs should render. +paths: + /widgets: + get: + summary: List widgets + responses: + "200": + description: OK +""" + + result = OpenApiExtractor().try_extract(spec, "https://example.com/openapi.yml") + + assert result is not None + assert result.source_type == "openapi" + assert result.title == "YAML API" + assert "# YAML API" in result.markdown + assert "_OpenAPI 3.1.0_" in result.markdown + assert "`GET /widgets`" in result.markdown + def test_renders_request_body_properties_and_resolves_refs(self): spec = { "openapi": "3.0.0", diff --git a/web/.gitignore b/web/.gitignore deleted file mode 100644 index 98cae26..0000000 --- a/web/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -# Dependencies -node_modules/ - -# Next.js -.next/ -out/ -# Auto-generated by `next build` / `next dev`. Path inside flips between -# .next/types/routes.d.ts and .next/dev/types/routes.d.ts depending on -# which command was last run, so keeping it tracked produces noise. -next-env.d.ts - -# Build -dist/ - -# macOS -.DS_Store - -# IDEs -.vscode/ -.idea/ -*.swp -*.swo - -# Claude Code -.claude/ - -# Environment -.env -.env.local -.env.*.local - -# Logs -*.log -npm-debug.log* - -# TypeScript -*.tsbuildinfo -.vercel diff --git a/web/README.md b/web/README.md deleted file mode 100644 index 2ebe1d8..0000000 --- a/web/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# docpull web - -Marketing website for [docpull](https://github.com/raintree-technology/docpull) - the public web-source fetcher for AI agents. - -## Tech Stack - -- Next.js 16 -- React 19 -- TypeScript -- Tailwind CSS -- Framer Motion - -## Development - -```bash -npm install -npm run dev -``` - -Open [http://localhost:3013](http://localhost:3013). - -## Build - -```bash -npm run build -npm run start -``` - -## License - -MIT diff --git a/web/app/docs/page.tsx b/web/app/docs/page.tsx deleted file mode 100644 index 5175b6f..0000000 --- a/web/app/docs/page.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import type { Metadata } from "next"; -import { DocsArticle } from "@/components/docs/DocsArticle"; -import { DocsShell } from "@/components/docs/DocsShell"; - -export const metadata: Metadata = { - title: "Docs - docpull", - description: - "Install and use docpull to turn public web pages into local Markdown, NDJSON, context packs, MCP tools, and agent-ready source artifacts.", - alternates: { - canonical: "/docs", - }, -}; - -export default function DocsPage() { - return ( - - - - ); -} diff --git a/web/app/globals.css b/web/app/globals.css deleted file mode 100644 index c3c5c06..0000000 --- a/web/app/globals.css +++ /dev/null @@ -1,282 +0,0 @@ -@import 'tailwindcss'; - -@custom-variant dark (&:is(.dark *)); - -@theme inline { - --color-border: hsl(var(--border)); - --color-input: hsl(var(--input)); - --color-ring: hsl(var(--ring)); - --color-background: hsl(var(--background)); - --color-foreground: hsl(var(--foreground)); - - --color-primary: hsl(var(--primary)); - --color-primary-foreground: hsl(var(--primary-foreground)); - - --color-secondary: hsl(var(--secondary)); - --color-secondary-foreground: hsl(var(--secondary-foreground)); - - --color-destructive: hsl(var(--destructive)); - --color-destructive-foreground: hsl(var(--destructive-foreground)); - - --color-muted: hsl(var(--muted)); - --color-muted-foreground: hsl(var(--muted-foreground)); - - --color-accent: hsl(var(--accent)); - --color-accent-foreground: hsl(var(--accent-foreground)); - - --color-popover: hsl(var(--popover)); - --color-popover-foreground: hsl(var(--popover-foreground)); - - --color-card: hsl(var(--card)); - --color-card-foreground: hsl(var(--card-foreground)); - - --radius-lg: var(--radius); - --radius-md: calc(var(--radius) - 2px); - --radius-sm: calc(var(--radius) - 4px); -} - -/* - The default border color has changed to `currentcolor` in Tailwind CSS v4, - so we've added these compatibility styles to make sure everything still - looks the same as it did with Tailwind CSS v3. - - If we ever want to remove these styles, we need to add an explicit border - color utility to any element that depends on these defaults. -*/ -@layer base { - *, - ::after, - ::before, - ::backdrop, - ::file-selector-button { - border-color: var(--color-gray-200, currentcolor); - } -} - -@utility safe-padding { - padding-left: max(1.5rem, env(safe-area-inset-left)); - padding-right: max(1.5rem, env(safe-area-inset-right)); -} - -@layer base { - :root { - --background: 0 0% 100%; - --foreground: 0 0% 3.9%; - --card: 0 0% 100%; - --card-foreground: 0 0% 3.9%; - --popover: 0 0% 100%; - --popover-foreground: 0 0% 3.9%; - --primary: 0 0% 9%; - --primary-foreground: 0 0% 98%; - --secondary: 0 0% 96.1%; - --secondary-foreground: 0 0% 9%; - --muted: 0 0% 96.1%; - --muted-foreground: 0 0% 34%; - --accent: 0 0% 96.1%; - --accent-foreground: 0 0% 9%; - --destructive: 0 84.2% 60.2%; - --destructive-foreground: 0 0% 98%; - --border: 0 0% 89.8%; - --input: 0 0% 89.8%; - --ring: 0 0% 3.9%; - --radius: 0.5rem; - } - - .dark { - --background: 0 0% 3.9%; - --foreground: 0 0% 98%; - --card: 0 0% 3.9%; - --card-foreground: 0 0% 98%; - --popover: 0 0% 3.9%; - --popover-foreground: 0 0% 98%; - --primary: 0 0% 98%; - --primary-foreground: 0 0% 9%; - --secondary: 0 0% 14.9%; - --secondary-foreground: 0 0% 98%; - --muted: 0 0% 14.9%; - --muted-foreground: 0 0% 74%; - --accent: 0 0% 14.9%; - --accent-foreground: 0 0% 98%; - --destructive: 0 62.8% 30.6%; - --destructive-foreground: 0 0% 98%; - --border: 0 0% 14.9%; - --input: 0 0% 14.9%; - --ring: 0 0% 83.1%; - } -} - -@layer base { - * { - @apply border-border; - } - body { - @apply bg-background text-foreground; - line-height: 1.5; - text-rendering: optimizeLegibility; - font-feature-settings: - "rlig" 1, - "calt" 1; - } -} - -@layer utilities { - html { - scroll-behavior: smooth; - } - - @media (prefers-reduced-motion: reduce) { - html { - scroll-behavior: auto; - } - - *, - *::before, - *::after { - animation-duration: 0.01ms !important; - animation-iteration-count: 1 !important; - scroll-behavior: auto !important; - transition-duration: 0.01ms !important; - } - } - - /* Minimal scrollbar */ - ::-webkit-scrollbar { - width: 6px; - height: 6px; - } - - ::-webkit-scrollbar-track { - background: transparent; - } - - ::-webkit-scrollbar-thumb { - background: hsl(var(--border)); - border-radius: 3px; - } - - ::-webkit-scrollbar-thumb:hover { - background: hsl(var(--muted-foreground)); - } - - /* Glass effect for code blocks and inputs */ - .glass { - background: rgba(255, 255, 255, 0.05); - backdrop-filter: blur(18px); - -webkit-backdrop-filter: blur(18px); - border: 1px solid rgba(255, 255, 255, 0.08); - box-shadow: - 0 1px 2px rgba(0, 0, 0, 0.04), - inset 0 1px 0 rgba(255, 255, 255, 0.1); - } - - :root .glass { - background: rgba(255, 255, 255, 0.78); - backdrop-filter: blur(18px); - -webkit-backdrop-filter: blur(18px); - border: 1px solid rgba(0, 0, 0, 0.08); - box-shadow: - 0 1px 2px rgba(0, 0, 0, 0.04), - inset 0 1px 0 rgba(255, 255, 255, 0.8); - } - - .dark .glass { - background: rgba(20, 20, 20, 0.82); - backdrop-filter: blur(18px); - -webkit-backdrop-filter: blur(18px); - border: 1px solid rgba(255, 255, 255, 0.08); - box-shadow: - 0 1px 2px rgba(0, 0, 0, 0.25), - inset 0 1px 0 rgba(255, 255, 255, 0.1); - } - - /* Code styling */ - pre { - @apply rounded-lg overflow-x-auto; - } - - code { - font-family: - 'SF Mono', 'Fira Code', 'Fira Mono', Menlo, Monaco, Consolas, monospace; - font-size: 0.875rem; - line-height: 1.6; - } - - /* Terminal */ - .terminal { - background: rgba(10, 10, 10, 0.95); - backdrop-filter: blur(16px); - -webkit-backdrop-filter: blur(16px); - border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 0.5rem; - overflow: hidden; - box-shadow: - 0 12px 28px rgba(0, 0, 0, 0.22), - inset 0 1px 0 rgba(255, 255, 255, 0.05); - } - - .terminal-header { - background: rgba(15, 15, 15, 0.6); - border-bottom: 1px solid rgba(255, 255, 255, 0.05); - padding: 0.75rem 1rem; - display: flex; - align-items: center; - gap: 0.5rem; - } - - .terminal-dot { - @apply w-2.5 h-2.5 rounded-full; - } - - .terminal-dot-close { - @apply bg-[#ff5f57]; - } - .terminal-dot-minimize { - @apply bg-[#febc2e]; - } - .terminal-dot-maximize { - @apply bg-[#28c840]; - } - - /* Responsive utilities */ -} - -/* Flow animations for How-it-works connectors */ -@keyframes flow-right { - 0% { - stroke-dashoffset: 56; - } - 100% { - stroke-dashoffset: 0; - } -} - -@keyframes flow-down { - 0% { - stroke-dashoffset: 28; - } - 100% { - stroke-dashoffset: 0; - } -} - -/* Prevent horizontal overflow on mobile */ -html, -body { - overflow-x: hidden; - max-width: 100vw; -} - -/* Better touch targets on mobile */ -@media (max-width: 640px) { - button, - a { - min-height: 44px; - min-width: 44px; - } -} - -button:focus-visible, -a:focus-visible { - outline: 2px solid hsl(var(--ring)); - outline-offset: 3px; -} diff --git a/web/app/layout.tsx b/web/app/layout.tsx deleted file mode 100644 index e095ae8..0000000 --- a/web/app/layout.tsx +++ /dev/null @@ -1,100 +0,0 @@ -import type { Metadata } from "next"; -import { Analytics } from "@vercel/analytics/next"; -import "./globals.css"; -import { ThemeProvider } from "@/components/ThemeProvider"; - -const baseUrl = "https://docpull.raintree.technology"; -const analyticsEnabled = process.env.VERCEL_ENV === "production"; - -export const metadata: Metadata = { - metadataBase: new URL(baseUrl), - title: "docpull - Public web to agent-ready Markdown.", - description: - "Python CLI, SDK, and MCP server that turns public static and server-rendered web pages into clean Markdown, NDJSON, and local context packs for AI agents and RAG.", - applicationName: "docpull", - authors: [{ name: "Raintree Technology", url: "https://raintree.technology" }], - creator: "Raintree Technology", - publisher: "Raintree Technology", - alternates: { - canonical: "/", - }, - robots: { - index: true, - follow: true, - googleBot: { - index: true, - follow: true, - "max-image-preview": "large", - "max-snippet": -1, - "max-video-preview": -1, - }, - }, - icons: { - icon: "/logo.svg", - }, - keywords: [ - "web extraction", - "web to markdown", - "website to markdown", - "source packs", - "web source packs", - "documentation", - "markdown", - "AI", - "LLM", - "RAG", - "MCP", - "AI agents", - "web scraping", - "web crawler", - "python", - "cli", - "sdk", - "docs", - "crawler", - "Parallel", - "context packs", - ], - - openGraph: { - title: "docpull - Public web to agent-ready Markdown.", - description: - "Turn public static and server-rendered web pages into clean Markdown, NDJSON, and local context packs for AI agents, MCP clients, and RAG pipelines.", - url: baseUrl, - type: "website", - siteName: "docpull", - locale: "en_US", - images: [ - { - url: "/og-image.png", - width: 1200, - height: 630, - alt: "docpull - Public web to agent-ready Markdown.", - }, - ], - }, - twitter: { - card: "summary_large_image", - title: "docpull - Public web to agent-ready Markdown.", - description: - "Turn public static and server-rendered web pages into clean Markdown, NDJSON, and local context packs for AI agents, MCP clients, and RAG pipelines.", - site: "@raintree_tech", - creator: "@raintree_tech", - images: ["/og-image.png"], - }, -}; - -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode; -}>) { - return ( - - - {children} - {analyticsEnabled ? : null} - - - ); -} diff --git a/web/app/not-found.tsx b/web/app/not-found.tsx deleted file mode 100644 index 783b625..0000000 --- a/web/app/not-found.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import type { Metadata } from "next"; -import Link from "next/link"; - -// Spec: SEO / Soft 404s. This route renders under Next's not-found boundary, -// which returns a real HTTP 404 — never a 200 "looks empty" page. noindex keeps -// any stray 404 URLs out of the index. -export const metadata: Metadata = { - title: "Page not found - docpull", - robots: { index: false, follow: true }, -}; - -export default function NotFound() { - return ( -
-
-

- 404 -

-

- Page not found -

-

- The page you're looking for doesn't exist or has moved. -

- - Back to home - -
-
- ); -} diff --git a/web/app/page.tsx b/web/app/page.tsx deleted file mode 100644 index d1bd9bf..0000000 --- a/web/app/page.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import AsciiBackgroundLoader from "@/components/AsciiBackgroundLoader"; -import Header from "@/components/Header"; -import Hero from "@/components/Hero"; -import Footer from "@/components/Footer"; -import Features from "@/components/Features"; -import Comparison from "@/components/Comparison"; -import HowItWorks from "@/components/HowItWorks"; -import Profiles from "@/components/Profiles"; -import CodeExamples from "@/components/CodeExamples"; -import ParallelPacks from "@/components/ParallelPacks"; -import Install from "@/components/Install"; -import FAQ from "@/components/FAQ"; -import StructuredData from "@/components/StructuredData"; - -export default function Home() { - return ( - <> - - -
-
- - - - - - - - - -
-