diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7759c8e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + cache: pip + - run: python -m pip install --upgrade pip + - run: pip install -e ".[dev]" + - name: Test (pytest) + run: pytest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ab3f60d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release + +# Publishes to PyPI via Trusted Publishing (OIDC) on a GitHub Release. +# One-time setup: on pypi.org add a Trusted Publisher for this repo + +# workflow (langchain-wellmarked / release.yml / environment: pypi). +# No API token. +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + - run: python -m pip install --upgrade pip build + - run: python -m build + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for Trusted Publishing + steps: + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0d2085 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.venv/ +dist/ +*.egg-info/ diff --git a/README.md b/README.md index 755fc3d..eb11565 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,57 @@ # langchain-wellmarked -The official langchain integration for WellMarked.io! + +The official [LangChain](https://www.langchain.com/) integration for **[WellMarked](https://wellmarked.io)** — load any URL as clean Markdown `Document`s. + +```bash +pip install langchain-wellmarked +``` + +## Quick start + +Set your API key (get one at [wellmarked.io](https://wellmarked.io)): + +```bash +export WELLMARKED_API_KEY="wm_..." +``` + +Load a single page: + +```python +from langchain_wellmarked import WellMarkedLoader + +loader = WellMarkedLoader("https://example.com/article") +docs = loader.load() + +docs[0].page_content # clean Markdown +docs[0].metadata # {"source": ..., "title": ..., "author": ..., "retrieved_at": ...} +``` + +Or crawl a whole site BFS-style — one `Document` per successfully extracted page (Pro plan and above): + +```python +loader = WellMarkedLoader("https://docs.example.com", mode="crawl", depth=2) +docs = loader.load() + +docs[0].metadata["depth"] # how far from the root URL this page sits +``` + +`lazy_load()` works too, and the loader passes `render_js=True` through to the API for JS-heavy pages. + +## Options + +| Parameter | Default | Description | +|---------------|-------------|--------------------------------------------------------------------| +| `url` | (required) | Page to extract, or root URL to crawl from | +| `api_key` | env var | WellMarked API key; falls back to `WELLMARKED_API_KEY` | +| `mode` | `"extract"` | `"extract"` (single page) or `"crawl"` (same-site BFS) | +| `depth` | `1` | Crawl depth (`mode="crawl"` only) | +| `render_js` | `False` | Render JS-heavy pages with a headless browser (Pro and above) | +| `job_timeout` | `300.0` | Seconds to wait for a crawl job; `None` waits forever | + +In `crawl` mode, pages that fail to extract (timeouts, robots-disallowed, no content) are skipped; only successful pages become `Document`s. + +## Related + +- [`wellmarked`](https://pypi.org/project/wellmarked/) — the underlying Python SDK (this package wraps it) +- [WellMarked API docs](https://wellmarked.io/docs) +- [`llama-index-readers-wellmarked`](https://github.com/WellMarkedAPI/llama-index-readers-wellmarked) — the LlamaIndex equivalent diff --git a/langchain_wellmarked/__init__.py b/langchain_wellmarked/__init__.py new file mode 100644 index 0000000..66028a5 --- /dev/null +++ b/langchain_wellmarked/__init__.py @@ -0,0 +1,12 @@ +"""Official LangChain integration for the WellMarked API. + + from langchain_wellmarked import WellMarkedLoader + + loader = WellMarkedLoader("https://example.com/article") + docs = loader.load() + +See https://wellmarked.io/docs for the full API reference. +""" +from langchain_wellmarked.document_loaders import WellMarkedLoader + +__all__ = ["WellMarkedLoader"] diff --git a/langchain_wellmarked/document_loaders.py b/langchain_wellmarked/document_loaders.py new file mode 100644 index 0000000..b93421c --- /dev/null +++ b/langchain_wellmarked/document_loaders.py @@ -0,0 +1,116 @@ +"""WellMarked document loader for LangChain.""" +from __future__ import annotations + +from typing import Any, Iterator, Literal, Optional + +from langchain_core.document_loaders import BaseLoader +from langchain_core.documents import Document +from wellmarked import ExtractionMeta, WellMarked + + +def _metadata(meta: ExtractionMeta, source: str, depth: Optional[int] = None) -> dict[str, Any]: + """Build JSON-serializable Document metadata from the SDK's ExtractionMeta.""" + out: dict[str, Any] = {"source": meta.url or source} + if meta.title is not None: + out["title"] = meta.title + if meta.author is not None: + out["author"] = meta.author + if meta.date is not None: + out["date"] = meta.date + if meta.retrieved_at is not None: + out["retrieved_at"] = meta.retrieved_at.isoformat() + if depth is not None: + out["depth"] = depth + return out + + +class WellMarkedLoader(BaseLoader): + """Load web pages as clean Markdown via the WellMarked API. + + Setup: + Install ``langchain-wellmarked`` and set the ``WELLMARKED_API_KEY`` + environment variable (or pass ``api_key=``). Get a key at + https://wellmarked.io. + + .. code-block:: bash + + pip install langchain-wellmarked + + Instantiate: + .. code-block:: python + + from langchain_wellmarked import WellMarkedLoader + + loader = WellMarkedLoader("https://example.com/article") + + # Or crawl a whole site, one Document per page: + loader = WellMarkedLoader( + "https://docs.example.com", mode="crawl", depth=2 + ) + + Load: + .. code-block:: python + + docs = loader.load() + docs[0].page_content # clean Markdown + docs[0].metadata # {"source": ..., "title": ..., ...} + + In ``crawl`` mode the loader blocks until the crawl job finishes + (up to ``job_timeout`` seconds) and yields only successfully + extracted pages; failed pages (timeouts, robots-disallowed) are + skipped. Crawling requires a Pro plan or above. + """ + + def __init__( + self, + url: str, + *, + api_key: Optional[str] = None, + mode: Literal["extract", "crawl"] = "extract", + depth: int = 1, + render_js: bool = False, + job_timeout: Optional[float] = 300.0, + ) -> None: + """Create the loader. + + Args: + url: The URL to extract (``mode="extract"``) or the root URL + to crawl from (``mode="crawl"``). + api_key: WellMarked API key (``wm_...``). Falls back to the + ``WELLMARKED_API_KEY`` environment variable. + mode: ``"extract"`` loads the single page at ``url``; + ``"crawl"`` BFS-crawls same-site links from ``url``. + depth: Crawl depth (``mode="crawl"`` only). + render_js: Render JS-heavy pages with a headless browser + (Pro plan and above). + job_timeout: Seconds to wait for a crawl job to finish. + ``None`` waits forever. + """ + if mode not in ("extract", "crawl"): + raise ValueError(f"mode must be 'extract' or 'crawl', got {mode!r}") + self.url = url + self.api_key = api_key + self.mode = mode + self.depth = depth + self.render_js = render_js + self.job_timeout = job_timeout + + def lazy_load(self) -> Iterator[Document]: + with WellMarked(api_key=self.api_key) as wm: + if self.mode == "extract": + result = wm.extract(self.url, render_js=self.render_js) + yield Document( + page_content=result.markdown, + metadata=_metadata(result.metadata, self.url), + ) + else: + job = wm.crawl(self.url, depth=self.depth, render_js=self.render_js) + job = wm.wait_for_job(job.job_id, timeout=self.job_timeout) + for page in job.results: + if not page.ok: + continue + assert page.markdown is not None and page.metadata is not None + yield Document( + page_content=page.markdown, + metadata=_metadata(page.metadata, page.url, depth=page.depth), + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f9ab508 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["hatchling>=1.21"] +build-backend = "hatchling.build" + +[project] +name = "langchain-wellmarked" +version = "0.1.0" +description = "Official LangChain integration for the WellMarked API — load any URL as clean Markdown." +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.9" +authors = [ + { name = "WellMarked", email = "support@wellmarked.io" }, +] +keywords = ["wellmarked", "langchain", "document-loader", "markdown", "scraping", "rag", "llm"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +dependencies = [ + "langchain-core>=0.3,<2.0", + "wellmarked>=1.1,<2.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7", +] + +[project.urls] +Homepage = "https://wellmarked.io" +Documentation = "https://wellmarked.io/docs" +Source = "https://github.com/WellMarkedAPI/langchain-wellmarked" +Issues = "https://github.com/WellMarkedAPI/langchain-wellmarked/issues" + +[tool.hatch.build.targets.wheel] +packages = ["langchain_wellmarked"] + +[tool.hatch.build.targets.sdist] +include = [ + "langchain_wellmarked", + "README.md", + "LICENSE", + "pyproject.toml", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/tests/test_document_loaders.py b/tests/test_document_loaders.py new file mode 100644 index 0000000..7b07388 --- /dev/null +++ b/tests/test_document_loaders.py @@ -0,0 +1,112 @@ +"""Unit tests for WellMarkedLoader — the SDK client is faked, no network.""" +from datetime import datetime, timezone + +import pytest +from wellmarked import CrawlItem, CrawlJob, ExtractionMeta, ExtractResult + +import langchain_wellmarked.document_loaders as mod +from langchain_wellmarked import WellMarkedLoader + +RETRIEVED = datetime(2026, 6, 10, 12, 0, 0, tzinfo=timezone.utc) + + +class FakeWellMarked: + """Stands in for wellmarked.WellMarked. Records calls, returns canned data.""" + + calls: list = [] + + def __init__(self, api_key=None, **kwargs): + FakeWellMarked.calls.append(("init", api_key)) + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + pass + + def extract(self, url, *, render_js=False): + FakeWellMarked.calls.append(("extract", url, render_js)) + return ExtractResult( + markdown="# Hello", + metadata=ExtractionMeta( + url=url, title="Hello", author="Ada", retrieved_at=RETRIEVED + ), + request_id="req_1", + ) + + def crawl(self, url, *, depth=1, render_js=False, **kwargs): + FakeWellMarked.calls.append(("crawl", url, depth, render_js)) + return CrawlJob(job_id="job_1", status="queued", total=0, completed=0, results=[]) + + def wait_for_job(self, job_id, *, timeout=300.0, **kwargs): + FakeWellMarked.calls.append(("wait_for_job", job_id, timeout)) + return CrawlJob( + job_id=job_id, + status="done", + total=2, + completed=2, + results=[ + CrawlItem( + url="https://docs.example.com", + depth=0, + markdown="# Root", + metadata=ExtractionMeta(url="https://docs.example.com", title="Root"), + ), + CrawlItem(url="https://docs.example.com/dead", depth=1, error="target_timeout"), + ], + ) + + +@pytest.fixture(autouse=True) +def fake_client(monkeypatch): + FakeWellMarked.calls = [] + monkeypatch.setattr(mod, "WellMarked", FakeWellMarked) + + +def test_extract_mode_yields_one_document(): + docs = WellMarkedLoader("https://example.com/article", api_key="wm_test").load() + + assert len(docs) == 1 + assert docs[0].page_content == "# Hello" + assert docs[0].metadata == { + "source": "https://example.com/article", + "title": "Hello", + "author": "Ada", + "retrieved_at": RETRIEVED.isoformat(), + } + assert ("init", "wm_test") in FakeWellMarked.calls + assert ("extract", "https://example.com/article", False) in FakeWellMarked.calls + + +def test_extract_mode_passes_render_js(): + WellMarkedLoader("https://example.com", render_js=True).load() + + assert ("extract", "https://example.com", True) in FakeWellMarked.calls + + +def test_crawl_mode_yields_ok_pages_and_skips_failures(): + docs = WellMarkedLoader( + "https://docs.example.com", mode="crawl", depth=2, job_timeout=60.0 + ).load() + + assert ("crawl", "https://docs.example.com", 2, False) in FakeWellMarked.calls + assert ("wait_for_job", "job_1", 60.0) in FakeWellMarked.calls + assert len(docs) == 1 # the failed page is skipped + assert docs[0].page_content == "# Root" + assert docs[0].metadata == { + "source": "https://docs.example.com", + "title": "Root", + "depth": 0, + } + + +def test_invalid_mode_raises(): + with pytest.raises(ValueError, match="mode must be"): + WellMarkedLoader("https://example.com", mode="bulk") + + +def test_lazy_load_is_lazy(): + it = WellMarkedLoader("https://example.com").lazy_load() + assert FakeWellMarked.calls == [] # nothing happens until iteration + next(it) + assert ("extract", "https://example.com", False) in FakeWellMarked.calls