Add WellMarkedLoader document loader

.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev]"
+      - name: Test (pytest)
+        run: pytest

.github/workflows/release.yml

@@ -0,0 +1,37 @@
+name: Release
+
+# Publishes to PyPI via Trusted Publishing (OIDC) on a GitHub Release.
+# One-time setup: on pypi.org add a Trusted Publisher for this repo +
+# workflow (langchain-wellmarked / release.yml / environment: pypi).
+# No API token.
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - run: python -m pip install --upgrade pip build
+      - run: python -m build
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write   # required for Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.venv/
+dist/
+*.egg-info/

README.md

@@ -1,2 +1,57 @@
 # langchain-wellmarked
-The official langchain integration for WellMarked.io!
+
+The official [LangChain](https://www.langchain.com/) integration for **[WellMarked](https://wellmarked.io)** — load any URL as clean Markdown `Document`s.
+
+```bash
+pip install langchain-wellmarked
+```
+
+## Quick start
+
+Set your API key (get one at [wellmarked.io](https://wellmarked.io)):
+
+```bash
+export WELLMARKED_API_KEY="wm_..."
+```
+
+Load a single page:
+
+```python
+from langchain_wellmarked import WellMarkedLoader
+
+loader = WellMarkedLoader("https://example.com/article")
+docs = loader.load()
+
+docs[0].page_content   # clean Markdown
+docs[0].metadata       # {"source": ..., "title": ..., "author": ..., "retrieved_at": ...}
+```
+
+Or crawl a whole site BFS-style — one `Document` per successfully extracted page (Pro plan and above):
+
+```python
+loader = WellMarkedLoader("https://docs.example.com", mode="crawl", depth=2)
+docs = loader.load()
+
+docs[0].metadata["depth"]   # how far from the root URL this page sits
+```
+
+`lazy_load()` works too, and the loader passes `render_js=True` through to the API for JS-heavy pages.
+
+## Options
+
+| Parameter     | Default     | Description                                                        |
+|---------------|-------------|--------------------------------------------------------------------|
+| `url`         | (required)  | Page to extract, or root URL to crawl from                          |
+| `api_key`     | env var     | WellMarked API key; falls back to `WELLMARKED_API_KEY`              |
+| `mode`        | `"extract"` | `"extract"` (single page) or `"crawl"` (same-site BFS)              |
+| `depth`       | `1`         | Crawl depth (`mode="crawl"` only)                                   |
+| `render_js`   | `False`     | Render JS-heavy pages with a headless browser (Pro and above)       |
+| `job_timeout` | `300.0`     | Seconds to wait for a crawl job; `None` waits forever               |
+
+In `crawl` mode, pages that fail to extract (timeouts, robots-disallowed, no content) are skipped; only successful pages become `Document`s.
+
+## Related
+
+- [`wellmarked`](https://pypi.org/project/wellmarked/) — the underlying Python SDK (this package wraps it)
+- [WellMarked API docs](https://wellmarked.io/docs)
+- [`llama-index-readers-wellmarked`](https://github.com/WellMarkedAPI/llama-index-readers-wellmarked) — the LlamaIndex equivalent

langchain_wellmarked/__init__.py

@@ -0,0 +1,12 @@
+"""Official LangChain integration for the WellMarked API.
+
+    from langchain_wellmarked import WellMarkedLoader
+
+    loader = WellMarkedLoader("https://example.com/article")
+    docs = loader.load()
+
+See https://wellmarked.io/docs for the full API reference.
+"""
+from langchain_wellmarked.document_loaders import WellMarkedLoader
+
+__all__ = ["WellMarkedLoader"]

langchain_wellmarked/document_loaders.py

@@ -0,0 +1,116 @@
+"""WellMarked document loader for LangChain."""
+from __future__ import annotations
+
+from typing import Any, Iterator, Literal, Optional
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+from wellmarked import ExtractionMeta, WellMarked
+
+
+def _metadata(meta: ExtractionMeta, source: str, depth: Optional[int] = None) -> dict[str, Any]:
+    """Build JSON-serializable Document metadata from the SDK's ExtractionMeta."""
+    out: dict[str, Any] = {"source": meta.url or source}
+    if meta.title is not None:
+        out["title"] = meta.title
+    if meta.author is not None:
+        out["author"] = meta.author
+    if meta.date is not None:
+        out["date"] = meta.date
+    if meta.retrieved_at is not None:
+        out["retrieved_at"] = meta.retrieved_at.isoformat()
+    if depth is not None:
+        out["depth"] = depth
+    return out
+
+
+class WellMarkedLoader(BaseLoader):
+    """Load web pages as clean Markdown via the WellMarked API.
+
+    Setup:
+        Install ``langchain-wellmarked`` and set the ``WELLMARKED_API_KEY``
+        environment variable (or pass ``api_key=``). Get a key at
+        https://wellmarked.io.
+
+        .. code-block:: bash
+
+            pip install langchain-wellmarked
+
+    Instantiate:
+        .. code-block:: python
+
+            from langchain_wellmarked import WellMarkedLoader
+
+            loader = WellMarkedLoader("https://example.com/article")
+
+            # Or crawl a whole site, one Document per page:
+            loader = WellMarkedLoader(
+                "https://docs.example.com", mode="crawl", depth=2
+            )
+
+    Load:
+        .. code-block:: python
+
+            docs = loader.load()
+            docs[0].page_content   # clean Markdown
+            docs[0].metadata       # {"source": ..., "title": ..., ...}
+
+    In ``crawl`` mode the loader blocks until the crawl job finishes
+    (up to ``job_timeout`` seconds) and yields only successfully
+    extracted pages; failed pages (timeouts, robots-disallowed) are
+    skipped. Crawling requires a Pro plan or above.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        api_key: Optional[str] = None,
+        mode: Literal["extract", "crawl"] = "extract",
+        depth: int = 1,
+        render_js: bool = False,
+        job_timeout: Optional[float] = 300.0,
+    ) -> None:
+        """Create the loader.
+
+        Args:
+            url: The URL to extract (``mode="extract"``) or the root URL
+                to crawl from (``mode="crawl"``).
+            api_key: WellMarked API key (``wm_...``). Falls back to the
+                ``WELLMARKED_API_KEY`` environment variable.
+            mode: ``"extract"`` loads the single page at ``url``;
+                ``"crawl"`` BFS-crawls same-site links from ``url``.
+            depth: Crawl depth (``mode="crawl"`` only).
+            render_js: Render JS-heavy pages with a headless browser
+                (Pro plan and above).
+            job_timeout: Seconds to wait for a crawl job to finish.
+                ``None`` waits forever.
+        """
+        if mode not in ("extract", "crawl"):
+            raise ValueError(f"mode must be 'extract' or 'crawl', got {mode!r}")
+        self.url = url
+        self.api_key = api_key
+        self.mode = mode
+        self.depth = depth
+        self.render_js = render_js
+        self.job_timeout = job_timeout
+
+    def lazy_load(self) -> Iterator[Document]:
+        with WellMarked(api_key=self.api_key) as wm:
+            if self.mode == "extract":
+                result = wm.extract(self.url, render_js=self.render_js)
+                yield Document(
+                    page_content=result.markdown,
+                    metadata=_metadata(result.metadata, self.url),
+                )
+            else:
+                job = wm.crawl(self.url, depth=self.depth, render_js=self.render_js)
+                job = wm.wait_for_job(job.job_id, timeout=self.job_timeout)
+                for page in job.results:
+                    if not page.ok:
+                        continue
+                    assert page.markdown is not None and page.metadata is not None
+                    yield Document(
+                        page_content=page.markdown,
+                        metadata=_metadata(page.metadata, page.url, depth=page.depth),
+                    )

pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["hatchling>=1.21"]
+build-backend = "hatchling.build"
+
+[project]
+name = "langchain-wellmarked"
+version = "0.1.0"
+description = "Official LangChain integration for the WellMarked API — load any URL as clean Markdown."
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+authors = [
+    { name = "WellMarked", email = "support@wellmarked.io" },
+]
+keywords = ["wellmarked", "langchain", "document-loader", "markdown", "scraping", "rag", "llm"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+dependencies = [
+    "langchain-core>=0.3,<2.0",
+    "wellmarked>=1.1,<2.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7",
+]
+
+[project.urls]
+Homepage = "https://wellmarked.io"
+Documentation = "https://wellmarked.io/docs"
+Source = "https://github.com/WellMarkedAPI/langchain-wellmarked"
+Issues = "https://github.com/WellMarkedAPI/langchain-wellmarked/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["langchain_wellmarked"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "langchain_wellmarked",
+    "README.md",
+    "LICENSE",
+    "pyproject.toml",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]