From c99eab956ed2ff93e37ce92d1658318386ede9ad Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 10 Jun 2026 18:41:11 -0400 Subject: [PATCH 1/3] Add WellMarkedLoader document loader Official LangChain integration for the WellMarked API, wrapping the wellmarked Python SDK. WellMarkedLoader supports extract mode (one URL, one Document) and crawl mode (same-site BFS, one Document per successfully extracted page, blocking on the crawl job). Markdown maps to page_content; title/author/date/retrieved_at/depth map to metadata. Includes hatchling packaging mirroring the Python SDK conventions, unit tests with a faked SDK client, and README docs. Co-Authored-By: Claude Fable 5 --- README.md | 57 ++++++++- langchain_wellmarked/__init__.py | 12 ++ langchain_wellmarked/document_loaders.py | 116 ++++++++++++++++++ pyproject.toml | 56 +++++++++ ...ument_loaders.cpython-314-pytest-9.0.3.pyc | Bin 0 -> 14387 bytes tests/test_document_loaders.py | 112 +++++++++++++++++ 6 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 langchain_wellmarked/__init__.py create mode 100644 langchain_wellmarked/document_loaders.py create mode 100644 pyproject.toml create mode 100644 tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc create mode 100644 tests/test_document_loaders.py diff --git a/README.md b/README.md index 755fc3d..eb11565 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,57 @@ # langchain-wellmarked -The official langchain integration for WellMarked.io! + +The official [LangChain](https://www.langchain.com/) integration for **[WellMarked](https://wellmarked.io)** — load any URL as clean Markdown `Document`s. + +```bash +pip install langchain-wellmarked +``` + +## Quick start + +Set your API key (get one at [wellmarked.io](https://wellmarked.io)): + +```bash +export WELLMARKED_API_KEY="wm_..." +``` + +Load a single page: + +```python +from langchain_wellmarked import WellMarkedLoader + +loader = WellMarkedLoader("https://example.com/article") +docs = loader.load() + +docs[0].page_content # clean Markdown +docs[0].metadata # {"source": ..., "title": ..., "author": ..., "retrieved_at": ...} +``` + +Or crawl a whole site BFS-style — one `Document` per successfully extracted page (Pro plan and above): + +```python +loader = WellMarkedLoader("https://docs.example.com", mode="crawl", depth=2) +docs = loader.load() + +docs[0].metadata["depth"] # how far from the root URL this page sits +``` + +`lazy_load()` works too, and the loader passes `render_js=True` through to the API for JS-heavy pages. + +## Options + +| Parameter | Default | Description | +|---------------|-------------|--------------------------------------------------------------------| +| `url` | (required) | Page to extract, or root URL to crawl from | +| `api_key` | env var | WellMarked API key; falls back to `WELLMARKED_API_KEY` | +| `mode` | `"extract"` | `"extract"` (single page) or `"crawl"` (same-site BFS) | +| `depth` | `1` | Crawl depth (`mode="crawl"` only) | +| `render_js` | `False` | Render JS-heavy pages with a headless browser (Pro and above) | +| `job_timeout` | `300.0` | Seconds to wait for a crawl job; `None` waits forever | + +In `crawl` mode, pages that fail to extract (timeouts, robots-disallowed, no content) are skipped; only successful pages become `Document`s. + +## Related + +- [`wellmarked`](https://pypi.org/project/wellmarked/) — the underlying Python SDK (this package wraps it) +- [WellMarked API docs](https://wellmarked.io/docs) +- [`llama-index-readers-wellmarked`](https://github.com/WellMarkedAPI/llama-index-readers-wellmarked) — the LlamaIndex equivalent diff --git a/langchain_wellmarked/__init__.py b/langchain_wellmarked/__init__.py new file mode 100644 index 0000000..66028a5 --- /dev/null +++ b/langchain_wellmarked/__init__.py @@ -0,0 +1,12 @@ +"""Official LangChain integration for the WellMarked API. + + from langchain_wellmarked import WellMarkedLoader + + loader = WellMarkedLoader("https://example.com/article") + docs = loader.load() + +See https://wellmarked.io/docs for the full API reference. +""" +from langchain_wellmarked.document_loaders import WellMarkedLoader + +__all__ = ["WellMarkedLoader"] diff --git a/langchain_wellmarked/document_loaders.py b/langchain_wellmarked/document_loaders.py new file mode 100644 index 0000000..b93421c --- /dev/null +++ b/langchain_wellmarked/document_loaders.py @@ -0,0 +1,116 @@ +"""WellMarked document loader for LangChain.""" +from __future__ import annotations + +from typing import Any, Iterator, Literal, Optional + +from langchain_core.document_loaders import BaseLoader +from langchain_core.documents import Document +from wellmarked import ExtractionMeta, WellMarked + + +def _metadata(meta: ExtractionMeta, source: str, depth: Optional[int] = None) -> dict[str, Any]: + """Build JSON-serializable Document metadata from the SDK's ExtractionMeta.""" + out: dict[str, Any] = {"source": meta.url or source} + if meta.title is not None: + out["title"] = meta.title + if meta.author is not None: + out["author"] = meta.author + if meta.date is not None: + out["date"] = meta.date + if meta.retrieved_at is not None: + out["retrieved_at"] = meta.retrieved_at.isoformat() + if depth is not None: + out["depth"] = depth + return out + + +class WellMarkedLoader(BaseLoader): + """Load web pages as clean Markdown via the WellMarked API. + + Setup: + Install ``langchain-wellmarked`` and set the ``WELLMARKED_API_KEY`` + environment variable (or pass ``api_key=``). Get a key at + https://wellmarked.io. + + .. code-block:: bash + + pip install langchain-wellmarked + + Instantiate: + .. code-block:: python + + from langchain_wellmarked import WellMarkedLoader + + loader = WellMarkedLoader("https://example.com/article") + + # Or crawl a whole site, one Document per page: + loader = WellMarkedLoader( + "https://docs.example.com", mode="crawl", depth=2 + ) + + Load: + .. code-block:: python + + docs = loader.load() + docs[0].page_content # clean Markdown + docs[0].metadata # {"source": ..., "title": ..., ...} + + In ``crawl`` mode the loader blocks until the crawl job finishes + (up to ``job_timeout`` seconds) and yields only successfully + extracted pages; failed pages (timeouts, robots-disallowed) are + skipped. Crawling requires a Pro plan or above. + """ + + def __init__( + self, + url: str, + *, + api_key: Optional[str] = None, + mode: Literal["extract", "crawl"] = "extract", + depth: int = 1, + render_js: bool = False, + job_timeout: Optional[float] = 300.0, + ) -> None: + """Create the loader. + + Args: + url: The URL to extract (``mode="extract"``) or the root URL + to crawl from (``mode="crawl"``). + api_key: WellMarked API key (``wm_...``). Falls back to the + ``WELLMARKED_API_KEY`` environment variable. + mode: ``"extract"`` loads the single page at ``url``; + ``"crawl"`` BFS-crawls same-site links from ``url``. + depth: Crawl depth (``mode="crawl"`` only). + render_js: Render JS-heavy pages with a headless browser + (Pro plan and above). + job_timeout: Seconds to wait for a crawl job to finish. + ``None`` waits forever. + """ + if mode not in ("extract", "crawl"): + raise ValueError(f"mode must be 'extract' or 'crawl', got {mode!r}") + self.url = url + self.api_key = api_key + self.mode = mode + self.depth = depth + self.render_js = render_js + self.job_timeout = job_timeout + + def lazy_load(self) -> Iterator[Document]: + with WellMarked(api_key=self.api_key) as wm: + if self.mode == "extract": + result = wm.extract(self.url, render_js=self.render_js) + yield Document( + page_content=result.markdown, + metadata=_metadata(result.metadata, self.url), + ) + else: + job = wm.crawl(self.url, depth=self.depth, render_js=self.render_js) + job = wm.wait_for_job(job.job_id, timeout=self.job_timeout) + for page in job.results: + if not page.ok: + continue + assert page.markdown is not None and page.metadata is not None + yield Document( + page_content=page.markdown, + metadata=_metadata(page.metadata, page.url, depth=page.depth), + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f9ab508 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["hatchling>=1.21"] +build-backend = "hatchling.build" + +[project] +name = "langchain-wellmarked" +version = "0.1.0" +description = "Official LangChain integration for the WellMarked API — load any URL as clean Markdown." +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.9" +authors = [ + { name = "WellMarked", email = "support@wellmarked.io" }, +] +keywords = ["wellmarked", "langchain", "document-loader", "markdown", "scraping", "rag", "llm"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +dependencies = [ + "langchain-core>=0.3,<2.0", + "wellmarked>=1.1,<2.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7", +] + +[project.urls] +Homepage = "https://wellmarked.io" +Documentation = "https://wellmarked.io/docs" +Source = "https://github.com/WellMarkedAPI/langchain-wellmarked" +Issues = "https://github.com/WellMarkedAPI/langchain-wellmarked/issues" + +[tool.hatch.build.targets.wheel] +packages = ["langchain_wellmarked"] + +[tool.hatch.build.targets.sdist] +include = [ + "langchain_wellmarked", + "README.md", + "LICENSE", + "pyproject.toml", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc b/tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb752222596959ca651e94e6460ed639ed22101a GIT binary patch literal 14387 zcmeHOTWlLwdOkxA$svcLBwuu~B%8i&7%P%*vED-GN%9e&pH1&bIyOx`ETcFYoaN}@O=7TsW<;I!B~b8&cjQPxEBx@ zn_>zx*%(uVqkSv9b6VT75$re#+^P_g2CK5j$nB zUd>P=(#{lVSD=>H_C|EL&!cuQ&_$Kl07gTPBDb?$Hpqz*zf#DVNmDaSBRO8wlc%(N{>8L@R?8kQrn8!!{P(F3lja#M`SJ_L zl9_x?E11a~NYkL`O%{sDf@YpC>Sy~-qymhLjd1*v|5T8CFCvBBF!&{@a(RU96o{pH$&4$_X442W6H0&>Y9)?3rVkP3oFi)(ey@6trv-JWcnV$kf&XA1Do1W=y(A z6F9|2c_x8jKUFPvIOG<)h{8sMoq=7om9q})LH7fen8RW?kqYXxRjd&2j}=arN?IYC z3RMdr8J(jzQ&HbZXm*swz6A>F7hpl0MW*R#P3?$y02udF5qUr;H^Z1cCEG1s~Mg zL8Ny8qyqe_mZV+CV7%i+gXVL~({=~I1MR{?cwiS$8wO4H6#KOl=O4GCIwlXje$zPO zt|1x~O)($&JDmQDtf5mYb_TsQG+L8jkVL|06piv{L{I`saMY2#Oc4h}yF>{=>e1T4 zgVh(YBAP7{9z-@Khp*z@4`$PlBf+wsw?byl%xhSbW%Eo?x8m3=dQN*w%c^P9 z3hCN8bvst7M7u0oJYN6?=|Wq`ilML!Ij82bsW4yRdJ{gaLrpN9re7z{SyAQiN#*F_ zk;5-wAJ7`mHxpO~P+RGn_;l#&iMp+BCDevM>(Dp|uyirLb|YBX^zB*cz@6sqxz%g$ ztll`cs{P9OYvo_R4 zsc5G2Ry0$bEaf#*%UThgOLikA*7Kg$Ln`Fsbw4sV!}V5Rd_iy7vT0MV&G+rdvsTsd zop*~S%An$4jP8@SSF2_l1wo{nNq9sn=$kU$027+gy&2(khnC2{ggj zC;9=G;k02|@SijM29Pf%bRV{1ut|imz*xZBHpG5x8+^}xu!%$^G*A=o*f_*AJiZzc zXe~*JI$Np_+RHa}vK2p{hVX!nS20(Xn1ynF&~e5zOU85k{gAmvpLQWlv(<-5dl253 zREkB@5(z>s@lCJ=U&Q=zoKMoz$k4X}TZ)5H{=ug}VA$KHUR8|U7^!SJ zJS!c+Dw$i^Nxu~<=i1tT)6zNDv14vc*Idu$J3U(~+rC>-UY+fEZ6O+JU9k{nt!pZ4 z_kZ$I<;2)*i#p#dtxhb+3EIMm+7{+YfR+(G)L$CYJ!0rSf%MK;VDLT0vdHabF9~Oa ze)h8PJ+?QX2t-5L`FAw=Tx7ur67x9&(_)1&pOcV$O{$v7rwt>U%RotTwXtE!RvU3u zg&~3MXlhVh^?dOck~lO-*Zt4?>>c)bn8jPB#Z`ABOc3}6;W~-l6NHysFOes0TQS>ZC{ggD%AH;G{n{S0b6X42I7*3kMbp?FiPl2 zQTc4es;R>Zq7WLm8x}&V2sF)yrBLizXo2B1zbX*=o^W;jJfruu@&bR)OD&+#78qXl z_kcz3GMh>Cp_7=R2vk(DY-o$re+Y$q<`p1QkguNHE;y?LtD%w2HX241{wTg$yQnoh zt_>ISEUgoMS!+cMXhN{f3SzdJp&^5A=$-;`o}N^ATQnFDKHR}fFsL> zYXflW0dd@z5*sCk@n?iNimD`x3Fp?21O{RKp4<7YTKQpN0;{N-d2AXc6?ny=$RZfKO`34eSouX6Id_j)!Ga(dys&j^aVS11X41$_sEG3#C9(VrV1 zZu3$ORuYu>(p5O|^E$3QY7KTST7w~9uN(6&?DY04<8yEkN548ghabUfa$@Qp@VW_7 zBcU|Ie{WS*jJmTkOq!$aaimAT_$B9O%{TL-bd|!RYaIp;CmqatpW8hWB$;@w)sY+% z|MFOSpc^3wLFqY_@yX#Q+5^hUCz6AYC$rtLI(y9-)nm%+CG{Z}S1PNP-P^7s-M@Gb zMwHb`+X$jli*}~;1hP(YM&i6{vJQJtU#YD9W`6Cp{Qq=*Ww;LO9&|b0na{=RPwTLx z5le}jpHwOQ2BVqujuk9j+-3#yT7j_bcczTk4{Atq zKmXjel<^ehY)=`f*xSkdcJA(!fjYY$QU|nsrSt?W+9INN1+%{1!@foJoY>K#I&g-` zK-Av=xGq?MTmfHyrq(Tr^b-24(|I7Z&SN}%Jfa$V8}2Wp5!1=|Kay(IBY4B5L%35{ z35^{nU8EUN(>k0Rs-)^#NvB{+TGy;Niq+HQT;9wT3@e;3g2J#mRO3vsoX>Jnbs}Fp zoz8O?4UtPDJ+4s;R^lL^JVf0OlhbXjq<{b%NJJJ(h|9vaDi*A`pI_X)*Po$oMRP{c zCig-(si>Z&GMV~huZ$i3#IydyE<0hjG2l<^@sn-ym+bZ@1`P@v*)W~r~R81J1_5XQ!et;oi2J)?DVDFY&3M)bWB*n$kCB6soJyH4&rS`oKw`6(I7-s#yEYAV=P3CvW+1m*6(9?U3K}QYb)ME}b8l6;Cd8!D+K_ z7rdJ<`7h#vBd!pG8}7QY!#+n0t@Gzx%ZR{DzzvSER5Jlx=VzD%f87IDTpkI)1dN}e zv#8n^5cKqFHFU&1my?{~QP-UvZs;^Q&VbKt_u8jeSKZUd?RG|0GZDP{4g2Tu$}zZL z0+Q3dOg+yvt}_9HN<@)Hf^{aKI7(+r{PDHiz)^*}>&Ez<9>D~R!Ubb60oN;0mEGX!z~bYRHU+51SG)QgmV4j>ihicQxkgV*EdEe0h` z0(4lbmkGQ@;5>ng1TF#KMBp95|AfHX1l}d^9soZP;5xuo02`JnfD)-91FEog_N8A& zJqr36e+~fYFRrSre)>jsR@_?^4@`>(X2nCF$vZ3JvA_Fab>P_az%if|Ao8)QSo_T& zM~1Mc|F|LEw^{QP}O4hO{6242BE6Q-@A=M zRWT=gqI7;Uzu3=szJzT48qe=&r#(7`GXY0-mx2g?XGX$2{OxL7H~j6b)W$k@4cEC{ zr*`{vF0QwH1$xvCE4b;G@N;qM5AK^ugG9$%(33JH-+3Bn%OvLyW7I_!p9oz;jSd{Aj! zx(aLIZ;uO)T7%t-*5DKAj%9iG#rV8fc7Jtz#xcV~c})jxbq{#G0^x6j5tY?rtUvfA z`QGyl-<23_-nII-!DxQs{9BvjCl1gpH;N$W{m8Loy46yyTh#zVL>tj?*(HkBQ z(KCc)31|ck5Ev)GV`B#iqfg+3c01v`Cr5xC@!8Koeh{wjAc7wgc#}Y$z$AeJK&tu6 z7}<7*e?X1VEsk0+=b?tT{zIaiBJd-CM!}dJ+!p88aTbSS&Z-nuGH@3ms~Tr>B|{xg z=kjIT3etauwoXxd|IFP~L}uos4cFQ~7Ctd?B`O~NZ8-L$?7QRhAr^|xODw*(A}YTu zRG(L-pI3lZfXGT!to>$?BR^*+Z~1VS;5M)Akl*xDC_q~-ogV?!ocwe}JoJk~weQe$ z-yxtCAo8KASo_T&M}FE)-tyrt!EIjKA;0ORP=K~vIzIxcJ74dR)K(`Xwe{*?MFg0Z zw^qg4Z${k8LsFEyC2pmV6rr=eY~WW1-3opLC2mMc^g>cVc}S|4LQ-u6TIVAo1))#^ zKtU*xf>1~<5cs`>AXF3xk1XMR*U6oyTna+rg-SLl&vZx^=knG$bsgZ}rtjRR7J1=4 zYHz7)7r{E`A(_7fvfM2)e=ve>O*nTZ_6n*{4wQqcdn;ib^IgA$dx&e6*h};vfD@vB zzKbHEjV#HwGRM@J0A)AJ;p%%2sOg^qioL-emJ?FpOL3Bi$;q-|CQoa)Hxfd)ICDmS zA8lLW>2m%of&@Q70=G@9upPIy!g@Lf|J#a<|5ed@vCuIqW`eJS-t{<_k`V;@WYK_NO21OP7Q`*5Jt5J`P>ZCW`TE zDe|HtXIzsRb?we!*IRaa{g7)Z-j^Ci#Jn*qYnL~)m6z6s?Mrujz3(=x=kxvjwq9e} zhx^lrr2U`!(-Fs?Zsc!aJH&BFiWOr&4WlSqYEhK)?xuRYS#KJNAOe~=b=v2RNy~E5 z^$c(k<_~9ud>v<<4#9OL0&B3DOElYzBd>*yDR<~3f_#xqpAks5SgLmLx5GK@1L7T3 zI;VZkIVFw){gG0?rPsuzqZQ4kFJ0t+_Ruh1S{Ewi;ZrwWYKa9Lfu$rINSgk`wCy~C z>d)EfMN(}`kD{vv?cT-bYHeINyIoRqhDr(jBT$kNYRHhEB$_)bYx-xR+p5v$rlZfz zME6a>X6bayg3VWVR78Mjd2>~){bt0?+$`{N?4qCIu&tJNQ{3#NZu_WbsVSGpkD$UG zQM$b1y`edIS4BMj%S3hX`1Ig$pcNqU@v2z+%^*kKWhZa>aF^gVukDcE^in85TP~d+ znH7(7y#}vh7`(A0=X{n2xUEjr){K4j5h+et`*iKRo-;Ko>Soa$d7h{97&&PtowT_A z7t~_R`)e8dLNrMvek+>)^@6q8{ks~~qwK5yV;ZZaK1Y9*T3^+uw!JEYWi!JMfcc`d zBICIW_yZ-4-woxbsyw*Pt$41s`2w-;3R6hJ*7VXom=U-m{Jj1*$fKbdwCv`Cf*|~w zo%oco_gIyE>pxi2r|jqqJNln&=fAS;{}x~M!O1JzubjTR;cEI?>y5#`-}krsX5$B@ zqW8nSLhN$}Fh3@+fb{;_ch+7JXN8{60|8<0y&wy$ohR&WEH1nv++760B_NC_k1+eOdga7~l literal 0 HcmV?d00001 diff --git a/tests/test_document_loaders.py b/tests/test_document_loaders.py new file mode 100644 index 0000000..7b07388 --- /dev/null +++ b/tests/test_document_loaders.py @@ -0,0 +1,112 @@ +"""Unit tests for WellMarkedLoader — the SDK client is faked, no network.""" +from datetime import datetime, timezone + +import pytest +from wellmarked import CrawlItem, CrawlJob, ExtractionMeta, ExtractResult + +import langchain_wellmarked.document_loaders as mod +from langchain_wellmarked import WellMarkedLoader + +RETRIEVED = datetime(2026, 6, 10, 12, 0, 0, tzinfo=timezone.utc) + + +class FakeWellMarked: + """Stands in for wellmarked.WellMarked. Records calls, returns canned data.""" + + calls: list = [] + + def __init__(self, api_key=None, **kwargs): + FakeWellMarked.calls.append(("init", api_key)) + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + pass + + def extract(self, url, *, render_js=False): + FakeWellMarked.calls.append(("extract", url, render_js)) + return ExtractResult( + markdown="# Hello", + metadata=ExtractionMeta( + url=url, title="Hello", author="Ada", retrieved_at=RETRIEVED + ), + request_id="req_1", + ) + + def crawl(self, url, *, depth=1, render_js=False, **kwargs): + FakeWellMarked.calls.append(("crawl", url, depth, render_js)) + return CrawlJob(job_id="job_1", status="queued", total=0, completed=0, results=[]) + + def wait_for_job(self, job_id, *, timeout=300.0, **kwargs): + FakeWellMarked.calls.append(("wait_for_job", job_id, timeout)) + return CrawlJob( + job_id=job_id, + status="done", + total=2, + completed=2, + results=[ + CrawlItem( + url="https://docs.example.com", + depth=0, + markdown="# Root", + metadata=ExtractionMeta(url="https://docs.example.com", title="Root"), + ), + CrawlItem(url="https://docs.example.com/dead", depth=1, error="target_timeout"), + ], + ) + + +@pytest.fixture(autouse=True) +def fake_client(monkeypatch): + FakeWellMarked.calls = [] + monkeypatch.setattr(mod, "WellMarked", FakeWellMarked) + + +def test_extract_mode_yields_one_document(): + docs = WellMarkedLoader("https://example.com/article", api_key="wm_test").load() + + assert len(docs) == 1 + assert docs[0].page_content == "# Hello" + assert docs[0].metadata == { + "source": "https://example.com/article", + "title": "Hello", + "author": "Ada", + "retrieved_at": RETRIEVED.isoformat(), + } + assert ("init", "wm_test") in FakeWellMarked.calls + assert ("extract", "https://example.com/article", False) in FakeWellMarked.calls + + +def test_extract_mode_passes_render_js(): + WellMarkedLoader("https://example.com", render_js=True).load() + + assert ("extract", "https://example.com", True) in FakeWellMarked.calls + + +def test_crawl_mode_yields_ok_pages_and_skips_failures(): + docs = WellMarkedLoader( + "https://docs.example.com", mode="crawl", depth=2, job_timeout=60.0 + ).load() + + assert ("crawl", "https://docs.example.com", 2, False) in FakeWellMarked.calls + assert ("wait_for_job", "job_1", 60.0) in FakeWellMarked.calls + assert len(docs) == 1 # the failed page is skipped + assert docs[0].page_content == "# Root" + assert docs[0].metadata == { + "source": "https://docs.example.com", + "title": "Root", + "depth": 0, + } + + +def test_invalid_mode_raises(): + with pytest.raises(ValueError, match="mode must be"): + WellMarkedLoader("https://example.com", mode="bulk") + + +def test_lazy_load_is_lazy(): + it = WellMarkedLoader("https://example.com").lazy_load() + assert FakeWellMarked.calls == [] # nothing happens until iteration + next(it) + assert ("extract", "https://example.com", False) in FakeWellMarked.calls From 47ee6aa044cf3a6511d036a0e96033fd464ace0e Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 10 Jun 2026 18:41:54 -0400 Subject: [PATCH 2/3] Add .gitignore; drop committed __pycache__ Co-Authored-By: Claude Fable 5 --- .gitignore | 5 +++++ ...ocument_loaders.cpython-314-pytest-9.0.3.pyc | Bin 14387 -> 0 bytes 2 files changed, 5 insertions(+) create mode 100644 .gitignore delete mode 100644 tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0d2085 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.venv/ +dist/ +*.egg-info/ diff --git a/tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc b/tests/__pycache__/test_document_loaders.cpython-314-pytest-9.0.3.pyc deleted file mode 100644 index fb752222596959ca651e94e6460ed639ed22101a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14387 zcmeHOTWlLwdOkxA$svcLBwuu~B%8i&7%P%*vED-GN%9e&pH1&bIyOx`ETcFYoaN}@O=7TsW<;I!B~b8&cjQPxEBx@ zn_>zx*%(uVqkSv9b6VT75$re#+^P_g2CK5j$nB zUd>P=(#{lVSD=>H_C|EL&!cuQ&_$Kl07gTPBDb?$Hpqz*zf#DVNmDaSBRO8wlc%(N{>8L@R?8kQrn8!!{P(F3lja#M`SJ_L zl9_x?E11a~NYkL`O%{sDf@YpC>Sy~-qymhLjd1*v|5T8CFCvBBF!&{@a(RU96o{pH$&4$_X442W6H0&>Y9)?3rVkP3oFi)(ey@6trv-JWcnV$kf&XA1Do1W=y(A z6F9|2c_x8jKUFPvIOG<)h{8sMoq=7om9q})LH7fen8RW?kqYXxRjd&2j}=arN?IYC z3RMdr8J(jzQ&HbZXm*swz6A>F7hpl0MW*R#P3?$y02udF5qUr;H^Z1cCEG1s~Mg zL8Ny8qyqe_mZV+CV7%i+gXVL~({=~I1MR{?cwiS$8wO4H6#KOl=O4GCIwlXje$zPO zt|1x~O)($&JDmQDtf5mYb_TsQG+L8jkVL|06piv{L{I`saMY2#Oc4h}yF>{=>e1T4 zgVh(YBAP7{9z-@Khp*z@4`$PlBf+wsw?byl%xhSbW%Eo?x8m3=dQN*w%c^P9 z3hCN8bvst7M7u0oJYN6?=|Wq`ilML!Ij82bsW4yRdJ{gaLrpN9re7z{SyAQiN#*F_ zk;5-wAJ7`mHxpO~P+RGn_;l#&iMp+BCDevM>(Dp|uyirLb|YBX^zB*cz@6sqxz%g$ ztll`cs{P9OYvo_R4 zsc5G2Ry0$bEaf#*%UThgOLikA*7Kg$Ln`Fsbw4sV!}V5Rd_iy7vT0MV&G+rdvsTsd zop*~S%An$4jP8@SSF2_l1wo{nNq9sn=$kU$027+gy&2(khnC2{ggj zC;9=G;k02|@SijM29Pf%bRV{1ut|imz*xZBHpG5x8+^}xu!%$^G*A=o*f_*AJiZzc zXe~*JI$Np_+RHa}vK2p{hVX!nS20(Xn1ynF&~e5zOU85k{gAmvpLQWlv(<-5dl253 zREkB@5(z>s@lCJ=U&Q=zoKMoz$k4X}TZ)5H{=ug}VA$KHUR8|U7^!SJ zJS!c+Dw$i^Nxu~<=i1tT)6zNDv14vc*Idu$J3U(~+rC>-UY+fEZ6O+JU9k{nt!pZ4 z_kZ$I<;2)*i#p#dtxhb+3EIMm+7{+YfR+(G)L$CYJ!0rSf%MK;VDLT0vdHabF9~Oa ze)h8PJ+?QX2t-5L`FAw=Tx7ur67x9&(_)1&pOcV$O{$v7rwt>U%RotTwXtE!RvU3u zg&~3MXlhVh^?dOck~lO-*Zt4?>>c)bn8jPB#Z`ABOc3}6;W~-l6NHysFOes0TQS>ZC{ggD%AH;G{n{S0b6X42I7*3kMbp?FiPl2 zQTc4es;R>Zq7WLm8x}&V2sF)yrBLizXo2B1zbX*=o^W;jJfruu@&bR)OD&+#78qXl z_kcz3GMh>Cp_7=R2vk(DY-o$re+Y$q<`p1QkguNHE;y?LtD%w2HX241{wTg$yQnoh zt_>ISEUgoMS!+cMXhN{f3SzdJp&^5A=$-;`o}N^ATQnFDKHR}fFsL> zYXflW0dd@z5*sCk@n?iNimD`x3Fp?21O{RKp4<7YTKQpN0;{N-d2AXc6?ny=$RZfKO`34eSouX6Id_j)!Ga(dys&j^aVS11X41$_sEG3#C9(VrV1 zZu3$ORuYu>(p5O|^E$3QY7KTST7w~9uN(6&?DY04<8yEkN548ghabUfa$@Qp@VW_7 zBcU|Ie{WS*jJmTkOq!$aaimAT_$B9O%{TL-bd|!RYaIp;CmqatpW8hWB$;@w)sY+% z|MFOSpc^3wLFqY_@yX#Q+5^hUCz6AYC$rtLI(y9-)nm%+CG{Z}S1PNP-P^7s-M@Gb zMwHb`+X$jli*}~;1hP(YM&i6{vJQJtU#YD9W`6Cp{Qq=*Ww;LO9&|b0na{=RPwTLx z5le}jpHwOQ2BVqujuk9j+-3#yT7j_bcczTk4{Atq zKmXjel<^ehY)=`f*xSkdcJA(!fjYY$QU|nsrSt?W+9INN1+%{1!@foJoY>K#I&g-` zK-Av=xGq?MTmfHyrq(Tr^b-24(|I7Z&SN}%Jfa$V8}2Wp5!1=|Kay(IBY4B5L%35{ z35^{nU8EUN(>k0Rs-)^#NvB{+TGy;Niq+HQT;9wT3@e;3g2J#mRO3vsoX>Jnbs}Fp zoz8O?4UtPDJ+4s;R^lL^JVf0OlhbXjq<{b%NJJJ(h|9vaDi*A`pI_X)*Po$oMRP{c zCig-(si>Z&GMV~huZ$i3#IydyE<0hjG2l<^@sn-ym+bZ@1`P@v*)W~r~R81J1_5XQ!et;oi2J)?DVDFY&3M)bWB*n$kCB6soJyH4&rS`oKw`6(I7-s#yEYAV=P3CvW+1m*6(9?U3K}QYb)ME}b8l6;Cd8!D+K_ z7rdJ<`7h#vBd!pG8}7QY!#+n0t@Gzx%ZR{DzzvSER5Jlx=VzD%f87IDTpkI)1dN}e zv#8n^5cKqFHFU&1my?{~QP-UvZs;^Q&VbKt_u8jeSKZUd?RG|0GZDP{4g2Tu$}zZL z0+Q3dOg+yvt}_9HN<@)Hf^{aKI7(+r{PDHiz)^*}>&Ez<9>D~R!Ubb60oN;0mEGX!z~bYRHU+51SG)QgmV4j>ihicQxkgV*EdEe0h` z0(4lbmkGQ@;5>ng1TF#KMBp95|AfHX1l}d^9soZP;5xuo02`JnfD)-91FEog_N8A& zJqr36e+~fYFRrSre)>jsR@_?^4@`>(X2nCF$vZ3JvA_Fab>P_az%if|Ao8)QSo_T& zM~1Mc|F|LEw^{QP}O4hO{6242BE6Q-@A=M zRWT=gqI7;Uzu3=szJzT48qe=&r#(7`GXY0-mx2g?XGX$2{OxL7H~j6b)W$k@4cEC{ zr*`{vF0QwH1$xvCE4b;G@N;qM5AK^ugG9$%(33JH-+3Bn%OvLyW7I_!p9oz;jSd{Aj! zx(aLIZ;uO)T7%t-*5DKAj%9iG#rV8fc7Jtz#xcV~c})jxbq{#G0^x6j5tY?rtUvfA z`QGyl-<23_-nII-!DxQs{9BvjCl1gpH;N$W{m8Loy46yyTh#zVL>tj?*(HkBQ z(KCc)31|ck5Ev)GV`B#iqfg+3c01v`Cr5xC@!8Koeh{wjAc7wgc#}Y$z$AeJK&tu6 z7}<7*e?X1VEsk0+=b?tT{zIaiBJd-CM!}dJ+!p88aTbSS&Z-nuGH@3ms~Tr>B|{xg z=kjIT3etauwoXxd|IFP~L}uos4cFQ~7Ctd?B`O~NZ8-L$?7QRhAr^|xODw*(A}YTu zRG(L-pI3lZfXGT!to>$?BR^*+Z~1VS;5M)Akl*xDC_q~-ogV?!ocwe}JoJk~weQe$ z-yxtCAo8KASo_T&M}FE)-tyrt!EIjKA;0ORP=K~vIzIxcJ74dR)K(`Xwe{*?MFg0Z zw^qg4Z${k8LsFEyC2pmV6rr=eY~WW1-3opLC2mMc^g>cVc}S|4LQ-u6TIVAo1))#^ zKtU*xf>1~<5cs`>AXF3xk1XMR*U6oyTna+rg-SLl&vZx^=knG$bsgZ}rtjRR7J1=4 zYHz7)7r{E`A(_7fvfM2)e=ve>O*nTZ_6n*{4wQqcdn;ib^IgA$dx&e6*h};vfD@vB zzKbHEjV#HwGRM@J0A)AJ;p%%2sOg^qioL-emJ?FpOL3Bi$;q-|CQoa)Hxfd)ICDmS zA8lLW>2m%of&@Q70=G@9upPIy!g@Lf|J#a<|5ed@vCuIqW`eJS-t{<_k`V;@WYK_NO21OP7Q`*5Jt5J`P>ZCW`TE zDe|HtXIzsRb?we!*IRaa{g7)Z-j^Ci#Jn*qYnL~)m6z6s?Mrujz3(=x=kxvjwq9e} zhx^lrr2U`!(-Fs?Zsc!aJH&BFiWOr&4WlSqYEhK)?xuRYS#KJNAOe~=b=v2RNy~E5 z^$c(k<_~9ud>v<<4#9OL0&B3DOElYzBd>*yDR<~3f_#xqpAks5SgLmLx5GK@1L7T3 zI;VZkIVFw){gG0?rPsuzqZQ4kFJ0t+_Ruh1S{Ewi;ZrwWYKa9Lfu$rINSgk`wCy~C z>d)EfMN(}`kD{vv?cT-bYHeINyIoRqhDr(jBT$kNYRHhEB$_)bYx-xR+p5v$rlZfz zME6a>X6bayg3VWVR78Mjd2>~){bt0?+$`{N?4qCIu&tJNQ{3#NZu_WbsVSGpkD$UG zQM$b1y`edIS4BMj%S3hX`1Ig$pcNqU@v2z+%^*kKWhZa>aF^gVukDcE^in85TP~d+ znH7(7y#}vh7`(A0=X{n2xUEjr){K4j5h+et`*iKRo-;Ko>Soa$d7h{97&&PtowT_A z7t~_R`)e8dLNrMvek+>)^@6q8{ks~~qwK5yV;ZZaK1Y9*T3^+uw!JEYWi!JMfcc`d zBICIW_yZ-4-woxbsyw*Pt$41s`2w-;3R6hJ*7VXom=U-m{Jj1*$fKbdwCv`Cf*|~w zo%oco_gIyE>pxi2r|jqqJNln&=fAS;{}x~M!O1JzubjTR;cEI?>y5#`-}krsX5$B@ zqW8nSLhN$}Fh3@+fb{;_ch+7JXN8{60|8<0y&wy$ohR&WEH1nv++760B_NC_k1+eOdga7~l From cb28b4719db9809173c5f4d59a0b457e3b86b6ee Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 10 Jun 2026 18:52:41 -0400 Subject: [PATCH 3/3] Add CI and PyPI release workflows CI runs pytest across Python 3.9-3.14 on pushes to main and PRs. Release publishes to PyPI via OIDC Trusted Publishing on a GitHub Release, mirroring the Python-SDK workflows. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++ .github/workflows/release.yml | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7759c8e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + cache: pip + - run: python -m pip install --upgrade pip + - run: pip install -e ".[dev]" + - name: Test (pytest) + run: pytest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ab3f60d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release + +# Publishes to PyPI via Trusted Publishing (OIDC) on a GitHub Release. +# One-time setup: on pypi.org add a Trusted Publisher for this repo + +# workflow (langchain-wellmarked / release.yml / environment: pypi). +# No API token. +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + - run: python -m pip install --upgrade pip build + - run: python -m build + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for Trusted Publishing + steps: + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1