From 75f8e9e56c6de5b7e726b88c35fea4b0e968d617 Mon Sep 17 00:00:00 2001 From: jbutler Date: Tue, 14 Jul 2026 12:00:06 -0400 Subject: [PATCH] fix: data-quality improvements for EPSS precision, stats totals, and KB known-issues retrieval - Round EPSS score/percentile to the source's 5-decimal precision once at parse time so no output view can surface float-repr artifacts. - Add explicit Unrated / Unspecified buckets to the include_stats severity and impact breakdowns so bucket counts always sum to the reported total (entries like Chromium CVEs ship without a severity rating). - Known-issues fetch: support.microsoft.com moved KB articles to /servicing/os/... pages, adding a second redirect hop and a new markup generation (details/summary blocks, strong labels under an h3 heading). Follow a bounded, validated redirect chain (MAX_REDIRECT_HOPS = 3) and parse both markup generations; the honest three-way status contract (published / none_published / unavailable) is unchanged. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 4 +- src/patch_tuesday_mcp/feeds/enrichment.py | 7 +- src/patch_tuesday_mcp/feeds/known_issues.py | 121 +++++++++++++----- src/patch_tuesday_mcp/models/vulnerability.py | 14 ++ tests/fixtures/kb_servicing_layout.html | 41 ++++++ tests/test_enrichment.py | 21 +++ tests/test_known_issues.py | 113 ++++++++++++++-- tests/test_models.py | 25 ++++ 8 files changed, 296 insertions(+), 50 deletions(-) create mode 100644 tests/fixtures/kb_servicing_layout.html diff --git a/CLAUDE.md b/CLAUDE.md index 587202c..625ed5f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,10 +29,10 @@ On Windows in this repo use `.venv/Scripts/python -m pytest` etc. — the venv w - `tools/formatters.py` — optional `format="markdown"|"csv"` triage renderings; JSON is always included and unchanged. - `tools/profiles.py` — named product watchlists for `msrc_search` (`product_profile=`, plus ad-hoc `products=[...]`/`product_families=[...]`). Built-ins (`identity-core`, `endpoint`, `server-infrastructure`) merged under a `MSRC_PROFILES_PATH` JSON override with strict validation; unknown/invalid → `invalid_input`, never an unscoped fallback. Matching is local substring (union: any product OR family); profile contents are never sent upstream or to telemetry. - `tools/prompts.py` — the `monthly_triage` MCP prompt (registered in `server.py` via `mcp.prompt`), a guided analyst workflow built entirely on `msrc_search`; optional `product_profile`/`month` args. Portable copies live in `prompts/` (plain-text prompt) and `skills/patch-tuesday-triage/` (agent skill) — keep them in sync with `tools/prompts.py` when the workflow changes. -- `feeds/http_client.py` — shared httpx client: `follow_redirects=False` (hardcoded hosts; redirects are never followed automatically), `get_bounded()` which streams responses with a byte cap instead of buffering unbounded bodies, and `get_location()` which returns a redirect's status+Location without reading the body so a caller can follow one hop manually after validating the target. +- `feeds/http_client.py` — shared httpx client: `follow_redirects=False` (hardcoded hosts; redirects are never followed automatically), `get_bounded()` which streams responses with a byte cap instead of buffering unbounded bodies, and `get_location()` which returns a redirect's status+Location without reading the body so a caller can follow hops manually after validating each target. - `feeds/msrc_api.py` — MSRC index + monthly CVRF fetch with in-process TTL caches (`MAX_FULL_MONTHS_CACHED = 12` — matches `MAX_TREND_MONTHS` so a max trend doesn't evict its own months; 40 slim), LRU eviction (hits refresh recency), per-month asyncio locks, `FETCH_CONCURRENCY = 3` semaphore, `force_refresh` bypass, freshness metadata. Response bodies capped via `MCP_MSRC_MAX_RESPONSE_BYTES` (64 MiB default). - `feeds/enrichment.py` — KEV catalog + EPSS fetches (batches of 100, fetched concurrently with `EPSS_FETCH_CONCURRENCY = 3`), cached; EPSS cache capped at `MAX_EPSS_CACHE_ENTRIES = 50_000`; failures return empty ({}) — enrichment must never break a search. Bodies capped via `MCP_ENRICHMENT_MAX_RESPONSE_BYTES` (32 MiB default). -- `feeds/known_issues.py` — Microsoft-confirmed known issues per KB (`include_known_issues=True`), scraped best-effort from the public support.microsoft.com KB page (no keyless API exists; the Graph windowsupdates API needs AAD). Honest three-way status: `published` / `none_published` / `unavailable` — a fetch/parse failure is never reported as "none". Coverage boundary: known-issues sections exist mainly for Windows OS cumulative/preview updates; Office/SharePoint/SQL/.NET pages usually have none → `none_published`. Source quirks handled: `/help/{kb}` answers one same-host redirect (followed manually via `get_location()` after validating the target host), and bogus KB numbers redirect to *unrelated* articles that still echo the requested id in an analytics meta tag, so a landing page is only trusted if its URL slug or title names the requested KB (pages that never self-reference, like some .NET update pages, conservatively report no per-KB page). TTL cache 6 h (`MAX_CACHE_ENTRIES = 500`, LRU; failures aren't cached), `FETCH_CONCURRENCY = 3` semaphore shared by batch prefetch, bodies capped via `MCP_KNOWN_ISSUES_MAX_RESPONSE_BYTES` (4 MiB default). Parser is stdlib-only (regex slicing + `html.parser`); on layout drift it degrades to `unavailable` with the source URL — the `--run-live` smoke test is the drift canary. +- `feeds/known_issues.py` — Microsoft-confirmed known issues per KB (`include_known_issues=True`), scraped best-effort from the public support.microsoft.com KB page (no keyless API exists; the Graph windowsupdates API needs AAD). Honest three-way status: `published` / `none_published` / `unavailable` — a fetch/parse failure is never reported as "none". Coverage boundary: known-issues sections exist mainly for Windows OS cumulative/preview updates; Office/SharePoint/SQL/.NET pages usually have none → `none_published`. Source quirks handled: `/help/{kb}` answers a short same-host redirect chain (bounded at `MAX_REDIRECT_HOPS = 3`; each hop followed manually via `get_location()` after validating the target host — the 2026 site migration added a second hop to `/servicing/os/...`), two markup generations are parsed (legacy `ocpSection` divs with `` labels, and the newer `
/` blocks with `` labels under an `

` heading), and bogus KB numbers redirect to *unrelated* articles that still echo the requested id in an analytics meta tag, so a landing page is only trusted if its URL slug or title names the requested KB (pages that never self-reference, like some .NET update pages, conservatively report no per-KB page). TTL cache 6 h (`MAX_CACHE_ENTRIES = 500`, LRU; failures aren't cached), `FETCH_CONCURRENCY = 3` semaphore shared by batch prefetch, bodies capped via `MCP_KNOWN_ISSUES_MAX_RESPONSE_BYTES` (4 MiB default). Parser is stdlib-only (regex slicing + `html.parser`); on layout drift it degrades to `unavailable` with the source URL — the `--run-live` smoke test is the drift canary. - `models/vulnerability.py` — CVRF parsing into `Vulnerability`; numeric CVRF enums are documented constants (remediation types: 0=workaround, 1=mitigation, 2=vendor fix/KB, 4=will-not-fix). `to_summary_dict()` vs `to_detail_dict()` control output size; new fields are opt-in flags (`include_references`, `include_kb_details`, `include_kev_details`, `include_temporal`, filter-triggered `cwe`/`exploitation_assessment`). - `models/cvss.py` — lenient CVSS v3.x vector parser; fails open to `None`, never raises. - `middleware/` — per-IP token-bucket rate limit and request body cap, both with telemetry callbacks (`on_request`/`on_throttled`, `on_rejected`). X-Forwarded-For is honored only from private/loopback peers or `MCP_TRUSTED_PROXIES` members — a public direct peer can never forge it. diff --git a/src/patch_tuesday_mcp/feeds/enrichment.py b/src/patch_tuesday_mcp/feeds/enrichment.py index 9fbcebc..5751659 100644 --- a/src/patch_tuesday_mcp/feeds/enrichment.py +++ b/src/patch_tuesday_mcp/feeds/enrichment.py @@ -187,8 +187,11 @@ async def fetch_batch(batch: list[str]) -> tuple[list[str], dict | None]: for entry in data.get("data", []): cve = entry.get("cve") try: - score = float(entry["epss"]) - percentile = float(entry["percentile"]) + # EPSS publishes decimal strings at 5-significant-decimal + # precision; round once here so no downstream view can + # surface float-repr digits beyond what the source states. + score = round(float(entry["epss"]), 5) + percentile = round(float(entry["percentile"]), 5) except (KeyError, TypeError, ValueError): continue if cve: diff --git a/src/patch_tuesday_mcp/feeds/known_issues.py b/src/patch_tuesday_mcp/feeds/known_issues.py index 8863e07..64c5265 100644 --- a/src/patch_tuesday_mcp/feeds/known_issues.py +++ b/src/patch_tuesday_mcp/feeds/known_issues.py @@ -16,12 +16,18 @@ masquerade as "no known issues". Fetch failures never raise into a search; ``fetch_known_issues`` always -returns a dict. Two quirks of the source shape this module: the /help/ -URL answers with a single same-host redirect to the canonical article (we -follow exactly one validated hop; the shared client never follows redirects -on its own), and unknown KB numbers redirect to *unrelated* articles — which +returns a dict. Quirks of the source that shape this module: the /help/ +URL answers with a short chain of same-host redirects to the canonical +article (historically one hop to /topic/...; the 2026 site migration added a +second hop to /servicing/os/...), so we follow a bounded number of hops, +validating each target ourselves — the shared client never follows redirects +on its own. Unknown KB numbers redirect to *unrelated* articles — which still echo the requested id in an analytics meta tag — so a landing page is -only trusted when its URL slug or its title names the requested KB. +only trusted when its URL slug or its title names the requested KB. Two +markup generations coexist: legacy pages use ``ocpSection``/``ocpExpando`` +divs with ```` segment labels, while the newer +/servicing/ pages use ``
``/```` blocks with ```` +labels under an ``

`` heading; both are parsed. """ import asyncio @@ -46,6 +52,7 @@ KNOWN_ISSUES_TTL_SECONDS = 6 * 3600 MAX_CACHE_ENTRIES = 500 # parsed results are ~1-4 KB, so worst case ~2 MB FETCH_CONCURRENCY = 3 +MAX_REDIRECT_HOPS = 3 # /help -> /topic -> /servicing today; one spare # A known-issues section with no parseable issue entries is an explicit # "none" statement when its prose is short; anything longer suggests issue @@ -93,23 +100,36 @@ def _elapsed_ms(start: float) -> float: # --- HTML parsing (stdlib only; the source publishes no structured format) --- -# The section heading as observed on Windows/.NET update pages; tolerate the -# small wording variants Microsoft has used over the years. +# The section heading across both markup generations: legacy pages carry the +# text directly in an

, the newer /servicing/ pages wrap it in inline +# tags (e.g. ) under an

. Tolerate the small wording variants +# Microsoft has used over the years. _HEADING_RE = re.compile( - r"]*>[^<]*known issues (?:in|with) this (?:security )?update", re.IGNORECASE + r"[23])[^>]*>\s*(?:<[a-z][^>]*>\s*)*[^<]*" + r"known issues (?:in|with) this (?:security )?update", + re.IGNORECASE, ) -# The known-issues section ends where the next top-level section or heading -# starts; issue entries inside it are div-based, never
. +# Legacy layout: the section ends at the next top-level section or

; +# issue entries inside it are div-based, never
, and legacy issue +# bodies may legitimately contain

tags. _REGION_END_RE = re.compile(r"]| section ends at the next heading of either +# level (the page repeats the section under a second

for the combined +# SSU+LCU package — only the first is parsed). +_REGION_END_NEW_RE = re.compile(r"]", re.IGNORECASE) _ISSUE_BLOCK_RE = re.compile(r'
', re.IGNORECASE) _ISSUE_TITLE_RE = re.compile( r'class="ocpExpandoHeadTitleContainer"[^>]*>(?P.*?)</div>', re.DOTALL ) _ISSUE_BODY_RE = re.compile(r'class="ocpExpandoBody"[^>]*>', re.IGNORECASE) -# Bold paragraph labels segmenting an issue body. +# /servicing/ layout: one collapsible <details> element per issue. +_DETAILS_RE = re.compile(r"<details[\s>].*?</details>", re.IGNORECASE | re.DOTALL) +_SUMMARY_RE = re.compile(r"<summary[^>]*>(?P<title>.*?)</summary>", re.IGNORECASE | re.DOTALL) +# Bold paragraph labels segmenting an issue body (legacy <b>, servicing <strong>). _MARKER_RE = re.compile( - r'<b class="ocpLegacyBold">[^<]*?(?P<label>symptoms?|workaround|next steps?|resolution)' - r"[^<]*?</b>", + r'<(?:b class="ocpLegacyBold"|strong)[^>]*>[^<]*?' + r"(?P<label>symptoms?|workaround|next steps?|resolution)" + r"[^<]*?</(?:b|strong)>", re.IGNORECASE, ) _RESOLVED_BY_RE = re.compile( @@ -168,7 +188,17 @@ def _segment_issue_body(body_html: str) -> dict[str, str]: return {field: " ".join(parts) for field, parts in segments.items()} +def _attach_resolved_by(issue: dict) -> dict: + resolved = _RESOLVED_BY_RE.search( + f"{issue.get('workaround', '')} {issue.get('resolution', '')}" + ) + if resolved: + issue["resolved_by"] = f"KB{resolved.group(1)}" + return issue + + def _parse_issue_block(block_html: str) -> dict | None: + """Parse a legacy ocpSection issue block.""" title_match = _ISSUE_TITLE_RE.search(block_html) if not title_match: return None @@ -180,13 +210,21 @@ def _parse_issue_block(block_html: str) -> dict | None: body_match = _ISSUE_BODY_RE.search(block_html) if body_match: issue.update(_segment_issue_body(block_html[body_match.end() :])) + return _attach_resolved_by(issue) - resolved = _RESOLVED_BY_RE.search( - f"{issue.get('workaround', '')} {issue.get('resolution', '')}" - ) - if resolved: - issue["resolved_by"] = f"KB{resolved.group(1)}" - return issue + +def _parse_details_block(block_html: str) -> dict | None: + """Parse a /servicing/-layout <details> issue block.""" + title_match = _SUMMARY_RE.search(block_html) + if not title_match: + return None + title = _text(title_match.group("title")) + if not title: + return None + + issue: dict = {"title": title} + issue.update(_segment_issue_body(block_html[title_match.end() :])) + return _attach_resolved_by(issue) def _parse_known_issues(html: str, kb_number: str, source_url: str) -> dict: @@ -207,7 +245,8 @@ def _parse_known_issues(html: str, kb_number: str, source_url: str) -> dict: } tail = html[heading.end() :] - region_end = _REGION_END_RE.search(tail) + end_re = _REGION_END_RE if heading.group("level") == "2" else _REGION_END_NEW_RE + region_end = end_re.search(tail) region = tail[: region_end.start()] if region_end else tail issues = [] @@ -215,6 +254,11 @@ def _parse_known_issues(html: str, kb_number: str, source_url: str) -> dict: issue = _parse_issue_block(block) if issue: issues.append(issue) + if not issues: + for match in _DETAILS_RE.finditer(region): + issue = _parse_details_block(match.group(0)) + if issue: + issues.append(issue) if issues: return {"status": "published", "issues": issues, "source_url": source_url} @@ -256,23 +300,32 @@ def _resolve_redirect(location: str | None) -> str: async def _fetch_kb_page(kb_number: str) -> tuple[str | None, str | None]: - """Fetch the support page for a KB, following one validated redirect hop. - - Returns (final_url, html), or (None, None) when Microsoft has no page for - this KB. Raises KnownIssuesError for anything that is a retrieval failure - rather than a definitive absence. + """Fetch the support page for a KB, following bounded validated redirects. + + The /help/{kb} resolver answers with a short same-host redirect chain to + the canonical article (currently two hops: /topic/... then + /servicing/os/...); each hop's target is validated before following and + the hop count is capped at MAX_REDIRECT_HOPS. Returns (final_url, html), + or (None, None) when Microsoft has no page for this KB. Raises + KnownIssuesError for anything that is a retrieval failure rather than a + definitive absence. """ - url = SUPPORT_KB_URL.format(kb=kb_number) + target = SUPPORT_KB_URL.format(kb=kb_number) try: - status, location = await http_client.get_location(url, timeout=30.0) - if status == 404: - return None, None - if status in _REDIRECT_STATUSES: - target = _resolve_redirect(location) - elif status == 200: - target = url - else: + for _ in range(MAX_REDIRECT_HOPS + 1): + status, location = await http_client.get_location(target, timeout=30.0) + if status == 404: + return None, None + if status == 200: + break + if status in _REDIRECT_STATUSES: + target = _resolve_redirect(location) + continue raise KnownIssuesError(f"KB page lookup returned HTTP {status}") + else: + raise KnownIssuesError( + f"KB page redirected more than {MAX_REDIRECT_HOPS} times" + ) body_status, body = await http_client.get_bounded( target, timeout=30.0, max_bytes=MAX_RESPONSE_BYTES diff --git a/src/patch_tuesday_mcp/models/vulnerability.py b/src/patch_tuesday_mcp/models/vulnerability.py index 6cade20..6e89ea2 100644 --- a/src/patch_tuesday_mcp/models/vulnerability.py +++ b/src/patch_tuesday_mcp/models/vulnerability.py @@ -289,11 +289,17 @@ def compute_stats(vulnerabilities: list[Vulnerability]) -> dict: publicly_disclosed = 0 kev = 0 + unrated = 0 + unspecified_impact = 0 for v in vulnerabilities: if v.severity: by_severity[v.severity] = by_severity.get(v.severity, 0) + 1 + else: + unrated += 1 if v.impact: by_impact[v.impact] = by_impact.get(v.impact, 0) + 1 + else: + unspecified_impact += 1 for family in v.product_families: by_family[family] = by_family.get(family, 0) + 1 if v.exploited: @@ -303,6 +309,14 @@ def compute_stats(vulnerabilities: list[Vulnerability]) -> dict: if v.kev is not None: kev += 1 + # Entries MSRC ships without a severity rating or impact category + # (e.g. Chromium/third-party CVEs) get an explicit bucket so the + # breakdowns always sum to the reported total. + if unrated: + by_severity["Unrated"] = unrated + if unspecified_impact: + by_impact["Unspecified"] = unspecified_impact + def ranked(d: dict[str, int]) -> list[dict]: return [ {"name": name, "count": count} diff --git a/tests/fixtures/kb_servicing_layout.html b/tests/fixtures/kb_servicing_layout.html new file mode 100644 index 0000000..61f8189 --- /dev/null +++ b/tests/fixtures/kb_servicing_layout.html @@ -0,0 +1,41 @@ +<!DOCTYPE html> +<html lang="en-us"> +<head> +<meta charset="utf-8" /> +<meta name="awa-kb_id" content="5094126" /> +<title>June 9, 2026—KB5094126 (OS Build 17763.8880) | Microsoft Support + + +
+

Summary

+

This security update includes quality improvements. For an overview of Windows 10, version 1809, see its update history page.

+

Known issues in this update

+
+Microsoft Office applications might fail to open from certain third-party apps +

Symptoms

+

Microsoft has received reports of an issue in which certain third-party applications might be unable to launch Microsoft Office applications or open documents after installing this update. This issue affects certain third-party applications that use OLE automation to interact with Microsoft Office applications.

+

Third-party information disclaimer

+

The third-party products listed are manufactured by companies that are independent of Microsoft.

+

Resolution

+

A resolution is in progress and will be included in a future Windows update. To work around this issue, open the application or document directly instead of launching it from the affected third-party application.

+
+
+Deleting from Recycle Bin displays an internal file name +

Symptoms

+

After you install this update, the confirmation dialog might display the internal Recycle Bin file name (for example, $Rxxxxx.ext) instead of the original file name when you permanently delete a single item.

+

Workaround

+

This issue is addressed in KB5095093. Until you install that update, dismiss the dialog to continue.

+
+

Servicing stack update information

+

This update also includes the servicing stack update.

+

Known issues in this update

+
+Duplicate section issue that must not be parsed twice +

Symptoms

+

The combined SSU and LCU package repeats the known-issues section.

+
+

How to get this update

+

Highlights of the release channels follow.

+
+ + diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index 1fe07f7..f39d420 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -1,5 +1,8 @@ """Tests for the EPSS / CISA KEV enrichment clients (mocked HTTP).""" +import json +import re + import pytest from patch_tuesday_mcp.feeds import enrichment @@ -110,6 +113,24 @@ async def test_kev_caching_and_parse(mock_api): assert len(kev_calls) == 1, "catalog is cached within the TTL" +async def test_epss_values_rounded_to_source_precision(monkeypatch): + """EPSS publishes 5-decimal values; parsed floats must not carry more digits.""" + + async def fake_get_json(url, timeout=30.0): + return { + "status": "OK", + "data": [ + {"cve": "CVE-2026-11645", "epss": "0.016540000", "percentile": "0.831549999"} + ], + } + + monkeypatch.setattr(enrichment, "_get_json", fake_get_json) + score, percentile = (await fetch_epss(["CVE-2026-11645"]))["CVE-2026-11645"] + assert score == 0.01654 + assert percentile == 0.83155 + assert not re.search(r"\d\.\d{6,}", json.dumps([score, percentile])) + + async def test_fetch_failures_return_empty(monkeypatch): async def failing_get_json(url, timeout=30.0): raise EnrichmentError("boom") diff --git a/tests/test_known_issues.py b/tests/test_known_issues.py index 34750e3..4499cc1 100644 --- a/tests/test_known_issues.py +++ b/tests/test_known_issues.py @@ -124,6 +124,55 @@ def test_parse_block_without_title_is_skipped(): assert len(result["issues"]) == 1, "a titleless block must be skipped, not invented" +# --- Parser: the /servicing/ layout (2026 site migration) --- + + +def test_parse_servicing_layout_published(): + result = known_issues._parse_known_issues( + _load("kb_servicing_layout.html"), "5094126", SOURCE_URL + ) + assert result["status"] == "published" + issues = result["issues"] + assert len(issues) == 2 + + office = issues[0] + assert office["title"] == ( + "Microsoft Office applications might fail to open from certain third-party apps" + ) + assert "OLE automation" in office["symptoms"] + assert "open the application or document directly" in office["resolution"] + assert "resolved_by" not in office + + recycle = issues[1] + assert recycle["title"] == "Deleting from Recycle Bin displays an internal file name" + assert "$Rxxxxx.ext" in recycle["symptoms"] + assert recycle["resolved_by"] == "KB5095093" + + +def test_parse_servicing_layout_first_section_only(): + # The combined SSU+LCU page repeats the known-issues

; only the first + # section may be parsed, and neighbor sections must not bleed in. + result = known_issues._parse_known_issues( + _load("kb_servicing_layout.html"), "5094126", SOURCE_URL + ) + titles = " ".join(issue["title"] for issue in result["issues"]) + assert "Duplicate section" not in titles + assert "How to get this update" not in titles + assert "Servicing stack" not in titles + + +def test_parse_servicing_layout_none_aware_is_none_published(): + html = ( + 'June 9, 2026 KB5094126 | Microsoft Support' + '

Known issues in this update' + "

Microsoft is not currently aware of any issues with " + "this update.

How to get this update

" + ) + result = known_issues._parse_known_issues(html, "5094126", SOURCE_URL) + assert result["status"] == "none_published" + assert "not currently aware of any issues" in result["note"] + + # --- Parser: none published --- @@ -157,16 +206,17 @@ def test_parse_drifted_markup_is_unavailable_with_pointer(): assert result["source_url"] == SOURCE_URL -# --- _fetch_kb_page: redirect policy (manual, single hop, same host only) --- +# --- _fetch_kb_page: redirect policy (manual, bounded hops, same host only) --- -def _patch_transport(monkeypatch, *, location_status=301, location=None, - body_status=200, body=b"ok"): +def _patch_transport(monkeypatch, *, hops, body_status=200, body=b"ok"): + """Serve a scripted (status, location) sequence; the last hop repeats.""" fetched = [] async def fake_get_location(url, *, timeout): fetched.append(("head", url)) - return (location_status, location) + status, location = hops[min(len(fetched) - 1, len(hops) - 1)] + return (status, location) async def fake_get_bounded(url, *, headers=None, timeout, max_bytes): fetched.append(("get", url)) @@ -178,40 +228,79 @@ async def fake_get_bounded(url, *, headers=None, timeout, max_bytes): async def test_fetch_follows_single_same_host_relative_redirect(monkeypatch): - fetched = _patch_transport(monkeypatch, location="/en-us/topic/some-kb-slug") + fetched = _patch_transport( + monkeypatch, hops=[(301, "/en-us/topic/some-kb-slug"), (200, None)] + ) url, html = await known_issues._fetch_kb_page("5094126") assert url == "https://support.microsoft.com/en-us/topic/some-kb-slug" assert html == "ok" assert ("get", url) in fetched +async def test_fetch_follows_two_hop_chain_to_servicing_page(monkeypatch): + # The live site today: /help -> /topic -> /servicing/os/... -> 200. + servicing = "https://support.microsoft.com/en-US/servicing/os/windows-10/2026/06/x" + fetched = _patch_transport( + monkeypatch, + hops=[(301, "/en-us/topic/some-kb-slug"), (301, servicing), (200, None)], + ) + url, html = await known_issues._fetch_kb_page("5094126") + assert url == servicing + assert html == "ok" + assert ("get", servicing) in fetched + + +async def test_fetch_redirect_loop_is_bounded(monkeypatch): + fetched = _patch_transport(monkeypatch, hops=[(301, "/en-us/topic/loop")]) + with pytest.raises(KnownIssuesError, match="redirected more than"): + await known_issues._fetch_kb_page("5094126") + heads = [f for f in fetched if f[0] == "head"] + assert len(heads) == known_issues.MAX_REDIRECT_HOPS + 1 + assert not [f for f in fetched if f[0] == "get"], "a looping chain must never be read" + + async def test_fetch_accepts_absolute_same_host_https_redirect(monkeypatch): _patch_transport( - monkeypatch, location="https://support.microsoft.com/en-us/topic/some-kb-slug" + monkeypatch, + hops=[(301, "https://support.microsoft.com/en-us/topic/some-kb-slug"), (200, None)], ) url, _ = await known_issues._fetch_kb_page("5094126") assert url == "https://support.microsoft.com/en-us/topic/some-kb-slug" async def test_fetch_rejects_cross_host_redirect(monkeypatch): - _patch_transport(monkeypatch, location="https://evil.example.com/en-us/topic/x") + _patch_transport(monkeypatch, hops=[(301, "https://evil.example.com/en-us/topic/x")]) + with pytest.raises(KnownIssuesError): + await known_issues._fetch_kb_page("5094126") + + +async def test_fetch_rejects_cross_host_redirect_mid_chain(monkeypatch): + _patch_transport( + monkeypatch, + hops=[(301, "/en-us/topic/fine"), (301, "https://evil.example.com/x")], + ) with pytest.raises(KnownIssuesError): await known_issues._fetch_kb_page("5094126") async def test_fetch_rejects_redirect_without_location(monkeypatch): - _patch_transport(monkeypatch, location=None) + _patch_transport(monkeypatch, hops=[(301, None)]) with pytest.raises(KnownIssuesError): await known_issues._fetch_kb_page("5094126") async def test_fetch_404_means_no_page(monkeypatch): - _patch_transport(monkeypatch, location_status=404) + _patch_transport(monkeypatch, hops=[(404, None)]) + assert await known_issues._fetch_kb_page("5094126") == (None, None) + + +async def test_fetch_404_mid_chain_means_no_page(monkeypatch): + _patch_transport(monkeypatch, hops=[(301, "/en-us/topic/gone"), (404, None)]) assert await known_issues._fetch_kb_page("5094126") == (None, None) async def test_fetch_direct_200_reads_original_url(monkeypatch): - fetched = _patch_transport(monkeypatch, location_status=200) + fetched = _patch_transport(monkeypatch, hops=[(200, None)]) url, html = await known_issues._fetch_kb_page("5094126") assert url == known_issues.SUPPORT_KB_URL.format(kb="5094126") assert html == "ok" @@ -219,13 +308,13 @@ async def test_fetch_direct_200_reads_original_url(monkeypatch): async def test_fetch_landing_non_200_raises(monkeypatch): - _patch_transport(monkeypatch, location="/en-us/topic/x", body_status=500) + _patch_transport(monkeypatch, hops=[(301, "/en-us/topic/x"), (200, None)], body_status=500) with pytest.raises(KnownIssuesError): await known_issues._fetch_kb_page("5094126") async def test_fetch_unexpected_first_hop_status_raises(monkeypatch): - _patch_transport(monkeypatch, location_status=503) + _patch_transport(monkeypatch, hops=[(503, None)]) with pytest.raises(KnownIssuesError): await known_issues._fetch_kb_page("5094126") diff --git a/tests/test_models.py b/tests/test_models.py index 63c2a44..20d239d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -9,6 +9,7 @@ MonthlyRelease, Vulnerability, _parse_vulnerability, + compute_stats, parse_cvrf, parse_exploit_status, sort_vulnerabilities, @@ -118,6 +119,30 @@ def test_stats(release): assert stats["by_product_family"], "expected family counts" +def test_stats_severity_buckets_sum_to_total(): + """Entries without a rating get an explicit bucket so counts reconcile.""" + vulns = [ + Vulnerability(cve="CVE-2026-1", severity="Critical", impact="Remote Code Execution"), + Vulnerability(cve="CVE-2026-2"), # Chromium-style: no severity, no impact + Vulnerability(cve="CVE-2026-3", severity="Important"), + ] + stats = compute_stats(vulns) + assert sum(e["count"] for e in stats["by_severity"]) == stats["total"] == 3 + assert {"name": "Unrated", "count": 1} in stats["by_severity"] + assert sum(e["count"] for e in stats["by_impact"]) == stats["total"] + assert {"name": "Unspecified", "count": 2} in stats["by_impact"] + + +def test_stats_no_placeholder_buckets_when_fully_rated(): + vulns = [ + Vulnerability(cve="CVE-2026-1", severity="Critical", impact="Spoofing"), + Vulnerability(cve="CVE-2026-2", severity="Low", impact="Spoofing"), + ] + stats = compute_stats(vulns) + names = {e["name"] for e in stats["by_severity"]} | {e["name"] for e in stats["by_impact"]} + assert "Unrated" not in names and "Unspecified" not in names + + def test_sort_exploited_first(release): ordered = sort_vulnerabilities(release.vulnerabilities) assert ordered[0].cve == "CVE-2026-99999", "exploited CVE sorts first"