diff --git a/quantmind/preprocess/pr_newswire.py b/quantmind/preprocess/pr_newswire.py index 8f0e122..38ed0af 100644 --- a/quantmind/preprocess/pr_newswire.py +++ b/quantmind/preprocess/pr_newswire.py @@ -27,6 +27,7 @@ HttpFetcher, ) from quantmind.preprocess.news import ( + build_news_identity, canonicalize_source_url, news_document_from_fetched, preprocess_news_document, @@ -560,9 +561,10 @@ def _observation_from_row( canonical_url = canonicalize_source_url(urljoin(_BASE_URL, row.href)) payload_match = _PAYLOAD_ID_RE.search(canonical_url) payload_id = payload_match.group("id") if payload_match else None - identity = _build_identity( + identity = build_news_identity( + source_type="press_release", payload_id=payload_id, - canonical_url=canonical_url, + source_url=canonical_url, ) return PRNewswireObservation( identity=identity, @@ -688,16 +690,6 @@ def _artifact_from_fetched( ) -def _build_identity( - *, - payload_id: str | None, - canonical_url: str, -) -> str: - raw = "\x1f".join((_SOURCE, payload_id or canonical_url)) - digest = hashlib.sha256(raw.encode("utf-8")).hexdigest() - return f"news:{_SOURCE}:{digest}" - - def _failure( *, stage: NewsFailureStage, diff --git a/tests/preprocess/test_pr_newswire.py b/tests/preprocess/test_pr_newswire.py index 856b5f7..d7fca23 100644 --- a/tests/preprocess/test_pr_newswire.py +++ b/tests/preprocess/test_pr_newswire.py @@ -8,6 +8,7 @@ import respx from quantmind.preprocess.fetch.http import FetchPolicy +from quantmind.preprocess.news import build_news_identity from quantmind.preprocess.pr_newswire import ( _collect_pr_newswire, _discover_pr_newswire, @@ -107,6 +108,15 @@ def respond(request: httpx.Request) -> httpx.Response: self.assertEqual( len({item.identity for item in result.observations}), 3 ) + first = result.observations[0] + self.assertEqual( + first.identity, + build_news_identity( + source_type="press_release", + source_url=first.canonical_url, + payload_id=first.payload_id, + ), + ) artifact = result.observations[0].discovery_artifact self.assertIn( b"Repeated", result.observations[1].discovery_artifact.bytes @@ -368,7 +378,12 @@ async def test_collects_article_and_discards_raw_html_by_default( self.assertTrue(document.article_artifact.content_hash) self.assertTrue(document.discovery_artifact.bytes) self.assertEqual( - document.identity.split(":", 2)[:2], ["news", "pr-newswire"] + document.identity, + build_news_identity( + source_type="press_release", + source_url=document.canonical_url, + payload_id=document.payload_id, + ), ) self.assertEqual(document.ticker_hints[0].symbol, "EXCO")