From c4c73eb96eb9535a7585dda4582eb20bd273ed14 Mon Sep 17 00:00:00 2001 From: YAMRAJ13y Date: Sun, 9 Aug 2026 20:55:46 +0530 Subject: [PATCH] fix(ofac): stop downloading 267 MB per sweep; source always timed out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `briefing()` issued three full-body requests to the OFAC publication exports on every sweep: SDN.XML 27.5 MB SDN_ADVANCED.XML 120.0 MB (metadata) SDN_ADVANCED.XML 120.0 MB (again, for sample entries) ------------------------- 267.5 MB every 15 minutes = ~25.7 GB/day Sizes confirmed from the origin's own Content-Range headers. `safeFetch` reads the entire body with `res.text()` and only then truncates to `rawText: text.slice(0, 500)`, so all 267 MB was pulled into memory and discarded. The inline comment claiming it "will get the first 500 chars" described an optimisation that does not exist. The source therefore never completed: two 20s fetches in parallel followed by a sequential 25s fetch cannot finish inside the 30s per-source budget in apis/briefing.mjs, so every sweep logged Source OFAC timed out after 30s Two of the three downloads were also pointless. SDN_ADVANCED.XML uses a different schema — it has no , no and no — so `advancedList` was always all-null and `sampleEntries` was always empty. Fix: - request only the first 64 KB via `Range: bytes=0-65535`; the S3 origin advertises `Accept-Ranges: bytes` and answers 206 - bound the read with a streaming reader too, so a proxy that ignores Range still cannot pull 120 MB into memory - fetch each list once and reuse the buffer for metadata and sampling - take sample entries from SDN.XML, which actually contains - parse the advanced export's block so its date populates Measured before/after: requests 3 -> 2 transfer 267.5 MB -> 128.0 KB (~2,140x less) duration timeout -> 5.0 s sampleEntries 0 -> 10 advancedList.publishDate null -> 2026-08-07 This does not touch apis/utils/fetch.mjs, so it does not conflict with #121. Co-Authored-By: Claude Opus 5 (1M context) --- apis/sources/ofac.mjs | 90 +++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/apis/sources/ofac.mjs b/apis/sources/ofac.mjs index 38e37420..f73647f0 100644 --- a/apis/sources/ofac.mjs +++ b/apis/sources/ofac.mjs @@ -2,8 +2,6 @@ // No auth required. Monitors the Specially Designated Nationals (SDN) list // and consolidated sanctions list for changes. -import { safeFetch } from '../utils/fetch.mjs'; - const EXPORTS_BASE = 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports'; // SDN list endpoints @@ -11,18 +9,72 @@ const SDN_XML_URL = `${EXPORTS_BASE}/SDN.XML`; const SDN_ADVANCED_URL = `${EXPORTS_BASE}/SDN_ADVANCED.XML`; const CONS_ADVANCED_URL = `${EXPORTS_BASE}/CONS_ADVANCED.XML`; +// These exports are whole-database dumps — SDN.XML is ~27 MB and +// SDN_ADVANCED.XML ~120 MB — but everything this briefing reports (publish +// date, record count, a sample of entries) sits in the first few KB. Ask for +// just that range: the S3 origin sets `Accept-Ranges: bytes` and answers 206. +const HEAD_BYTES = 64 * 1024; + +// Read at most `bytes` from `url`. Uses a Range request, and still stops early +// if an intermediary ignores it and starts streaming the full body, so a 120 MB +// export can never be pulled into memory. +async function fetchHead(url, { bytes = HEAD_BYTES, timeout = 20000 } = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0', 'Range': `bytes=0-${bytes - 1}` }, + }); + // 206 = Range honoured, 200 = ignored (we bound the read below either way). + if (!res.ok && res.status !== 206) throw new Error(`HTTP ${res.status}`); + + const reader = res.body.getReader(); + const chunks = []; + let received = 0; + while (received < bytes) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + received += value.length; + } + await reader.cancel().catch(() => {}); + + return { rawText: Buffer.concat(chunks).toString('utf8') }; + } catch (e) { + return { error: e.message }; + } finally { + clearTimeout(timer); + } +} + +// The ADVANCED exports date themselves with +// rather than a string. Normalise it to YYYY-MM-DD. +function parseDateOfIssue(raw) { + const block = raw.match(/]*>([\s\S]*?)<\/DateOfIssue>/i)?.[1]; + if (!block) return null; + const y = block.match(/(\d+)<\/Year>/i)?.[1]; + const m = block.match(/(\d+)<\/Month>/i)?.[1]; + const d = block.match(/(\d+)<\/Day>/i)?.[1]; + return y && m && d ? `${y}-${m.padStart(2, '0')}-${d.padStart(2, '0')}` : null; +} + // Parse basic info from SDN XML (publish date, entry count) function parseSDNMetadata(xml) { if (!xml || xml.error) return { error: xml?.error || 'No data returned' }; const raw = xml.rawText || ''; - // Extract publish date + // Extract publish date. SDN.XML uses ; the ADVANCED exports + // carry a structured block instead, which is why the advanced + // list's publishDate was always null. const publishDate = raw.match(/(.*?)<\/Publish_Date>/)?.[1] || raw.match(/(.*?)<\/publish_date>/i)?.[1] + || parseDateOfIssue(raw) || null; - // Count SDN entries + // Entries visible in the sampled window — `recordCount` below is the + // authoritative total for the whole list. const entryMatches = raw.match(//gi); const entryCount = entryMatches ? entryMatches.length : null; @@ -40,24 +92,19 @@ function parseSDNMetadata(xml) { }; } -// Fetch SDN list metadata (smaller initial chunk via timeout) +// Fetch SDN list metadata from the head of the export export async function getSDNMetadata() { - // The full SDN XML is large; safeFetch will get the first 500 chars - // which should include the header/publish date - const data = await safeFetch(SDN_XML_URL, { timeout: 20000 }); - return parseSDNMetadata(data); + return parseSDNMetadata(await fetchHead(SDN_XML_URL)); } // Fetch advanced SDN data (includes more structured info) export async function getSDNAdvanced() { - const data = await safeFetch(SDN_ADVANCED_URL, { timeout: 20000 }); - return parseSDNMetadata(data); + return parseSDNMetadata(await fetchHead(SDN_ADVANCED_URL)); } // Fetch consolidated list metadata export async function getConsolidatedMetadata() { - const data = await safeFetch(CONS_ADVANCED_URL, { timeout: 20000 }); - return parseSDNMetadata(data); + return parseSDNMetadata(await fetchHead(CONS_ADVANCED_URL)); } // Parse recent SDN entries from XML snippet @@ -101,15 +148,18 @@ function parseRecentEntries(xml) { // Briefing — report on sanctions list status and metadata export async function briefing() { - const [sdnMeta, advancedMeta] = await Promise.all([ - getSDNMetadata(), - getSDNAdvanced(), + // One ranged read per list, reused for both metadata and sample entries. + // The advanced export was previously downloaded twice — once here and again + // for the sample — and neither pass could ever succeed on it: SDN_ADVANCED.XML + // contains no elements at all. Sample from SDN.XML, which does. + const [sdnHead, advancedHead] = await Promise.all([ + fetchHead(SDN_XML_URL), + fetchHead(SDN_ADVANCED_URL), ]); - // Try to extract any entries visible in the advanced data - const sampleEntries = parseRecentEntries( - await safeFetch(SDN_ADVANCED_URL, { timeout: 25000 }) - ); + const sdnMeta = parseSDNMetadata(sdnHead); + const advancedMeta = parseSDNMetadata(advancedHead); + const sampleEntries = parseRecentEntries(sdnHead); return { source: 'OFAC Sanctions',