From 8a8c6c9862201b6ac0b4a28cfc1ffa36a61ab279 Mon Sep 17 00:00:00 2001 From: Manasa Kankanaka Date: Sat, 4 Jul 2026 11:58:18 -0700 Subject: [PATCH 01/14] e2e-rag: Improve JSON parsing robustness and URL extraction This commit improves the reliability of the multi-shot retrieval system: 1. **Robust JSON Parsing** (multi_shot_retrieval.py, accuracy_eval.py): - Use regex-based extraction to handle JSON in markdown code blocks - Extract JSON objects from mixed text/markdown LLM responses - Add fallback behavior when JSON parsing fails instead of crashing - Applied to: evaluate_document_relevance(), check_sufficiency(), generate_search_queries(), query_rewriter(), call_judge() 2. **URL Extraction Fixes** (multi_shot_retrieval.py): - Support both 'original_url' and 'source' metadata fields - Convert Wikipedia filenames (en.wikipedia.org_wiki_*.html) to URLs - Debug logging for URL extraction issues 3. **Metrics Compatibility** (multi_shot_retrieval.py): - Add backward-compatible aliases (precision, recall, f1) - SUT expects different key names than evaluation module produces - Added 'retrieved_urls' to metrics output 4. **Config Updates** (user.conf): - Added e2e-datasetup.Offline configuration These changes make the system more resilient to LLM output variations without changing the underlying retrieval logic or model assignments. Co-Authored-By: Claude Sonnet 4.5 --- e2e-rag/accuracy_eval.py | 18 +++-- e2e-rag/multi_shot_retrieval.py | 123 ++++++++++++++++++++++++-------- e2e-rag/user.conf | 2 + 3 files changed, 109 insertions(+), 34 deletions(-) diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 29ce199482..026456ab09 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -91,11 +91,19 @@ def call_judge(question: str, ground_truth: str, llm_answer: str, # Parse JSON response content = content.strip() - if content.startswith("```"): - content = content.split("```")[1] - if content.startswith("json"): - content = content[4:] - content = content.strip() + + # Extract JSON from markdown code blocks + if "```" in content: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', content, re.DOTALL) + if json_block_match: + content = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', content, re.DOTALL) + if json_match: + content = json_match.group(0) + else: + return {"correct": False, "reasoning": "No JSON found in judge response"} judge_result = json.loads(content) return judge_result diff --git a/e2e-rag/multi_shot_retrieval.py b/e2e-rag/multi_shot_retrieval.py index 3ee84c731d..b557cda897 100644 --- a/e2e-rag/multi_shot_retrieval.py +++ b/e2e-rag/multi_shot_retrieval.py @@ -502,11 +502,19 @@ def evaluate_document_relevance(question: str, print(f" Warning: Relevance check returned empty, marking all as relevant") return {"relevance": [1] * len(new_documents)} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) + if json_match: + llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in relevance response, marking all as relevant") + return {"relevance": [1] * len(new_documents)} relevance_result = json.loads(llm_output) relevance = relevance_result.get("relevance", []) @@ -610,16 +618,21 @@ def check_sufficiency(question: str, return {"sufficient": True, "reasoning": "Max iterations reached"} return {"sufficient": False, "reasoning": "LLM returned empty"} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() # Try to extract JSON json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) if json_match: llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in sufficiency response") + if iteration >= max_iterations: + return {"sufficient": True, "reasoning": "Max iterations reached, no JSON"} + return {"sufficient": False, "reasoning": "No JSON in response"} result = json.loads(llm_output) sufficient = result.get("sufficient", False) @@ -815,16 +828,21 @@ def generate_search_queries(question: str, print(f" Warning: Query generation returned empty") return {"queries": [question], "feedback": "LLM returned empty"} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + # Find content between ```json and ``` or just between ``` markers + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() # Try to extract JSON object even from mixed text/markdown responses json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) if json_match: llm_output = json_match.group(0) + else: + # No JSON found in response + print(f" Warning: No JSON found in LLM response") + return {"queries": [question], "feedback": "No JSON in LLM response"} query_result = json.loads(llm_output) return { @@ -1001,14 +1019,27 @@ def query_rewriter(question: str, new_documents: List[tuple], } llm_output = llm_output.strip() - - # Parse JSON output - handle markdown code blocks - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() - + + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) + if json_match: + llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in query rewriter response, using fallback") + return { + "relevance": [0] * len(new_documents), + "summaries": [""] * len(new_documents), + "queries": [question], + "feedback": "No JSON in LLM response", + "answer": "" + } + result_data = json.loads(llm_output) # Validate format @@ -1424,12 +1455,33 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], # Add to new_docs for evaluation (avoid duplicates) for result in results: + # DEBUG: Log metadata structure for first result + if not all_retrieved_urls: # Log only once + print(f" [DEBUG] Sample result metadata keys: {list(result.metadata.keys())}") + print(f" [DEBUG] Sample result metadata: {result.metadata}") + + # Try to get URL from metadata - support both 'original_url' and 'source' fields + url = None if 'original_url' in result.metadata and result.metadata['original_url']: url = result.metadata['original_url'] - if url not in all_retrieved_urls: - all_retrieved_urls.add(url) - new_docs.append((url, result.page_content)) - iteration_results.append(result) + elif 'source' in result.metadata and result.metadata['source']: + # Convert source filename to Wikipedia URL + # e.g., "en.wikipedia.org_wiki_Kirk_Watson.html" -> "https://en.wikipedia.org/wiki/Kirk_Watson" + source = result.metadata['source'] + if source.startswith('en.wikipedia.org_wiki_'): + # Remove prefix and .html suffix + page_name = source.replace('en.wikipedia.org_wiki_', '').replace('.html', '') + url = f"https://en.wikipedia.org/wiki/{page_name}" + if not all_retrieved_urls: # Log conversion once + print(f" [DEBUG] Converted source to URL: {source} -> {url}") + + if url and url not in all_retrieved_urls: + all_retrieved_urls.add(url) + new_docs.append((url, result.page_content)) + iteration_results.append(result) + elif not url and not all_retrieved_urls: + # DEBUG: Log why URL was skipped + print(f" [DEBUG] Skipping result - no valid original_url or source in metadata") # Track how many NEW docs this query found docs_found_by_query = len(new_docs) - query_start_count @@ -1474,7 +1526,14 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], retrieved_urls.append(doc[0]) # url is first element elif len(doc) == 2: # Handle old format for backward compatibility retrieved_urls.append(doc[0]) # url is first element - + + # DEBUG: Log kept_docs structure + if not retrieved_urls and kept_docs: + print(f" WARNING: kept_docs has {len(kept_docs)} items but retrieved_urls is empty!") + print(f" First kept_doc structure: {type(kept_docs[0])}, len={len(kept_docs[0]) if hasattr(kept_docs[0], '__len__') else 'N/A'}") + if kept_docs: + print(f" First kept_doc sample: {str(kept_docs[0])[:200]}...") + # Limit to top_k_reranking (reranking already done per-subquery) retrieved_urls = retrieved_urls[:top_k_reranking] @@ -1482,7 +1541,12 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], from evaluation import calculate_retrieval_metrics expected_set = set(url for url in expected_urls if url and url.strip()) metrics = calculate_retrieval_metrics(list(expected_set), retrieved_urls) - + + # Add backward-compatible aliases for SUT (expects precision/recall/f1 not precision@N/recall@N/f1@N) + metrics['precision'] = metrics.get('precision@N', 0) + metrics['recall'] = metrics.get('recall@N', 0) + metrics['f1'] = metrics.get('f1@N', 0) + # Add iteration statistics query_llm_time = (llm_end_time - llm_start_time) if (llm_start_time is not None and llm_end_time is not None) else total_time metrics.update({ @@ -1494,6 +1558,7 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], 'sufficient': sufficient, 'avg_iteration_time': sum(iteration_times) / len(iteration_times) if iteration_times else 0, 'llm_answer': final_answer, + 'retrieved_urls': retrieved_urls, }) # Print final results diff --git a/e2e-rag/user.conf b/e2e-rag/user.conf index e920ef67ad..c215e19243 100644 --- a/e2e-rag/user.conf +++ b/e2e-rag/user.conf @@ -22,3 +22,5 @@ rag-db.Offline.target_qps = 0 rag-db.Offline.min_duration = 0 rag-db.Offline.min_query_count = 2503 rag-db.Offline.max_async_queries = 2503 +e2e-datasetup.Offline.max_async_queries = 2515 +e2e-datasetup.Offline.min_query_count = 2515 From 833790e1ff9300b22ea1ffaddae287cbab34bb85 Mon Sep 17 00:00:00 2001 From: rpoornac Date: Tue, 7 Jul 2026 14:01:22 -0700 Subject: [PATCH 02/14] Create download_additional_urls.py Download additional 470 URLs files. --- e2e-rag/download_additional_urls.py | 795 ++++++++++++++++++++++++++++ 1 file changed, 795 insertions(+) create mode 100644 e2e-rag/download_additional_urls.py diff --git a/e2e-rag/download_additional_urls.py b/e2e-rag/download_additional_urls.py new file mode 100644 index 0000000000..bd22e08d2a --- /dev/null +++ b/e2e-rag/download_additional_urls.py @@ -0,0 +1,795 @@ +#!/usr/bin/env python3 +"""Download the 470 additional Wikipedia HTML files. + +This script is self-contained: the URL list is embedded below. Pass the output +folder as the only command-line argument; files are written directly there. +""" + +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +import argparse +import json +import time +import urllib.error +import urllib.request + +WORKERS = 8 +USER_AGENT = "Mozilla/5.0 (compatible; html-fetch/1.0)" + +DOWNLOADS = [('en.wikipedia.org_wiki_%22Weird_Al%22_Yankovic.html', 'https://en.wikipedia.org/wiki/%22Weird_Al%22_Yankovic'), + ('en.wikipedia.org_wiki_%C5%BDAK_Subotica.html', 'https://en.wikipedia.org/wiki/%C5%BDAK_Subotica'), + ('en.wikipedia.org_wiki_1854_Broad_Street_cholera_outbreak#.html', + 'https://en.wikipedia.org/wiki/1854_Broad_Street_cholera_outbreak#'), + ('en.wikipedia.org_wiki_1900%E2%80%9301_Football_League#Final_league_tables.html', + 'https://en.wikipedia.org/wiki/1900%E2%80%9301_Football_League#Final_league_tables'), + ('en.wikipedia.org_wiki_1923#Deaths.html', 'https://en.wikipedia.org/wiki/1923#Deaths'), + ('en.wikipedia.org_wiki_1960_European_Nations%27_Cup.html', + 'https://en.wikipedia.org/wiki/1960_European_Nations%27_Cup'), + ('en.wikipedia.org_wiki_1967_Formula_One_season#Teams_and_drivers.html', + 'https://en.wikipedia.org/wiki/1967_Formula_One_season#Teams_and_drivers'), + ('en.wikipedia.org_wiki_1972#.html', 'https://en.wikipedia.org/wiki/1972#'), + ('en.wikipedia.org_wiki_1976%E2%80%9377_Portland_Trail_Blazers_season.html', + 'https://en.wikipedia.org/wiki/1976%E2%80%9377_Portland_Trail_Blazers_season'), + ('en.wikipedia.org_wiki_1982%E2%80%9383_Wolverhampton_Wanderers_F.C._season.html', + 'https://en.wikipedia.org/wiki/1982%E2%80%9383_Wolverhampton_Wanderers_F.C._season'), + ('en.wikipedia.org_wiki_1984%E2%80%9385_NHL_season.html', 'https://en.wikipedia.org/wiki/1984%E2%80%9385_NHL_season'), + ('en.wikipedia.org_wiki_1996%E2%80%9397_AC_Milan_season.html', + 'https://en.wikipedia.org/wiki/1996%E2%80%9397_AC_Milan_season'), + ('en.wikipedia.org_wiki_1996%E2%80%9397_Detroit_Red_Wings_season.html', + 'https://en.wikipedia.org/wiki/1996%E2%80%9397_Detroit_Red_Wings_season'), + ('en.wikipedia.org_wiki_1997_NFL_draft#Round_6.html', 'https://en.wikipedia.org/wiki/1997_NFL_draft#Round_6'), + ('en.wikipedia.org_wiki_1998_NFL_draft#Player_selections.html', + 'https://en.wikipedia.org/wiki/1998_NFL_draft#Player_selections'), + ('en.wikipedia.org_wiki_1999%E2%80%932000_Olympique_de_Marseille_season.html', + 'https://en.wikipedia.org/wiki/1999%E2%80%932000_Olympique_de_Marseille_season'), + ('en.wikipedia.org_wiki_19th_People%27s_Choice_Awards#Awards.html', + 'https://en.wikipedia.org/wiki/19th_People%27s_Choice_Awards#Awards'), + ('en.wikipedia.org_wiki_2002_Wimbledon_Championships_%E2%80%93_Men%27s_singles.html', + 'https://en.wikipedia.org/wiki/2002_Wimbledon_Championships_%E2%80%93_Men%27s_singles'), + ('en.wikipedia.org_wiki_2002_World_Series#Composite_box.html', + 'https://en.wikipedia.org/wiki/2002_World_Series#Composite_box'), + ('en.wikipedia.org_wiki_2003_US_Open_%E2%80%93_Men%27s_singles.html', + 'https://en.wikipedia.org/wiki/2003_US_Open_%E2%80%93_Men%27s_singles'), + ('en.wikipedia.org_wiki_2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in,clash%20and%20interlacing%20of%20cultures.%22.html', + 'https://en.wikipedia.org/wiki/2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in,clash%20and%20interlacing%20of%20cultures.%22'), + ('en.wikipedia.org_wiki_2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in.html', + 'https://en.wikipedia.org/wiki/2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in'), + ('en.wikipedia.org_wiki_2007_NHL_entry_draft#Round_one.html', + 'https://en.wikipedia.org/wiki/2007_NHL_entry_draft#Round_one'), + ('en.wikipedia.org_wiki_2011_T%C5%8Dhoku_earthquake_and_tsunami#Nuclear_power_plants.html', + 'https://en.wikipedia.org/wiki/2011_T%C5%8Dhoku_earthquake_and_tsunami#Nuclear_power_plants'), + ('en.wikipedia.org_wiki_2011_T%C5%8Dhoku_earthquake_and_tsunami.html', + 'https://en.wikipedia.org/wiki/2011_T%C5%8Dhoku_earthquake_and_tsunami'), + ('en.wikipedia.org_wiki_2013%E2%80%9314_Portland_Trail_Blazers_season#Playoffs.html', + 'https://en.wikipedia.org/wiki/2013%E2%80%9314_Portland_Trail_Blazers_season#Playoffs'), + ('en.wikipedia.org_wiki_2015_FIFA_Women%27s_World_Cup_knockout_stage.html', + 'https://en.wikipedia.org/wiki/2015_FIFA_Women%27s_World_Cup_knockout_stage'), + ('en.wikipedia.org_wiki_2015_World_Championships_in_Athletics_%E2%80%93_Women%27s_javelin_throw.html', + 'https://en.wikipedia.org/wiki/2015_World_Championships_in_Athletics_%E2%80%93_Women%27s_javelin_throw'), + ('en.wikipedia.org_wiki_2016_Australian_Open_%E2%80%93_Men%27s_singles.html', + 'https://en.wikipedia.org/wiki/2016_Australian_Open_%E2%80%93_Men%27s_singles'), + ('en.wikipedia.org_wiki_2016_CONCACAF_Women%27s_Olympic_Qualifying_Championship#Group_A.html', + 'https://en.wikipedia.org/wiki/2016_CONCACAF_Women%27s_Olympic_Qualifying_Championship#Group_A'), + ('en.wikipedia.org_wiki_2018%E2%80%9319_Brisbane_Roar_FC_season.html', + 'https://en.wikipedia.org/wiki/2018%E2%80%9319_Brisbane_Roar_FC_season'), + ('en.wikipedia.org_wiki_2018%E2%80%9319_Women%27s_Volleyball_Thailand_League.html', + 'https://en.wikipedia.org/wiki/2018%E2%80%9319_Women%27s_Volleyball_Thailand_League'), + ('en.wikipedia.org_wiki_2018_FIFA_World_Cup#Officiating.html', + 'https://en.wikipedia.org/wiki/2018_FIFA_World_Cup#Officiating'), + ('en.wikipedia.org_wiki_2019%E2%80%9320_NBA_season.html', 'https://en.wikipedia.org/wiki/2019%E2%80%9320_NBA_season'), + ('en.wikipedia.org_wiki_2020%E2%80%9321_Dallas_Mavericks_season.html', + 'https://en.wikipedia.org/wiki/2020%E2%80%9321_Dallas_Mavericks_season'), + ('en.wikipedia.org_wiki_2020%E2%80%9321_NBA_season.html', 'https://en.wikipedia.org/wiki/2020%E2%80%9321_NBA_season'), + ('en.wikipedia.org_wiki_2021_French_Open_%E2%80%93_Men%2527s_singles.html', + 'https://en.wikipedia.org/wiki/2021_French_Open_%E2%80%93_Men%27s_singles'), + ('en.wikipedia.org_wiki_2024_Yucat%C3%A1n_Open_%E2%80%93_Doubles.html', + 'https://en.wikipedia.org/wiki/2024_Yucat%C3%A1n_Open_%E2%80%93_Doubles'), + ('en.wikipedia.org_wiki_74th_Academy_Awards#Winners_and_nominees.html', + 'https://en.wikipedia.org/wiki/74th_Academy_Awards#Winners_and_nominees'), + ('en.wikipedia.org_wiki_95th_Academy_Awards#Awards.html', 'https://en.wikipedia.org/wiki/95th_Academy_Awards#Awards'), + ('en.wikipedia.org_wiki_AC_DC_discography#Live_albums.html', + 'https://en.wikipedia.org/wiki/AC/DC_discography#Live_albums'), + ('en.wikipedia.org_wiki_A_Brooklyn_State_of_Mind#Cast.html', + 'https://en.wikipedia.org/wiki/A_Brooklyn_State_of_Mind#Cast'), + ('en.wikipedia.org_wiki_Academy_Award_for_Best_Actor#1960s.html', + 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Actor#1960s'), + ('en.wikipedia.org_wiki_Academy_Award_for_Best_Animated_Feature#Multiple_wins_and_nominations.html', + 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Animated_Feature#Multiple_wins_and_nominations'), + ('en.wikipedia.org_wiki_Academy_Award_for_Best_Picture#1970s.html', + 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Picture#1970s'), + ('en.wikipedia.org_wiki_Academy_of_Science.html', 'https://en.wikipedia.org/wiki/Academy_of_Science'), + ('en.wikipedia.org_wiki_Albert_Bartholom%C3%A9#Main_works_(continued).html', + 'https://en.wikipedia.org/wiki/Albert_Bartholom%C3%A9#Main_works_(continued)'), + ('en.wikipedia.org_wiki_Alberto_Gin%C3%A9s_L%C3%B3pez.html', + 'https://en.wikipedia.org/wiki/Alberto_Gin%C3%A9s_L%C3%B3pez'), + ('en.wikipedia.org_wiki_Alex_Hern%C3%A1ndez_(tennis).html', + 'https://en.wikipedia.org/wiki/Alex_Hern%C3%A1ndez_(tennis)'), + ('en.wikipedia.org_wiki_Alexis_Clairaut#Focus_on_astronomical_motion.html', + 'https://en.wikipedia.org/wiki/Alexis_Clairaut#Focus_on_astronomical_motion'), + ('en.wikipedia.org_wiki_American_Athletic_Conference_Men%27s_Basketball_Player_of_the_Year.html', + 'https://en.wikipedia.org/wiki/American_Athletic_Conference_Men%27s_Basketball_Player_of_the_Year'), + ('en.wikipedia.org_wiki_Amplifier_(Dance_Exponents_album)#Track_listing.html', + 'https://en.wikipedia.org/wiki/Amplifier_(Dance_Exponents_album)#Track_listing'), + ('en.wikipedia.org_wiki_Andhra_Pradesh_(1956%E2%80%932014).html', + 'https://en.wikipedia.org/wiki/Andhra_Pradesh_(1956%E2%80%932014)'), + ('en.wikipedia.org_wiki_Andr%C3%A9_the_Giant.html', 'https://en.wikipedia.org/wiki/Andr%C3%A9_the_Giant'), + ('en.wikipedia.org_wiki_André_Silva_(footballer.html', 'https://en.wikipedia.org/wiki/Andr%C3%A9_Silva_(footballer)'), + ('en.wikipedia.org_wiki_Andy_Roddick_career_statistics#Singles__5_finals_(1–4).html', + 'https://en.wikipedia.org/wiki/Andy_Roddick_career_statistics#Singles__5_finals_(1%E2%80%934)'), + ('en.wikipedia.org_wiki_Anna_Stre%C5%BCy%C5%84ska.html', 'https://en.wikipedia.org/wiki/Anna_Stre%C5%BCy%C5%84ska'), + ('en.wikipedia.org_wiki_Anne.html', 'https://en.wikipedia.org/wiki/Anne'), + ('en.wikipedia.org_wiki_Apollo_11#Mission.html', 'https://en.wikipedia.org/wiki/Apollo_11#Mission'), + ('en.wikipedia.org_wiki_Archibald_Sinclair.html', 'https://en.wikipedia.org/wiki/Archibald_Sinclair'), + ('en.wikipedia.org_wiki_Arctic_Ocean#Climate.html', 'https://en.wikipedia.org/wiki/Arctic_Ocean#Climate'), + ('en.wikipedia.org_wiki_Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the,in%201930%2C%201990%20and%202014..html', + 'https://en.wikipedia.org/wiki/Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the,in%201930%2C%201990%20and%202014.'), + ('en.wikipedia.org_wiki_Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the.html', + 'https://en.wikipedia.org/wiki/Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the'), + ('en.wikipedia.org_wiki_Arlington.html', 'https://en.wikipedia.org/wiki/Arlington'), + ('en.wikipedia.org_wiki_Auckland_Island#Human_presence_on_the_island.html', + 'https://en.wikipedia.org/wiki/Auckland_Island#Human_presence_on_the_island'), + ('en.wikipedia.org_wiki_August_16#1901%E2%80%93present_2.html', + 'https://en.wikipedia.org/wiki/August_16#1901%E2%80%93present_2'), + ('en.wikipedia.org_wiki_Avalokite%C5%9Bvara.html', 'https://en.wikipedia.org/wiki/Avalokite%C5%9Bvara'), + ('en.wikipedia.org_wiki_Avukana_Buddha_statue#Location_and_appearance.html', + 'https://en.wikipedia.org/wiki/Avukana_Buddha_statue#Location_and_appearance'), + ('en.wikipedia.org_wiki_BTS#.html', 'https://en.wikipedia.org/wiki/BTS#'), + ('en.wikipedia.org_wiki_Baby_Vox#Studio_albums.html', 'https://en.wikipedia.org/wiki/Baby_Vox#Studio_albums'), + ('en.wikipedia.org_wiki_Baldur%27s_Gate_3#.html', 'https://en.wikipedia.org/wiki/Baldur%27s_Gate_3#'), + ('en.wikipedia.org_wiki_Balliol_College.html', 'https://en.wikipedia.org/wiki/Balliol_College'), + ('en.wikipedia.org_wiki_Bath,_Somerset#Culture.html', 'https://en.wikipedia.org/wiki/Bath,_Somerset#Culture'), + ('en.wikipedia.org_wiki_Bath.html', 'https://en.wikipedia.org/wiki/Bath'), + ('en.wikipedia.org_wiki_Bernhard_von_B%C3%BClow.html', 'https://en.wikipedia.org/wiki/Bernhard_von_B%C3%BClow'), + ('en.wikipedia.org_wiki_Bessemer.html', 'https://en.wikipedia.org/wiki/Bessemer'), + ('en.wikipedia.org_wiki_Beyonc%C3%A9#.html', 'https://en.wikipedia.org/wiki/Beyonc%C3%A9#'), + ('en.wikipedia.org_wiki_Big_Ben#Design.html', 'https://en.wikipedia.org/wiki/Big_Ben#Design'), + ('en.wikipedia.org_wiki_Big_Hit_Music#Groups.html', 'https://en.wikipedia.org/wiki/Big_Hit_Music#Groups'), + ('en.wikipedia.org_wiki_Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article,11.html', + 'https://en.wikipedia.org/wiki/Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article,11'), + ('en.wikipedia.org_wiki_Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article.html', + 'https://en.wikipedia.org/wiki/Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article'), + ('en.wikipedia.org_wiki_Bird#Anatomy_and_physiology.html', + 'https://en.wikipedia.org/wiki/Bird#Anatomy_and_physiology'), + ('en.wikipedia.org_wiki_Birut%C4%97_Galdikas.html', 'https://en.wikipedia.org/wiki/Birut%C4%97_Galdikas'), + ('en.wikipedia.org_wiki_Blink-182#Tours.html', 'https://en.wikipedia.org/wiki/Blink-182#Tours'), + ('en.wikipedia.org_wiki_Blue_moon#Blue_moon_dates.html', 'https://en.wikipedia.org/wiki/Blue_moon#Blue_moon_dates'), + ('en.wikipedia.org_wiki_Blyde_River_Canyon_Nature_Reserve#God.27s_Window.html', + 'https://en.wikipedia.org/wiki/Blyde_River_Canyon_Nature_Reserve#God.27s_Window'), + ('en.wikipedia.org_wiki_BoA#Discography.html', 'https://en.wikipedia.org/wiki/BoA#Discography'), + ('en.wikipedia.org_wiki_Boston_Marathon#Rosie_Ruiz,_the_impostor.html', + 'https://en.wikipedia.org/wiki/Boston_Marathon#Rosie_Ruiz,_the_impostor'), + ('en.wikipedia.org_wiki_Boston_Marathon#Rosie_Ruiz.html', 'https://en.wikipedia.org/wiki/Boston_Marathon#Rosie_Ruiz'), + ("en.wikipedia.org_wiki_Bowling_at_the_2011_Pan_American_Games_%E2%80%93_Women's_individual.html", + "https://en.wikipedia.org/wiki/Bowling_at_the_2011_Pan_American_Games_%E2%80%93_Women's_individual"), + ('en.wikipedia.org_wiki_Brazil_national_football_team#FIFA_World_Cup.html', + 'https://en.wikipedia.org/wiki/Brazil_national_football_team#FIFA_World_Cup'), + ('en.wikipedia.org_wiki_Bridgeton.html', 'https://en.wikipedia.org/wiki/Bridgeton'), + ('en.wikipedia.org_wiki_Britain%27s_Got_Talent.html', 'https://en.wikipedia.org/wiki/Britain%27s_Got_Talent'), + ('en.wikipedia.org_wiki_Bro%27Town.html', 'https://en.wikipedia.org/wiki/Bro%27Town'), + ('en.wikipedia.org_wiki_Broken_Arrow.html', 'https://en.wikipedia.org/wiki/Broken_Arrow'), + ('en.wikipedia.org_wiki_Bront%C3%AB_family.html', 'https://en.wikipedia.org/wiki/Bront%C3%AB_family'), + ('en.wikipedia.org_wiki_Brown_County.html', 'https://en.wikipedia.org/wiki/Brown_County'), + ('en.wikipedia.org_wiki_C%27%C3%A8_la_luna_mezzo_mare#Notable_recordings.html', + 'https://en.wikipedia.org/wiki/C%27%C3%A8_la_luna_mezzo_mare#Notable_recordings'), + ('en.wikipedia.org_wiki_Caldecott_Medal#Recipients.html', 'https://en.wikipedia.org/wiki/Caldecott_Medal#Recipients'), + ('en.wikipedia.org_wiki_Call_Me_Maybe#Year-end_charts.html', + 'https://en.wikipedia.org/wiki/Call_Me_Maybe#Year-end_charts'), + ('en.wikipedia.org_wiki_Calton.html', 'https://en.wikipedia.org/wiki/Calton'), + ('en.wikipedia.org_wiki_Capybara#.html', 'https://en.wikipedia.org/wiki/Capybara#'), + ('en.wikipedia.org_wiki_Carola_H%C3%A4ggkvist.html', 'https://en.wikipedia.org/wiki/Carola_H%C3%A4ggkvist'), + ('en.wikipedia.org_wiki_Charley%27s_Aunt.html', 'https://en.wikipedia.org/wiki/Charley%27s_Aunt'), + ('en.wikipedia.org_wiki_Charlotte_Bront%C3%AB.html', 'https://en.wikipedia.org/wiki/Charlotte_Bront%C3%AB'), + ('en.wikipedia.org_wiki_Charlotte_Checkers_(1956%E2%80%931977).html', + 'https://en.wikipedia.org/wiki/Charlotte_Checkers_(1956%E2%80%931977)'), + ('en.wikipedia.org_wiki_Chemotherapy#History.html', 'https://en.wikipedia.org/wiki/Chemotherapy#History'), + ('en.wikipedia.org_wiki_Cheyenne.html', 'https://en.wikipedia.org/wiki/Cheyenne'), + ('en.wikipedia.org_wiki_Chikhali.html', 'https://en.wikipedia.org/wiki/Chikhali'), + ('en.wikipedia.org_wiki_Chris_Benoit#Death.html', 'https://en.wikipedia.org/wiki/Chris_Benoit#Death'), + ('en.wikipedia.org_wiki_Chris_Columbus_(filmmaker)#Filmography.html', + 'https://en.wikipedia.org/wiki/Chris_Columbus_(filmmaker)#Filmography'), + ('en.wikipedia.org_wiki_Christie%27s.html', 'https://en.wikipedia.org/wiki/Christie%27s'), + ('en.wikipedia.org_wiki_Cinthia_Pi%C3%B1eiro#.html', 'https://en.wikipedia.org/wiki/Cinthia_Pi%C3%B1eiro#'), + ('en.wikipedia.org_wiki_Confucius#.html', 'https://en.wikipedia.org/wiki/Confucius#'), + ('en.wikipedia.org_wiki_Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton,productivity%20than%20manual%20cotton%20separation..html', + 'https://en.wikipedia.org/wiki/Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton,productivity%20than%20manual%20cotton%20separation.'), + ('en.wikipedia.org_wiki_Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton.html', + 'https://en.wikipedia.org/wiki/Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton'), + ('en.wikipedia.org_wiki_Cowboy_Bebop__Knockin%27_on_Heaven%27s_Door.html', + 'https://en.wikipedia.org/wiki/Cowboy_Bebop:_Knockin%27_on_Heaven%27s_Door'), + ('en.wikipedia.org_wiki_Cricket#Governance.html', 'https://en.wikipedia.org/wiki/Cricket#Governance'), + ('en.wikipedia.org_wiki_Croix_de_guerre_1914%E2%80%931918_(France)#Award_description.html', + 'https://en.wikipedia.org/wiki/Croix_de_guerre_1914%E2%80%931918_(France)#Award_description'), + ('en.wikipedia.org_wiki_Crossroads_(2002_film)#Reception.html', + 'https://en.wikipedia.org/wiki/Crossroads_(2002_film)#Reception'), + ('en.wikipedia.org_wiki_Danny_DeVito#Acting_credits_and_accolades.html', + 'https://en.wikipedia.org/wiki/Danny_DeVito#Acting_credits_and_accolades'), + ('en.wikipedia.org_wiki_Dax_Shepard#Film.html', 'https://en.wikipedia.org/wiki/Dax_Shepard#Film'), + ('en.wikipedia.org_wiki_Derek_Smith_(footballer.html', 'https://en.wikipedia.org/wiki/Derek_Smith_(footballer)'), + ('en.wikipedia.org_wiki_Devil%27s_Pass.html', 'https://en.wikipedia.org/wiki/Devil%27s_Pass'), + ('en.wikipedia.org_wiki_Dhatusena_of_Anuradhapura#Early_life_and_becoming_king.html', + 'https://en.wikipedia.org/wiki/Dhatusena_of_Anuradhapura#Early_life_and_becoming_king'), + ('en.wikipedia.org_wiki_Diana.html', 'https://en.wikipedia.org/wiki/Diana'), + ('en.wikipedia.org_wiki_Didier_Drogba#Honours.html', 'https://en.wikipedia.org/wiki/Didier_Drogba#Honours'), + ('en.wikipedia.org_wiki_Diego_Costa#Spain.html', 'https://en.wikipedia.org/wiki/Diego_Costa#Spain'), + ('en.wikipedia.org_wiki_Diego_L%C3%B3pez_V_de_Haro.html', 'https://en.wikipedia.org/wiki/Diego_L%C3%B3pez_V_de_Haro'), + ('en.wikipedia.org_wiki_Dippin%27_Dots.html', 'https://en.wikipedia.org/wiki/Dippin%27_Dots'), + ('en.wikipedia.org_wiki_Disneynature#Filmography.html', 'https://en.wikipedia.org/wiki/Disneynature#Filmography'), + ('en.wikipedia.org_wiki_Djokovic%E2%80%93Murray_rivalry.html', + 'https://en.wikipedia.org/wiki/Djokovic%E2%80%93Murray_rivalry'), + ('en.wikipedia.org_wiki_Donahoe.html', 'https://en.wikipedia.org/wiki/Donahoe'), + ('en.wikipedia.org_wiki_Dr._Quinn.html', 'https://en.wikipedia.org/wiki/Dr._Quinn'), + ('en.wikipedia.org_wiki_Elon_Musk#Personal_life.html', 'https://en.wikipedia.org/wiki/Elon_Musk#Personal_life'), + ('en.wikipedia.org_wiki_Emirates_(airline)#Services.html', + 'https://en.wikipedia.org/wiki/Emirates_(airline)#Services'), + ('en.wikipedia.org_wiki_Emmanuel_Lubezki#Feature_film.html', + 'https://en.wikipedia.org/wiki/Emmanuel_Lubezki#Feature_film'), + ('en.wikipedia.org_wiki_Emmy_Rossum#Awards_and_nominations.html', + 'https://en.wikipedia.org/wiki/Emmy_Rossum#Awards_and_nominations'), + ('en.wikipedia.org_wiki_England#Geography.html', 'https://en.wikipedia.org/wiki/England#Geography'), + ('en.wikipedia.org_wiki_Estadio_Jos%C3%A9_Mart%C3%ADn_Olaeta.html', + 'https://en.wikipedia.org/wiki/Estadio_Jos%C3%A9_Mart%C3%ADn_Olaeta'), + ('en.wikipedia.org_wiki_Ethereum#Ether.html', 'https://en.wikipedia.org/wiki/Ethereum#Ether'), + ('en.wikipedia.org_wiki_Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin,restrictions%20were%20overwhelmed%20and%20discarded..html', + 'https://en.wikipedia.org/wiki/Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin,restrictions%20were%20overwhelmed%20and%20discarded.'), + ('en.wikipedia.org_wiki_Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin.html', + 'https://en.wikipedia.org/wiki/Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin'), + ('en.wikipedia.org_wiki_Fastest_animals#Fish.html', 'https://en.wikipedia.org/wiki/Fastest_animals#Fish'), + ('en.wikipedia.org_wiki_Federative_units_of_Brazil#_media_File_Brazil,_administrative_divisions_(states)_-_en_-_colored.svg.html', + 'https://en.wikipedia.org/wiki/Federative_units_of_Brazil#_media_File_Brazil,_administrative_divisions_(states)_-_en_-_colored.svg'), + ('en.wikipedia.org_wiki_Federative_units_of_Brazil#_media_File_Brazil.html', + 'https://en.wikipedia.org/wiki/Federative_units_of_Brazil#_media_File_Brazil'), + ('en.wikipedia.org_wiki_Five_Nights_at_Freddy%27s.html', 'https://en.wikipedia.org/wiki/Five_Nights_at_Freddy%27s'), + ('en.wikipedia.org_wiki_Fleetwood_Mac#Grammy_Awards.html', + 'https://en.wikipedia.org/wiki/Fleetwood_Mac#Grammy_Awards'), + ('en.wikipedia.org_wiki_Flight_of_the_Earls#Journey.html', + 'https://en.wikipedia.org/wiki/Flight_of_the_Earls#Journey'), + ('en.wikipedia.org_wiki_Florida_A%26M_University.html', 'https://en.wikipedia.org/wiki/Florida_A%26M_University'), + ('en.wikipedia.org_wiki_Florida_Atlantic_Owls_men%27s_basketball.html', + 'https://en.wikipedia.org/wiki/Florida_Atlantic_Owls_men%27s_basketball'), + ('en.wikipedia.org_wiki_For_Whom_the_Bell_Tolls#Pulitzer_Prize_snub.html', + 'https://en.wikipedia.org/wiki/For_Whom_the_Bell_Tolls#Pulitzer_Prize_snub'), + ('en.wikipedia.org_wiki_Ford_Motor_Company#Sales_numbers.html', + 'https://en.wikipedia.org/wiki/Ford_Motor_Company#Sales_numbers'), + ('en.wikipedia.org_wiki_Franz_Kafka#Stories.html', 'https://en.wikipedia.org/wiki/Franz_Kafka#Stories'), + ('en.wikipedia.org_wiki_Friends_%26_Relatives.html', 'https://en.wikipedia.org/wiki/Friends_%26_Relatives'), + ('en.wikipedia.org_wiki_G%C3%A9za_K%C3%A1das.html', 'https://en.wikipedia.org/wiki/G%C3%A9za_K%C3%A1das'), + ('en.wikipedia.org_wiki_Galileo%27s_middle_finger#Exhibition_history.html', + 'https://en.wikipedia.org/wiki/Galileo%27s_middle_finger#Exhibition_history'), + ('en.wikipedia.org_wiki_Garypidae#Genera.html', 'https://en.wikipedia.org/wiki/Garypidae#Genera'), + ('en.wikipedia.org_wiki_Geffen_Records#History.html', 'https://en.wikipedia.org/wiki/Geffen_Records#History'), + ('en.wikipedia.org_wiki_George_Harrison#Discography.html', + 'https://en.wikipedia.org/wiki/George_Harrison#Discography'), + ('en.wikipedia.org_wiki_George_Howell_(entrepreneur)#The_Coffee_Connection.html', + 'https://en.wikipedia.org/wiki/George_Howell_(entrepreneur)#The_Coffee_Connection'), + ('en.wikipedia.org_wiki_George_Talbot.html', 'https://en.wikipedia.org/wiki/George_Talbot'), + ('en.wikipedia.org_wiki_George_Washington#Personal_life.html', + 'https://en.wikipedia.org/wiki/George_Washington#Personal_life'), + ('en.wikipedia.org_wiki_Girl_Scouts_of_the_USA#Presidents.html', + 'https://en.wikipedia.org/wiki/Girl_Scouts_of_the_USA#Presidents'), + ('en.wikipedia.org_wiki_Golden_Globe_Award_for_Best_Actor_in_a_Motion_Picture_%E2%80%93_Drama.html', + 'https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_in_a_Motion_Picture_%E2%80%93_Drama'), + ('en.wikipedia.org_wiki_Golden_Retriever#Health.html', 'https://en.wikipedia.org/wiki/Golden_Retriever#Health'), + ('en.wikipedia.org_wiki_Goodbye.html', 'https://en.wikipedia.org/wiki/Goodbye'), + ('en.wikipedia.org_wiki_Gothic_Revival_architecture#Roots.html', + 'https://en.wikipedia.org/wiki/Gothic_Revival_architecture#Roots'), + ('en.wikipedia.org_wiki_Grace%27s_High_Falls.html', 'https://en.wikipedia.org/wiki/Grace%27s_High_Falls'), + ('en.wikipedia.org_wiki_Grammy_Award_for_Best_New_Artist#1960s.html', + 'https://en.wikipedia.org/wiki/Grammy_Award_for_Best_New_Artist#1960s'), + ('en.wikipedia.org_wiki_Grammy_Award_for_Song_of_the_Year#2000s.html', + 'https://en.wikipedia.org/wiki/Grammy_Award_for_Song_of_the_Year#2000s'), + ('en.wikipedia.org_wiki_Grand_Slam_(tennis)#.html', 'https://en.wikipedia.org/wiki/Grand_Slam_(tennis)#'), + ('en.wikipedia.org_wiki_Grateful_Dead#Main_career_(1967–1995).html', + 'https://en.wikipedia.org/wiki/Grateful_Dead#Main_career_(1967%E2%80%931995)'), + ('en.wikipedia.org_wiki_Grenoble#Population.html', 'https://en.wikipedia.org/wiki/Grenoble#Population'), + ('en.wikipedia.org_wiki_Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018,serve%20non%2Dconsecutive%20presidential%20terms..html', + 'https://en.wikipedia.org/wiki/Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018,serve%20non%2Dconsecutive%20presidential%20terms.'), + ('en.wikipedia.org_wiki_Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018.html', + 'https://en.wikipedia.org/wiki/Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018'), + ('en.wikipedia.org_wiki_Gymnastics_at_the_1992_Summer_Olympics_%E2%80%93_Women%27s_artistic_team_all-around.html', + 'https://en.wikipedia.org/wiki/Gymnastics_at_the_1992_Summer_Olympics_%E2%80%93_Women%27s_artistic_team_all-around'), + ('en.wikipedia.org_wiki_H%C3%A4n.html', 'https://en.wikipedia.org/wiki/H%C3%A4n'), + ('en.wikipedia.org_wiki_H%C3%B4tel_Matignon.html', 'https://en.wikipedia.org/wiki/H%C3%B4tel_Matignon'), + ('en.wikipedia.org_wiki_Halley%27s_Comet.html', 'https://en.wikipedia.org/wiki/Halley%27s_Comet'), + ('en.wikipedia.org_wiki_Hans_Zimmer#Grammy_Awards.html', 'https://en.wikipedia.org/wiki/Hans_Zimmer#Grammy_Awards'), + ('en.wikipedia.org_wiki_Happy_Days#Characters.html', 'https://en.wikipedia.org/wiki/Happy_Days#Characters'), + ('en.wikipedia.org_wiki_Harper%27s_Magazine.html', 'https://en.wikipedia.org/wiki/Harper%27s_Magazine'), + ('en.wikipedia.org_wiki_Harry_Potter_and_the_Philosopher%27s_Stone.html', + 'https://en.wikipedia.org/wiki/Harry_Potter_and_the_Philosopher%27s_Stone'), + ('en.wikipedia.org_wiki_Hatfield_College.html', 'https://en.wikipedia.org/wiki/Hatfield_College'), + ('en.wikipedia.org_wiki_Hayao_Miyazaki#Views.html', 'https://en.wikipedia.org/wiki/Hayao_Miyazaki#Views'), + ('en.wikipedia.org_wiki_Heaven%27s_Gate_(religious_group)#Nike_Decades.html', + 'https://en.wikipedia.org/wiki/Heaven%27s_Gate_(religious_group)#Nike_Decades'), + ('en.wikipedia.org_wiki_Hemangiosarcoma#Treatments.html', 'https://en.wikipedia.org/wiki/Hemangiosarcoma#Treatments'), + ('en.wikipedia.org_wiki_Hermann_G%C3%B6ring.html', 'https://en.wikipedia.org/wiki/Hermann_G%C3%B6ring'), + ('en.wikipedia.org_wiki_Herzog_%26_de_Meuron.html', 'https://en.wikipedia.org/wiki/Herzog_%26_de_Meuron'), + ('en.wikipedia.org_wiki_Hind%27s_Hall.html', 'https://en.wikipedia.org/wiki/Hind%27s_Hall'), + ('en.wikipedia.org_wiki_Home_Alone_2__Lost_in_New_York#Cast.html', + 'https://en.wikipedia.org/wiki/Home_Alone_2:_Lost_in_New_York#Cast'), + ('en.wikipedia.org_wiki_Hot_Rock_%26_Alternative_Songs.html', + 'https://en.wikipedia.org/wiki/Hot_Rock_%26_Alternative_Songs'), + ('en.wikipedia.org_wiki_Hoxie.html', 'https://en.wikipedia.org/wiki/Hoxie'), + ('en.wikipedia.org_wiki_Hudson%27s_Bay_Company.html', 'https://en.wikipedia.org/wiki/Hudson%27s_Bay_Company'), + ('en.wikipedia.org_wiki_Hugh_Hefner#Early_life_and_education.html', + 'https://en.wikipedia.org/wiki/Hugh_Hefner#Early_life_and_education'), + ('en.wikipedia.org_wiki_ICC_men%27s_player_rankings#Top_10_Test_batsmen.html', + 'https://en.wikipedia.org/wiki/ICC_men%27s_player_rankings#Top_10_Test_batsmen'), + ('en.wikipedia.org_wiki_IPad#.html', 'https://en.wikipedia.org/wiki/IPad#'), + ('en.wikipedia.org_wiki_IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral,released%20on%20November%203%2C%202017..html', + 'https://en.wikipedia.org/wiki/IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral,released%20on%20November%203%2C%202017.'), + ('en.wikipedia.org_wiki_IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral.html', + 'https://en.wikipedia.org/wiki/IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral'), + ('en.wikipedia.org_wiki_Ice_hockey_at_the_2010_Winter_Olympics_%E2%80%93_Men%27s_tournament.html', + 'https://en.wikipedia.org/wiki/Ice_hockey_at_the_2010_Winter_Olympics_%E2%80%93_Men%27s_tournament'), + ('en.wikipedia.org_wiki_If_You_Want_Blood_You%27ve_Got_It.html', + 'https://en.wikipedia.org/wiki/If_You_Want_Blood_You%27ve_Got_It'), + ('en.wikipedia.org_wiki_Iga_%C5%9Awi%C4%85tek.html', 'https://en.wikipedia.org/wiki/Iga_%C5%9Awi%C4%85tek'), + ('en.wikipedia.org_wiki_Infinite_Jest#Adaptations.html', 'https://en.wikipedia.org/wiki/Infinite_Jest#Adaptations'), + ('en.wikipedia.org_wiki_Interstate_94_in_Michigan#.html', 'https://en.wikipedia.org/wiki/Interstate_94_in_Michigan#'), + ('en.wikipedia.org_wiki_Is%C3%A8re#Principal_towns.html', 'https://en.wikipedia.org/wiki/Is%C3%A8re#Principal_towns'), + ('en.wikipedia.org_wiki_Istiklal_Mosque.html', 'https://en.wikipedia.org/wiki/Istiklal_Mosque'), + ('en.wikipedia.org_wiki_It%27s_Always_Sunny_in_Philadelphia.html', + 'https://en.wikipedia.org/wiki/It%27s_Always_Sunny_in_Philadelphia'), + ('en.wikipedia.org_wiki_Iv%C3%A1n_Rodr%C3%ADguez.html', 'https://en.wikipedia.org/wiki/Iv%C3%A1n_Rodr%C3%ADguez'), + ('en.wikipedia.org_wiki_J%C3%BCrgen_Warnke.html', 'https://en.wikipedia.org/wiki/J%C3%BCrgen_Warnke'), + ('en.wikipedia.org_wiki_Jackie_Robinson#.html', 'https://en.wikipedia.org/wiki/Jackie_Robinson#'), + ('en.wikipedia.org_wiki_James_%22J.T.%22_Taylor.html', 'https://en.wikipedia.org/wiki/James_%22J.T.%22_Taylor'), + ('en.wikipedia.org_wiki_James_Cameron#Filmography.html', 'https://en.wikipedia.org/wiki/James_Cameron#Filmography'), + ('en.wikipedia.org_wiki_Jane_Austen#List_of_works.html', 'https://en.wikipedia.org/wiki/Jane_Austen#List_of_works'), + ('en.wikipedia.org_wiki_Japanese_aircraft_carrier_Hiy%C5%8D.html', + 'https://en.wikipedia.org/wiki/Japanese_aircraft_carrier_Hiy%C5%8D'), + ('en.wikipedia.org_wiki_Ji%C5%99%C3%AD_Bro%C5%BEek.html', 'https://en.wikipedia.org/wiki/Ji%C5%99%C3%AD_Bro%C5%BEek'), + ('en.wikipedia.org_wiki_Jo%C3%ABl_Retornaz.html', 'https://en.wikipedia.org/wiki/Jo%C3%ABl_Retornaz'), + ('en.wikipedia.org_wiki_Joe_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Joe_Jonas#Personal_life'), + ('en.wikipedia.org_wiki_Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November,actor%2C%20comedian%20and%20television%20presenter..html', + 'https://en.wikipedia.org/wiki/Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November,actor%2C%20comedian%20and%20television%20presenter.'), + ('en.wikipedia.org_wiki_Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November.html', + 'https://en.wikipedia.org/wiki/Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November'), + ('en.wikipedia.org_wiki_John_Snow#.html', 'https://en.wikipedia.org/wiki/John_Snow#'), + ('en.wikipedia.org_wiki_John_of_Lancaster.html', 'https://en.wikipedia.org/wiki/John_of_Lancaster'), + ('en.wikipedia.org_wiki_Jonas_Brothers#Members.html', 'https://en.wikipedia.org/wiki/Jonas_Brothers#Members'), + ('en.wikipedia.org_wiki_Jos%C3%A9_Loiola.html', 'https://en.wikipedia.org/wiki/Jos%C3%A9_Loiola'), + ('en.wikipedia.org_wiki_Jos%C3%A9_Mar%C3%ADa_Arizmendiarrieta.html', + 'https://en.wikipedia.org/wiki/Jos%C3%A9_Mar%C3%ADa_Arizmendiarrieta'), + ('en.wikipedia.org_wiki_José_Saramago#.html', 'https://en.wikipedia.org/wiki/Jos%C3%A9_Saramago#'), + ('en.wikipedia.org_wiki_Juan_Gonz%C3%A1lez_(baseball).html', + 'https://en.wikipedia.org/wiki/Juan_Gonz%C3%A1lez_(baseball)'), + ('en.wikipedia.org_wiki_Juneau.html', 'https://en.wikipedia.org/wiki/Juneau'), + ('en.wikipedia.org_wiki_K%C5%8Dnan_Railway_K%C5%8Dnan_Line.html', + 'https://en.wikipedia.org/wiki/K%C5%8Dnan_Railway_K%C5%8Dnan_Line'), + ('en.wikipedia.org_wiki_Kala_Wewa#History.html', 'https://en.wikipedia.org/wiki/Kala_Wewa#History'), + ('en.wikipedia.org_wiki_Kelley_O%27Hara.html', 'https://en.wikipedia.org/wiki/Kelley_O%27Hara'), + ('en.wikipedia.org_wiki_Kennebec_County.html', 'https://en.wikipedia.org/wiki/Kennebec_County'), + ('en.wikipedia.org_wiki_Kente_cloth#cite_note-CNN-2020-06-08-18.html', + 'https://en.wikipedia.org/wiki/Kente_cloth#cite_note-CNN-2020-06-08-18'), + ('en.wikipedia.org_wiki_Kevin_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Kevin_Jonas#Personal_life'), + ('en.wikipedia.org_wiki_Key_West#_~_text=The%20southernmost%20location%20that%20the,apart%20at%20their%20closest%20points..html', + 'https://en.wikipedia.org/wiki/Key_West#_~_text=The%20southernmost%20location%20that%20the,apart%20at%20their%20closest%20points.'), + ('en.wikipedia.org_wiki_Key_West#_~_text=The%20southernmost%20location%20that%20the.html', + 'https://en.wikipedia.org/wiki/Key_West#_~_text=The%20southernmost%20location%20that%20the'), + ('en.wikipedia.org_wiki_Kill_%27Em_All.html', 'https://en.wikipedia.org/wiki/Kill_%27Em_All'), + ('en.wikipedia.org_wiki_King%27s_Service_Medal.html', 'https://en.wikipedia.org/wiki/King%27s_Service_Medal'), + ('en.wikipedia.org_wiki_Kundavai_Pir%C4%81ttiy%C4%81r.html', + 'https://en.wikipedia.org/wiki/Kundavai_Pir%C4%81ttiy%C4%81r'), + ('en.wikipedia.org_wiki_LaMarcus_Aldridge#2013%E2%80%9314_season.html', + 'https://en.wikipedia.org/wiki/LaMarcus_Aldridge#2013%E2%80%9314_season'), + ('en.wikipedia.org_wiki_La_Casita-Garciasville.html', 'https://en.wikipedia.org/wiki/La_Casita-Garciasville'), + ('en.wikipedia.org_wiki_La_La_Land#Cast.html', 'https://en.wikipedia.org/wiki/La_La_Land#Cast'), + ('en.wikipedia.org_wiki_La_Llorona#Literature.html', 'https://en.wikipedia.org/wiki/La_Llorona#Literature'), + ('en.wikipedia.org_wiki_La_boh%C3%A8me.html', 'https://en.wikipedia.org/wiki/La_boh%C3%A8me'), + ('en.wikipedia.org_wiki_Latrobe.html', 'https://en.wikipedia.org/wiki/Latrobe'), + ('en.wikipedia.org_wiki_Laurentian_Library#Architecture.html', + 'https://en.wikipedia.org/wiki/Laurentian_Library#Architecture'), + ('en.wikipedia.org_wiki_Lauryn_Hill#Early_life.html', 'https://en.wikipedia.org/wiki/Lauryn_Hill#Early_life'), + ('en.wikipedia.org_wiki_Law_%26_Order__Special_Victims_Unit.html', + 'https://en.wikipedia.org/wiki/Law_%26_Order__Special_Victims_Unit'), + ('en.wikipedia.org_wiki_Legion_of_Honour#Legal_status_and_leadership.html', + 'https://en.wikipedia.org/wiki/Legion_of_Honour#Legal_status_and_leadership'), + ('en.wikipedia.org_wiki_Levi_Strauss_%26_Co.#Blue_jeans_era_(1960s%E2%80%931980s).html', + 'https://en.wikipedia.org/wiki/Levi_Strauss_%26_Co.#Blue_jeans_era_(1960s%E2%80%931980s)'), + ('en.wikipedia.org_wiki_Life_%26_Casualty_Tower.html', 'https://en.wikipedia.org/wiki/Life_%26_Casualty_Tower'), + ('en.wikipedia.org_wiki_Lincoln.html', 'https://en.wikipedia.org/wiki/Lincoln'), + ('en.wikipedia.org_wiki_List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became,major%20party%2C%20namely%20the%20Democr.html', + 'https://en.wikipedia.org/wiki/List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became,major%20party%2C%20namely%20the%20Democr'), + ('en.wikipedia.org_wiki_List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became.html', + 'https://en.wikipedia.org/wiki/List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became'), + ('en.wikipedia.org_wiki_List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling,nomination%20at%20the%20Grammy%20Awards..html', + 'https://en.wikipedia.org/wiki/List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling,nomination%20at%20the%20Grammy%20Awards.'), + ('en.wikipedia.org_wiki_List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling.html', + 'https://en.wikipedia.org/wiki/List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling'), + ('en.wikipedia.org_wiki_List_of_Copa_Am%C3%A9rica_finals#Finals.html', + 'https://en.wikipedia.org/wiki/List_of_Copa_Am%C3%A9rica_finals#Finals'), + ('en.wikipedia.org_wiki_List_of_Hot_Ones_episodes#Season_23_(2024).html', + 'https://en.wikipedia.org/wiki/List_of_Hot_Ones_episodes#Season_23_(2024)'), + ('en.wikipedia.org_wiki_List_of_How_I_Met_Your_Mother_episodes#Season_3_(2007%E2%80%9308).html', + 'https://en.wikipedia.org/wiki/List_of_How_I_Met_Your_Mother_episodes#Season_3_(2007%E2%80%9308)'), + ('en.wikipedia.org_wiki_List_of_Liverpool_F.C._managers#Managers.html', + 'https://en.wikipedia.org/wiki/List_of_Liverpool_F.C._managers#Managers'), + ('en.wikipedia.org_wiki_List_of_MLS_Cup_finals#Results_by_team.html', + 'https://en.wikipedia.org/wiki/List_of_MLS_Cup_finals#Results_by_team'), + ('en.wikipedia.org_wiki_List_of_New_Girl_episodes#Season_2_(2012%E2%80%9313).html', + 'https://en.wikipedia.org/wiki/List_of_New_Girl_episodes#Season_2_(2012%E2%80%9313)'), + ('en.wikipedia.org_wiki_List_of_New_Zealand_Test_cricket_records#Most_career_runs.html', + 'https://en.wikipedia.org/wiki/List_of_New_Zealand_Test_cricket_records#Most_career_runs'), + ('en.wikipedia.org_wiki_List_of_Nobel_laureates_in_Literature#1960.html', + 'https://en.wikipedia.org/wiki/List_of_Nobel_laureates_in_Literature#1960'), + ('en.wikipedia.org_wiki_List_of_Northern_Ireland_international_footballers#List_of_players.html', + 'https://en.wikipedia.org/wiki/List_of_Northern_Ireland_international_footballers#List_of_players'), + ('en.wikipedia.org_wiki_List_of_S%26P_500_companies.html', + 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'), + ('en.wikipedia.org_wiki_List_of_Star_Wars_film_actors#Introduced_in_The_Skywalker_Saga.html', + 'https://en.wikipedia.org/wiki/List_of_Star_Wars_film_actors#Introduced_in_The_Skywalker_Saga'), + ('en.wikipedia.org_wiki_List_of_Walt_Disney_Animation_Studios_films#ep29.html', + 'https://en.wikipedia.org/wiki/List_of_Walt_Disney_Animation_Studios_films#ep29'), + ('en.wikipedia.org_wiki_List_of_Wildlife_Species_at_Risk_(Canada)#Amphibians.html', + 'https://en.wikipedia.org/wiki/List_of_Wildlife_Species_at_Risk_(Canada)#Amphibians'), + ('en.wikipedia.org_wiki_List_of_World_Series_champions#World_Series_results.html', + 'https://en.wikipedia.org/wiki/List_of_World_Series_champions#World_Series_results'), + ('en.wikipedia.org_wiki_List_of_awards_and_nominations_received_by_Katy_Perry#Awards_and_nominations.html', + 'https://en.wikipedia.org/wiki/List_of_awards_and_nominations_received_by_Katy_Perry#Awards_and_nominations'), + ('en.wikipedia.org_wiki_List_of_awards_and_nominations_received_by_Taylor_Swift#Honorary_degree.html', + 'https://en.wikipedia.org/wiki/List_of_awards_and_nominations_received_by_Taylor_Swift#Honorary_degree'), + ('en.wikipedia.org_wiki_List_of_minor_planets__8001%E2%80%939000#323c.html', + 'https://en.wikipedia.org/wiki/List_of_minor_planets:_8001%E2%80%939000#323c'), + ('en.wikipedia.org_wiki_List_of_presidents_of_France#Presidents_2.html', + 'https://en.wikipedia.org/wiki/List_of_presidents_of_France#Presidents_2'), + ('en.wikipedia.org_wiki_List_of_presidents_of_the_United_States#Presidents.html', + 'https://en.wikipedia.org/wiki/List_of_presidents_of_the_United_States#Presidents'), + ('en.wikipedia.org_wiki_List_of_prime_ministers_of_Australia#.html', + 'https://en.wikipedia.org/wiki/List_of_prime_ministers_of_Australia#'), + ('en.wikipedia.org_wiki_List_of_prime_ministers_of_Canada#Prime_ministers.html', + 'https://en.wikipedia.org/wiki/List_of_prime_ministers_of_Canada#Prime_ministers'), + ('en.wikipedia.org_wiki_List_of_vetoed_United_Nations_Security_Council_resolutions#Resolutions.html', + 'https://en.wikipedia.org/wiki/List_of_vetoed_United_Nations_Security_Council_resolutions#Resolutions'), + ('en.wikipedia.org_wiki_List_of_waterfalls_by_height#By_overall_height.html', + 'https://en.wikipedia.org/wiki/List_of_waterfalls_by_height#By_overall_height'), + ('en.wikipedia.org_wiki_Lists_of_airports#By_code.html', 'https://en.wikipedia.org/wiki/Lists_of_airports#By_code'), + ('en.wikipedia.org_wiki_Lists_of_earthquakes#Largest_earthquakes_by_magnitude.html', + 'https://en.wikipedia.org/wiki/Lists_of_earthquakes#Largest_earthquakes_by_magnitude'), + ('en.wikipedia.org_wiki_Literary_and_Philosophical_Society_of_Newcastle_upon_Tyne#.html', + 'https://en.wikipedia.org/wiki/Literary_and_Philosophical_Society_of_Newcastle_upon_Tyne#'), + ('en.wikipedia.org_wiki_London_Broncos#1994%E2%80%932005__Broncos_and_Super_League.html', + 'https://en.wikipedia.org/wiki/London_Broncos#1994%E2%80%932005__Broncos_and_Super_League'), + ('en.wikipedia.org_wiki_Louis_XVI#.html', 'https://en.wikipedia.org/wiki/Louis_XVI#'), + ('en.wikipedia.org_wiki_Lynn_Rogers_(politician)#Career.html', + 'https://en.wikipedia.org/wiki/Lynn_Rogers_(politician)#Career'), + ('en.wikipedia.org_wiki_Ma%C5%82gorzata_Ro%C5%BCniecka.html', + 'https://en.wikipedia.org/wiki/Ma%C5%82gorzata_Ro%C5%BCniecka'), + ('en.wikipedia.org_wiki_MacArthur_Fellows_Program#.html', 'https://en.wikipedia.org/wiki/MacArthur_Fellows_Program#'), + ('en.wikipedia.org_wiki_Mali#Ethnic_groups.html', 'https://en.wikipedia.org/wiki/Mali#Ethnic_groups'), + ('en.wikipedia.org_wiki_Marathon_County.html', 'https://en.wikipedia.org/wiki/Marathon_County'), + ('en.wikipedia.org_wiki_Margaret_Mansfield.html', 'https://en.wikipedia.org/wiki/Margaret_Mansfield'), + ('en.wikipedia.org_wiki_Mark_Harvey_(arachnologist)#Achievements,_awards_and_recognition.html', + 'https://en.wikipedia.org/wiki/Mark_Harvey_(arachnologist)#Achievements,_awards_and_recognition'), + ('en.wikipedia.org_wiki_Mark_Harvey_(arachnologist)#Achievements.html', + 'https://en.wikipedia.org/wiki/Mark_Harvey_(arachnologist)#Achievements'), + ('en.wikipedia.org_wiki_Mark_O%27Halloran_(rugby_league).html', + 'https://en.wikipedia.org/wiki/Mark_O%27Halloran_(rugby_league)'), + ('en.wikipedia.org_wiki_McDonald%27s.html', 'https://en.wikipedia.org/wiki/McDonald%27s'), + ('en.wikipedia.org_wiki_Meat_Loaf#Personal_life.html', 'https://en.wikipedia.org/wiki/Meat_Loaf#Personal_life'), + ('en.wikipedia.org_wiki_Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as,former%20members%20of%20the%20Commonwealth..html', + 'https://en.wikipedia.org/wiki/Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as,former%20members%20of%20the%20Commonwealth.'), + ('en.wikipedia.org_wiki_Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as.html', + 'https://en.wikipedia.org/wiki/Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as'), + ('en.wikipedia.org_wiki_Miasma_theory#.html', 'https://en.wikipedia.org/wiki/Miasma_theory#'), + ('en.wikipedia.org_wiki_Michelson%E2%80%93Morley_experiment.html', + 'https://en.wikipedia.org/wiki/Michelson%E2%80%93Morley_experiment'), + ('en.wikipedia.org_wiki_Milo%C5%A1_Beleslin.html', 'https://en.wikipedia.org/wiki/Milo%C5%A1_Beleslin'), + ('en.wikipedia.org_wiki_Minamidait%C5%8Djima.html', 'https://en.wikipedia.org/wiki/Minamidait%C5%8Djima'), + ('en.wikipedia.org_wiki_Mona_Lisa#Refuge,_theft,_and_vandalism.html', + 'https://en.wikipedia.org/wiki/Mona_Lisa#Refuge,_theft,_and_vandalism'), + ('en.wikipedia.org_wiki_Mona_Lisa#Refuge.html', 'https://en.wikipedia.org/wiki/Mona_Lisa#Refuge'), + ('en.wikipedia.org_wiki_Morgana_King#Film_debut.html', 'https://en.wikipedia.org/wiki/Morgana_King#Film_debut'), + ('en.wikipedia.org_wiki_Mount_Rushmore#History.html', 'https://en.wikipedia.org/wiki/Mount_Rushmore#History'), + ('en.wikipedia.org_wiki_Murray_State_University#Former_Presidents_of_the_University.html', + 'https://en.wikipedia.org/wiki/Murray_State_University#Former_Presidents_of_the_University'), + ('en.wikipedia.org_wiki_NCAA_Division_III_men%27s_cross_country_championships.html', + 'https://en.wikipedia.org/wiki/NCAA_Division_III_men%27s_cross_country_championships'), + ('en.wikipedia.org_wiki_NCT_(group)#Members.html', 'https://en.wikipedia.org/wiki/NCT_(group)#Members'), + ('en.wikipedia.org_wiki_Nancy_Farmer#Bibliography.html', 'https://en.wikipedia.org/wiki/Nancy_Farmer#Bibliography'), + ('en.wikipedia.org_wiki_Nashville.html', 'https://en.wikipedia.org/wiki/Nashville'), + ('en.wikipedia.org_wiki_Navy_%E2%80%93_Merchant_Marine_Memorial.html', + 'https://en.wikipedia.org/wiki/Navy_%E2%80%93_Merchant_Marine_Memorial'), + ('en.wikipedia.org_wiki_Nelson_Mandela#Imprisonment.html', + 'https://en.wikipedia.org/wiki/Nelson_Mandela#Imprisonment'), + ('en.wikipedia.org_wiki_New_Girl#Episodes.html', 'https://en.wikipedia.org/wiki/New_Girl#Episodes'), + ('en.wikipedia.org_wiki_Nick_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Nick_Jonas#Personal_life'), + ('en.wikipedia.org_wiki_Nike.html', 'https://en.wikipedia.org/wiki/Nike'), + ('en.wikipedia.org_wiki_Nim_Chimpsky#Quotations.html', 'https://en.wikipedia.org/wiki/Nim_Chimpsky#Quotations'), + ('en.wikipedia.org_wiki_Nineteenth_Dynasty_of_Egypt#Pharaohs_of_the_19th_Dynasty.html', + 'https://en.wikipedia.org/wiki/Nineteenth_Dynasty_of_Egypt#Pharaohs_of_the_19th_Dynasty'), + ('en.wikipedia.org_wiki_Noam_Chomsky#In_academia.html', 'https://en.wikipedia.org/wiki/Noam_Chomsky#In_academia'), + ('en.wikipedia.org_wiki_Nobel_Prize#Multiple_laureates.html', + 'https://en.wikipedia.org/wiki/Nobel_Prize#Multiple_laureates'), + ('en.wikipedia.org_wiki_Norman.html', 'https://en.wikipedia.org/wiki/Norman'), + ('en.wikipedia.org_wiki_Northern_California#Cities.html', 'https://en.wikipedia.org/wiki/Northern_California#Cities'), + ('en.wikipedia.org_wiki_Novake,_Polj%C4%8Dane.html', 'https://en.wikipedia.org/wiki/Novake,_Polj%C4%8Dane'), + ('en.wikipedia.org_wiki_Ol%27_Dirty_Bastard.html', 'https://en.wikipedia.org/wiki/Ol%27_Dirty_Bastard'), + ('en.wikipedia.org_wiki_Ospedale_della_Piet%C3%A0.html', 'https://en.wikipedia.org/wiki/Ospedale_della_Piet%C3%A0'), + ('en.wikipedia.org_wiki_Paducah.html', 'https://en.wikipedia.org/wiki/Paducah'), + ('en.wikipedia.org_wiki_Pain_%26_Gain.html', 'https://en.wikipedia.org/wiki/Pain_%26_Gain'), + ('en.wikipedia.org_wiki_Panic_Room#Theatrical_run.html', 'https://en.wikipedia.org/wiki/Panic_Room#Theatrical_run'), + ('en.wikipedia.org_wiki_Patrick_Kane#Early_life.html', 'https://en.wikipedia.org/wiki/Patrick_Kane#Early_life'), + ('en.wikipedia.org_wiki_Pelee.html', 'https://en.wikipedia.org/wiki/Pelee'), + ('en.wikipedia.org_wiki_Phil_Quartararo#Warner_Bros._Records.html', + 'https://en.wikipedia.org/wiki/Phil_Quartararo#Warner_Bros._Records'), + ('en.wikipedia.org_wiki_Philadelphia_Waterdogs#Season_results.html', + 'https://en.wikipedia.org/wiki/Philadelphia_Waterdogs#Season_results'), + ('en.wikipedia.org_wiki_Philip_K._Dick#Career.html', 'https://en.wikipedia.org/wiki/Philip_K._Dick#Career'), + ('en.wikipedia.org_wiki_Philosophi%C3%A6_Naturalis_Principia_Mathematica.html', + 'https://en.wikipedia.org/wiki/Philosophi%C3%A6_Naturalis_Principia_Mathematica'), + ('en.wikipedia.org_wiki_Pok%C3%A9mon.html', 'https://en.wikipedia.org/wiki/Pok%C3%A9mon'), + ('en.wikipedia.org_wiki_Pok%C3%A9mon_Diamond_and_Pearl.html', + 'https://en.wikipedia.org/wiki/Pok%C3%A9mon_Diamond_and_Pearl'), + ('en.wikipedia.org_wiki_Pok%C3%A9mon_World_Championships.html', + 'https://en.wikipedia.org/wiki/Pok%C3%A9mon_World_Championships'), + ('en.wikipedia.org_wiki_Portage_County.html', 'https://en.wikipedia.org/wiki/Portage_County'), + ('en.wikipedia.org_wiki_Portland.html', 'https://en.wikipedia.org/wiki/Portland'), + ('en.wikipedia.org_wiki_Prague#_~_text=Prague%20is%20located%20approximately%20at,N%2014%C2%B025%E2%80%B2E..html', + 'https://en.wikipedia.org/wiki/Prague#_~_text=Prague%20is%20located%20approximately%20at,N%2014%C2%B025%E2%80%B2E.'), + ('en.wikipedia.org_wiki_Prague#_~_text=Prague%20is%20located%20approximately%20at.html', + 'https://en.wikipedia.org/wiki/Prague#_~_text=Prague%20is%20located%20approximately%20at'), + ('en.wikipedia.org_wiki_President_of_the_United_States#History_and_development.html', + 'https://en.wikipedia.org/wiki/President_of_the_United_States#History_and_development'), + ('en.wikipedia.org_wiki_Pride_%26_Prejudice_(2005_film).html', + 'https://en.wikipedia.org/wiki/Pride_%26_Prejudice_(2005_film)'), + ('en.wikipedia.org_wiki_Pride_and_Prejudice#Film,_television_and_theatre.html', + 'https://en.wikipedia.org/wiki/Pride_and_Prejudice#Film,_television_and_theatre'), + ('en.wikipedia.org_wiki_Pride_and_Prejudice#Film.html', 'https://en.wikipedia.org/wiki/Pride_and_Prejudice#Film'), + ('en.wikipedia.org_wiki_Prince_Andrew.html', 'https://en.wikipedia.org/wiki/Prince_Andrew'), + ('en.wikipedia.org_wiki_Prince_Edward.html', 'https://en.wikipedia.org/wiki/Prince_Edward'), + ('en.wikipedia.org_wiki_Prince_Philip.html', 'https://en.wikipedia.org/wiki/Prince_Philip'), + ('en.wikipedia.org_wiki_Prinsjesdag#History.html', 'https://en.wikipedia.org/wiki/Prinsjesdag#History'), + ('en.wikipedia.org_wiki_Pro_Bowl#Players_with_most_invitations.html', + 'https://en.wikipedia.org/wiki/Pro_Bowl#Players_with_most_invitations'), + ('en.wikipedia.org_wiki_Professional_wrestling#Occupational_hazards.html', + 'https://en.wikipedia.org/wiki/Professional_wrestling#Occupational_hazards'), + ('en.wikipedia.org_wiki_Province_of_A_Coru%C3%B1a.html', 'https://en.wikipedia.org/wiki/Province_of_A_Coru%C3%B1a'), + ('en.wikipedia.org_wiki_Provo.html', 'https://en.wikipedia.org/wiki/Provo'), + ('en.wikipedia.org_wiki_Pseudoscorpion#Classification.html', + 'https://en.wikipedia.org/wiki/Pseudoscorpion#Classification'), + ('en.wikipedia.org_wiki_Pterostylis_aestiva#Description.html', + 'https://en.wikipedia.org/wiki/Pterostylis_aestiva#Description'), + ('en.wikipedia.org_wiki_Pulitzer_Prize_for_Fiction#Repeat_winners.html', + 'https://en.wikipedia.org/wiki/Pulitzer_Prize_for_Fiction#Repeat_winners'), + ('en.wikipedia.org_wiki_Qufu#Geography.html', 'https://en.wikipedia.org/wiki/Qufu#Geography'), + ('en.wikipedia.org_wiki_Red_Hot_Chili_Peppers#Discography.html', + 'https://en.wikipedia.org/wiki/Red_Hot_Chili_Peppers#Discography'), + ('en.wikipedia.org_wiki_Remy%27s_Ratatouille_Adventure.html', + 'https://en.wikipedia.org/wiki/Remy%27s_Ratatouille_Adventure'), + ('en.wikipedia.org_wiki_Ren%C4%8De.html', 'https://en.wikipedia.org/wiki/Ren%C4%8De'), + ('en.wikipedia.org_wiki_Resident_Evil__Revelations_2#.html', + 'https://en.wikipedia.org/wiki/Resident_Evil__Revelations_2#'), + ('en.wikipedia.org_wiki_Revive_%26_Restore.html', 'https://en.wikipedia.org/wiki/Revive_%26_Restore'), + ('en.wikipedia.org_wiki_River_Avon,_Bristol#Hydrology_and_water_quality.html', + 'https://en.wikipedia.org/wiki/River_Avon,_Bristol#Hydrology_and_water_quality'), + ('en.wikipedia.org_wiki_River_Avon.html', 'https://en.wikipedia.org/wiki/River_Avon'), + ('en.wikipedia.org_wiki_Roberto_%C3%81lamo.html', 'https://en.wikipedia.org/wiki/Roberto_%C3%81lamo'), + ('en.wikipedia.org_wiki_Rodrigo_Pacheco_M%C3%A9ndez.html', + 'https://en.wikipedia.org/wiki/Rodrigo_Pacheco_M%C3%A9ndez'), + ('en.wikipedia.org_wiki_Rubik%27s_Cube.html', 'https://en.wikipedia.org/wiki/Rubik%27s_Cube'), + ('en.wikipedia.org_wiki_Russian_Bishop%27s_House.html', 'https://en.wikipedia.org/wiki/Russian_Bishop%27s_House'), + ('en.wikipedia.org_wiki_Sailing_at_the_1984_Summer_Olympics_%E2%80%93_470.html', + 'https://en.wikipedia.org/wiki/Sailing_at_the_1984_Summer_Olympics_%E2%80%93_470'), + ('en.wikipedia.org_wiki_San_Jose.html', 'https://en.wikipedia.org/wiki/San_Jose'), + ('en.wikipedia.org_wiki_Saturday_Night_Live#Controversies.html', + 'https://en.wikipedia.org/wiki/Saturday_Night_Live#Controversies'), + ('en.wikipedia.org_wiki_Saving_Private_Ryan#Reception.html', + 'https://en.wikipedia.org/wiki/Saving_Private_Ryan#Reception'), + ('en.wikipedia.org_wiki_Schindler%27s_List.html', 'https://en.wikipedia.org/wiki/Schindler%27s_List'), + ('en.wikipedia.org_wiki_Se%C3%A1n_Mac_Diarmada.html', 'https://en.wikipedia.org/wiki/Se%C3%A1n_Mac_Diarmada'), + ('en.wikipedia.org_wiki_Sea_Cliff.html', 'https://en.wikipedia.org/wiki/Sea_Cliff'), + ('en.wikipedia.org_wiki_Sens%C5%8D-ji.html', 'https://en.wikipedia.org/wiki/Sens%C5%8D-ji'), + ('en.wikipedia.org_wiki_Seven_Wonders_of_the_Ancient_World#Wonders.html', + 'https://en.wikipedia.org/wiki/Seven_Wonders_of_the_Ancient_World#Wonders'), + ('en.wikipedia.org_wiki_She%27s_All_That.html', 'https://en.wikipedia.org/wiki/She%27s_All_That'), + ('en.wikipedia.org_wiki_Shrek#Accolades.html', 'https://en.wikipedia.org/wiki/Shrek#Accolades'), + ('en.wikipedia.org_wiki_Shreve,_Lamb_%26_Harmon.html', 'https://en.wikipedia.org/wiki/Shreve,_Lamb_%26_Harmon'), + ('en.wikipedia.org_wiki_Shreve.html', 'https://en.wikipedia.org/wiki/Shreve'), + ('en.wikipedia.org_wiki_Solomon_Sea#Deepest_point.html', 'https://en.wikipedia.org/wiki/Solomon_Sea#Deepest_point'), + ('en.wikipedia.org_wiki_Speech_from_the_throne#Netherlands.html', + 'https://en.wikipedia.org/wiki/Speech_from_the_throne#Netherlands'), + ('en.wikipedia.org_wiki_Spokane_University#Notable_alumni.html', + 'https://en.wikipedia.org/wiki/Spokane_University#Notable_alumni'), + ('en.wikipedia.org_wiki_SpongeBob_SquarePants#Franchise.html', + 'https://en.wikipedia.org/wiki/SpongeBob_SquarePants#Franchise'), + ('en.wikipedia.org_wiki_Spyro_2__Ripto%27s_Rage!.html', 'https://en.wikipedia.org/wiki/Spyro_2:_Ripto%27s_Rage!'), + ('en.wikipedia.org_wiki_St._Philip%27s_College_(United_States).html', + 'https://en.wikipedia.org/wiki/St._Philip%27s_College_(United_States)'), + ('en.wikipedia.org_wiki_Stephen_Hawking#1975%E2%80%931990.html', + 'https://en.wikipedia.org/wiki/Stephen_Hawking#1975%E2%80%931990'), + ('en.wikipedia.org_wiki_Steve_Carell#2004%E2%80%932013__The_Office_and_comedic_roles.html', + 'https://en.wikipedia.org/wiki/Steve_Carell#2004%E2%80%932013__The_Office_and_comedic_roles'), + ('en.wikipedia.org_wiki_Strong_Love_Affair#Track_listing.html', + 'https://en.wikipedia.org/wiki/Strong_Love_Affair#Track_listing'), + ('en.wikipedia.org_wiki_Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers,ravaged%20the%20city%20in%201992..html', + 'https://en.wikipedia.org/wiki/Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers,ravaged%20the%20city%20in%201992.'), + ('en.wikipedia.org_wiki_Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers.html', + 'https://en.wikipedia.org/wiki/Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers'), + ('en.wikipedia.org_wiki_Superman_%26_Bugs_Bunny.html', 'https://en.wikipedia.org/wiki/Superman_%26_Bugs_Bunny'), + ('en.wikipedia.org_wiki_Swimming_at_the_1948_Summer_Olympics_%E2%80%93_Men%27s_100_metre_freestyle.html', + 'https://en.wikipedia.org/wiki/Swimming_at_the_1948_Summer_Olympics_%E2%80%93_Men%27s_100_metre_freestyle'), + ('en.wikipedia.org_wiki_Synsphyronus#Species.html', 'https://en.wikipedia.org/wiki/Synsphyronus#Species'), + ('en.wikipedia.org_wiki_Takarazuka,_Hy%C5%8Dgo.html', 'https://en.wikipedia.org/wiki/Takarazuka,_Hy%C5%8Dgo'), + ('en.wikipedia.org_wiki_Takarazuka.html', 'https://en.wikipedia.org/wiki/Takarazuka'), + ('en.wikipedia.org_wiki_Target_House.html', 'https://en.wikipedia.org/wiki/Target_House'), + ('en.wikipedia.org_wiki_Taylor_Swift#.html', 'https://en.wikipedia.org/wiki/Taylor_Swift#'), + ('en.wikipedia.org_wiki_Tennessee_Coal.html', 'https://en.wikipedia.org/wiki/Tennessee_Coal'), + ('en.wikipedia.org_wiki_Tennis_at_the_1988_Summer_Olympics_%E2%80%93_Women%27s_singles.html', + 'https://en.wikipedia.org/wiki/Tennis_at_the_1988_Summer_Olympics_%E2%80%93_Women%27s_singles'), + ('en.wikipedia.org_wiki_Terrorism_in_the_United_States#Deadliest_attacks.html', + 'https://en.wikipedia.org/wiki/Terrorism_in_the_United_States#Deadliest_attacks'), + ('en.wikipedia.org_wiki_Thailand_national_football_team#Coaching_history.html', + 'https://en.wikipedia.org/wiki/Thailand_national_football_team#Coaching_history'), + ('en.wikipedia.org_wiki_That%27s_What_Friends_Are_For.html', + 'https://en.wikipedia.org/wiki/That%27s_What_Friends_Are_For'), + ('en.wikipedia.org_wiki_The_Apollo.html', 'https://en.wikipedia.org/wiki/The_Apollo'), + ('en.wikipedia.org_wiki_The_Band#Discography.html', 'https://en.wikipedia.org/wiki/The_Band#Discography'), + ('en.wikipedia.org_wiki_The_Beach_Boys#History.html', 'https://en.wikipedia.org/wiki/The_Beach_Boys#History'), + ('en.wikipedia.org_wiki_The_Duke_of_Edinburgh%27s_Award.html', + 'https://en.wikipedia.org/wiki/The_Duke_of_Edinburgh%27s_Award'), + ('en.wikipedia.org_wiki_The_Old_Man_and_the_Sea#Reception_and_legacy.html', + 'https://en.wikipedia.org/wiki/The_Old_Man_and_the_Sea#Reception_and_legacy'), + ('en.wikipedia.org_wiki_The_Seduction_of_Joe_Tynan#Awards.html', + 'https://en.wikipedia.org/wiki/The_Seduction_of_Joe_Tynan#Awards'), + ('en.wikipedia.org_wiki_Thirteen_Years%27_War_(1454%E2%80%931466).html', + 'https://en.wikipedia.org/wiki/Thirteen_Years%27_War_(1454%E2%80%931466)'), + ('en.wikipedia.org_wiki_Tiny_Tina%27s_Wonderlands.html', 'https://en.wikipedia.org/wiki/Tiny_Tina%27s_Wonderlands'), + ('en.wikipedia.org_wiki_Tom%C3%A1%C5%A1_Pekhart.html', 'https://en.wikipedia.org/wiki/Tom%C3%A1%C5%A1_Pekhart'), + ('en.wikipedia.org_wiki_Tony_Hawk%27s_Pro_Skater.html', 'https://en.wikipedia.org/wiki/Tony_Hawk%27s_Pro_Skater'), + ('en.wikipedia.org_wiki_Toronto_Maple_Leafs#Season-by-season_record.html', + 'https://en.wikipedia.org/wiki/Toronto_Maple_Leafs#Season-by-season_record'), + ('en.wikipedia.org_wiki_Trans%E2%80%93West_African_Coastal_Highway.html', + 'https://en.wikipedia.org/wiki/Trans%E2%80%93West_African_Coastal_Highway'), + ('en.wikipedia.org_wiki_Triple_Crown_of_Thoroughbred_Racing_(United_States)#Winners_of_the_Triple_Crown.html', + 'https://en.wikipedia.org/wiki/Triple_Crown_of_Thoroughbred_Racing_(United_States)#Winners_of_the_Triple_Crown'), + ('en.wikipedia.org_wiki_Tulsa.html', 'https://en.wikipedia.org/wiki/Tulsa'), + ('en.wikipedia.org_wiki_United_States_federal_executive_departments#Former_departments.html', + 'https://en.wikipedia.org/wiki/United_States_federal_executive_departments#Former_departments'), + ('en.wikipedia.org_wiki_University_of_Illinois_Urbana-Champaign#Notable_alumni_and_faculty.html', + 'https://en.wikipedia.org/wiki/University_of_Illinois_Urbana-Champaign#Notable_alumni_and_faculty'), + ('en.wikipedia.org_wiki_University_of_Maryland,_College_Park#Notable_alumni.html', + 'https://en.wikipedia.org/wiki/University_of_Maryland,_College_Park#Notable_alumni'), + ('en.wikipedia.org_wiki_University_of_Maryland.html', 'https://en.wikipedia.org/wiki/University_of_Maryland'), + ('en.wikipedia.org_wiki_University_of_Oxford#Mathematics_and_sciences.html', + 'https://en.wikipedia.org/wiki/University_of_Oxford#Mathematics_and_sciences'), + ('en.wikipedia.org_wiki_Upper_Franconia#Coat_of_arms.html', + 'https://en.wikipedia.org/wiki/Upper_Franconia#Coat_of_arms'), + ('en.wikipedia.org_wiki_Utah_County.html', 'https://en.wikipedia.org/wiki/Utah_County'), + ('en.wikipedia.org_wiki_Warner_Records#End_of_an_era__Ostin_and_Waronker_depart.html', + 'https://en.wikipedia.org/wiki/Warner_Records#End_of_an_era__Ostin_and_Waronker_depart'), + ('en.wikipedia.org_wiki_Waterville.html', 'https://en.wikipedia.org/wiki/Waterville'), + ('en.wikipedia.org_wiki_Weekly_Sh%C5%8Dnen_Jump.html', 'https://en.wikipedia.org/wiki/Weekly_Sh%C5%8Dnen_Jump'), + ('en.wikipedia.org_wiki_Wendy%27s.html', 'https://en.wikipedia.org/wiki/Wendy%27s'), + ('en.wikipedia.org_wiki_Westfield.html', 'https://en.wikipedia.org/wiki/Westfield'), + ('en.wikipedia.org_wiki_Wheatland.html', 'https://en.wikipedia.org/wiki/Wheatland'), + ('en.wikipedia.org_wiki_Whiplash_(2014_film)#.html', 'https://en.wikipedia.org/wiki/Whiplash_(2014_film)#'), + ('en.wikipedia.org_wiki_William.html', 'https://en.wikipedia.org/wiki/William'), + ('en.wikipedia.org_wiki_William_Mansfield.html', 'https://en.wikipedia.org/wiki/William_Mansfield'), + ('en.wikipedia.org_wiki_Willie_Mays#.html', 'https://en.wikipedia.org/wiki/Willie_Mays#'), + ('en.wikipedia.org_wiki_Women%27s_Tennis_Association.html', + 'https://en.wikipedia.org/wiki/Women%27s_Tennis_Association'), + ('en.wikipedia.org_wiki_Women_in_Film_Crystal_%2B_Lucy_Awards#Dorothy_Arzner_Directors_award.html', + 'https://en.wikipedia.org/wiki/Women_in_Film_Crystal_%2B_Lucy_Awards#Dorothy_Arzner_Directors_award'), + ('en.wikipedia.org_wiki_Wood_County.html', 'https://en.wikipedia.org/wiki/Wood_County'), + ('en.wikipedia.org_wiki_World%27s_Columbian_Exposition.html', + 'https://en.wikipedia.org/wiki/World%27s_Columbian_Exposition'), + ('en.wikipedia.org_wiki_World_War_I#.html', 'https://en.wikipedia.org/wiki/World_War_I#'), + ('en.wikipedia.org_wiki_WrestleMania_13#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_13#Results'), + ('en.wikipedia.org_wiki_WrestleMania_21#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_21#Results'), + ('en.wikipedia.org_wiki_WrestleMania_22#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_22#Results'), + ('en.wikipedia.org_wiki_WrestleMania_23#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_23#Results'), + ('en.wikipedia.org_wiki_WrestleMania_25#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_25#Results'), + ('en.wikipedia.org_wiki_WrestleMania_29#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_29#Results'), + ('en.wikipedia.org_wiki_WrestleMania_IX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_IX#Results'), + ('en.wikipedia.org_wiki_WrestleMania_VII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_VII#Results'), + ('en.wikipedia.org_wiki_WrestleMania_VIII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_VIII#Results'), + ('en.wikipedia.org_wiki_WrestleMania_X-Seven#Results.html', + 'https://en.wikipedia.org/wiki/WrestleMania_X-Seven#Results'), + ('en.wikipedia.org_wiki_WrestleMania_X8#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_X8#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XI#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XI#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XII#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XIV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XIV#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XIX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XIX#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XV#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XX#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XXIV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXIV#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XXVI#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXVI#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XXVII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXVII#Results'), + ('en.wikipedia.org_wiki_WrestleMania_XXVIII#Results.html', + 'https://en.wikipedia.org/wiki/WrestleMania_XXVIII#Results'), + ('en.wikipedia.org_wiki_Wrestling_at_the_1960_Summer_Olympics_%E2%80%93_Men%27s_Greco-Roman_bantamweight.html', + 'https://en.wikipedia.org/wiki/Wrestling_at_the_1960_Summer_Olympics_%E2%80%93_Men%27s_Greco-Roman_bantamweight'), + ('en.wikipedia.org_wiki_Wrestling_at_the_2020_Summer_Olympics#Medalists.html', + 'https://en.wikipedia.org/wiki/Wrestling_at_the_2020_Summer_Olympics#Medalists'), + ('en.wikipedia.org_wiki_Wrestling_at_the_2020_Summer_Olympics_%E2%80%93_Men%27s_freestyle_86_kg.html', + 'https://en.wikipedia.org/wiki/Wrestling_at_the_2020_Summer_Olympics_%E2%80%93_Men%27s_freestyle_86_kg'), + ('en.wikipedia.org_wiki_Wroc%C5%82aw.html', 'https://en.wikipedia.org/wiki/Wroc%C5%82aw'), + ('en.wikipedia.org_wiki_Wroc%C5%82aw_Dwarfs.html', 'https://en.wikipedia.org/wiki/Wroc%C5%82aw_Dwarfs'), + ('en.wikipedia.org_wiki_Wyandotte.html', 'https://en.wikipedia.org/wiki/Wyandotte'), + ('en.wikipedia.org_wiki_You_(TV_series)#Season_2_(2019).html', + 'https://en.wikipedia.org/wiki/You_(TV_series)#Season_2_(2019)'), + ('en.wikipedia.org_wiki_Ypsilanti.html', 'https://en.wikipedia.org/wiki/Ypsilanti'), + ('en.wikipedia.org_wiki_Zac_Efron#Awards_and_nominations.html', + 'https://en.wikipedia.org/wiki/Zac_Efron#Awards_and_nominations'), + ('en.wikipedia.org_wiki_Zack_Wheat#.html', 'https://en.wikipedia.org/wiki/Zack_Wheat#')] + + +def download_one(filename: str, url: str, out_dir: Path) -> dict: + out_path = out_dir / filename + result = {"filename": filename, "url": url, "ok": False, "status": None, "bytes": 0, "error": None} + headers = {"User-Agent": USER_AGENT} + + for attempt in range(1, 4): + try: + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + result["status"] = getattr(response, "status", None) + result["bytes"] = len(data) + if result["status"] and 200 <= result["status"] < 300 and data: + out_path.write_bytes(data) + result["ok"] = True + return result + result["error"] = f"unexpected status/content: status={result['status']} bytes={len(data)}" + except urllib.error.HTTPError as exc: + result["status"] = exc.code + result["error"] = f"HTTPError: {exc.code} {exc.reason}" + if exc.code == 429: + time.sleep(15 * attempt) + continue + if exc.code in (404, 410): + break + except Exception as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + + time.sleep(0.75 * attempt) + + return result + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Download the embedded 470 additional Wikipedia HTML files." + ) + parser.add_argument("output_folder", type=Path, help="Folder where HTML files and manifests will be written.") + args = parser.parse_args() + + out_dir = args.output_folder + out_dir.mkdir(parents=True, exist_ok=True) + manifest_path = out_dir / "manifest.jsonl" + + results = [] + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + futures = [pool.submit(download_one, filename, url, out_dir) for filename, url in DOWNLOADS] + with manifest_path.open("w", encoding="utf-8") as manifest: + for future in as_completed(futures): + result = future.result() + results.append(result) + manifest.write(json.dumps(result, ensure_ascii=False) + "\n") + manifest.flush() + + failures = [result for result in results if not result["ok"]] + if failures: + failures_path = out_dir / "failures.txt" + with failures_path.open("w", encoding="utf-8") as failure_file: + for result in sorted(failures, key=lambda item: item["filename"]): + failure_file.write( + f"{result['filename']}\t{result['url']}\t{result['status']}\t{result['error']}\n" + ) + else: + failures_path = None + + summary = { + "output_folder": str(out_dir), + "requested": len(DOWNLOADS), + "downloaded": sum(1 for result in results if result["ok"]), + "failed": len(failures), + "manifest": str(manifest_path), + "failures": str(failures_path) if failures_path else None, + } + (out_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() From 1aadd3cb498d0b9d48de6b0af7e1fad8b7d72a66 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Thu, 9 Jul 2026 13:42:17 -0700 Subject: [PATCH 03/14] model name update to match the path and folder names that come with mlc storage download --- e2e-rag/accuracy_eval.py | 4 +- e2e-rag/config.template.sh | 14 +- e2e-rag/datasetup_accuracy_eval.py | 2 +- e2e-rag/db_manifest.py | 2 +- e2e-rag/download_additional_urls.py | 795 --------------------------- e2e-rag/evaluate.py | 4 +- e2e-rag/oracle_single_shot.py | 4 +- e2e-rag/params.py | 18 +- e2e-rag/reference_mlperf_accuracy.sh | 18 +- e2e-rag/reference_mlperf_perf.sh | 17 +- e2e-rag/run_compliance_test09.sh | 20 +- e2e-rag/scripts/run_ingestion.sh | 2 +- e2e-rag/scripts/run_multi_shot.sh | 16 +- e2e-rag/scripts/run_oracle.sh | 2 +- e2e-rag/scripts/run_single_shot.sh | 6 +- e2e-rag/scripts/start_vllm_server.sh | 4 +- e2e-rag/scripts/write_db_manifest.sh | 2 +- 17 files changed, 75 insertions(+), 855 deletions(-) delete mode 100644 e2e-rag/download_additional_urls.py diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 026456ab09..36f7e4138f 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -32,8 +32,8 @@ # OpenRouter configuration -DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions" -DEFAULT_JUDGE_MODEL = "gpt-oss-20b" +DEFAULT_JUDGE_URL = "http://127.0.0.1:8192/v1/chat/completions" +DEFAULT_JUDGE_MODEL = "/data/gpt-oss-20b-mxfp4" # Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter) OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', 'sk-or-v1-****') diff --git a/e2e-rag/config.template.sh b/e2e-rag/config.template.sh index 33b80080c1..6be88f0c34 100644 --- a/e2e-rag/config.template.sh +++ b/e2e-rag/config.template.sh @@ -36,22 +36,22 @@ INFERENCE_N_QUERIES=5 INFERENCE_NUM_WORKERS=1 # LLM endpoints (vLLM, OpenRouter, etc.) -INFERENCE_LLM_URL="http://127.0.0.1:8123/v1/chat/completions" -INFERENCE_MODEL="/model/gpt-oss-20b-mxfp4" -INFERENCE_QUERY_MODEL="/model/gpt-oss-120b-mxfp4" +INFERENCE_LLM_URL="http://127.0.0.1:8192/v1/chat/completions" +INFERENCE_MODEL="gpt-oss-20b-mxfp4" +INFERENCE_QUERY_MODEL="gpt-oss-120b-mxfp4" # Per-component endpoint splits. Each defaults to INFERENCE_LLM_URL / # INFERENCE_MODEL when empty. Set when components live on different servers # (e.g. small grader on one vLLM, large query/sufficiency on another). -# INFERENCE_GRADER_URL="http://127.0.0.1:8124/v1/chat/completions" -# INFERENCE_GRADER_MODEL="/model/gpt-oss-20b" +# INFERENCE_GRADER_URL="http://127.0.0.1:8192/v1/chat/completions" +# INFERENCE_GRADER_MODEL="gpt-oss-20b-mxfp4" # INFERENCE_QUERY_URL="http://127.0.0.1:8123/v1/chat/completions" # INFERENCE_SUFFICIENCY_URL="http://127.0.0.1:8123/v1/chat/completions" -# INFERENCE_SUFFICIENCY_MODEL="/model/gpt-oss-120b" +# INFERENCE_SUFFICIENCY_MODEL="gpt-oss-120b-mxfp4" # Judge (used by evaluate.py at the end of the run scripts) INFERENCE_JUDGE_URL="https://openrouter.ai/api/v1/chat/completions" -INFERENCE_JUDGE_MODEL="openai/gpt-oss-20b" +INFERENCE_JUDGE_MODEL="Llama3.1-8B-v1" # ── Oracle evaluation (run_oracle.sh) ───────────────────────────────────────── INFERENCE_ORACLE_BATCH_SIZE=4 diff --git a/e2e-rag/datasetup_accuracy_eval.py b/e2e-rag/datasetup_accuracy_eval.py index 7df71d03f0..735170f563 100755 --- a/e2e-rag/datasetup_accuracy_eval.py +++ b/e2e-rag/datasetup_accuracy_eval.py @@ -520,7 +520,7 @@ def main(): ) parser.add_argument( "--retriever_model", - default="/data/model/e5-base-v2", + default="intfloat_e5-base-v2/e5-base-v2", help="Path to retriever model (for validation)" ) diff --git a/e2e-rag/db_manifest.py b/e2e-rag/db_manifest.py index 1cfe20f85f..6ecf87f957 100644 --- a/e2e-rag/db_manifest.py +++ b/e2e-rag/db_manifest.py @@ -254,7 +254,7 @@ def main(): pw = sub.add_parser("write", help="Generate a reference manifest from a DB.") pw.add_argument("--db", required=True) - pw.add_argument("--retriever_model", default="intfloat/e5-base-v2") + pw.add_argument("--retriever_model", default="intfloat_e5-base-v2/e5-base-v2") pw.add_argument("--dataset", default="data/frames_dataset.tsv") pw.add_argument("--output", required=True) pw.set_defaults(func=cmd_write) diff --git a/e2e-rag/download_additional_urls.py b/e2e-rag/download_additional_urls.py deleted file mode 100644 index bd22e08d2a..0000000000 --- a/e2e-rag/download_additional_urls.py +++ /dev/null @@ -1,795 +0,0 @@ -#!/usr/bin/env python3 -"""Download the 470 additional Wikipedia HTML files. - -This script is self-contained: the URL list is embedded below. Pass the output -folder as the only command-line argument; files are written directly there. -""" - -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path -import argparse -import json -import time -import urllib.error -import urllib.request - -WORKERS = 8 -USER_AGENT = "Mozilla/5.0 (compatible; html-fetch/1.0)" - -DOWNLOADS = [('en.wikipedia.org_wiki_%22Weird_Al%22_Yankovic.html', 'https://en.wikipedia.org/wiki/%22Weird_Al%22_Yankovic'), - ('en.wikipedia.org_wiki_%C5%BDAK_Subotica.html', 'https://en.wikipedia.org/wiki/%C5%BDAK_Subotica'), - ('en.wikipedia.org_wiki_1854_Broad_Street_cholera_outbreak#.html', - 'https://en.wikipedia.org/wiki/1854_Broad_Street_cholera_outbreak#'), - ('en.wikipedia.org_wiki_1900%E2%80%9301_Football_League#Final_league_tables.html', - 'https://en.wikipedia.org/wiki/1900%E2%80%9301_Football_League#Final_league_tables'), - ('en.wikipedia.org_wiki_1923#Deaths.html', 'https://en.wikipedia.org/wiki/1923#Deaths'), - ('en.wikipedia.org_wiki_1960_European_Nations%27_Cup.html', - 'https://en.wikipedia.org/wiki/1960_European_Nations%27_Cup'), - ('en.wikipedia.org_wiki_1967_Formula_One_season#Teams_and_drivers.html', - 'https://en.wikipedia.org/wiki/1967_Formula_One_season#Teams_and_drivers'), - ('en.wikipedia.org_wiki_1972#.html', 'https://en.wikipedia.org/wiki/1972#'), - ('en.wikipedia.org_wiki_1976%E2%80%9377_Portland_Trail_Blazers_season.html', - 'https://en.wikipedia.org/wiki/1976%E2%80%9377_Portland_Trail_Blazers_season'), - ('en.wikipedia.org_wiki_1982%E2%80%9383_Wolverhampton_Wanderers_F.C._season.html', - 'https://en.wikipedia.org/wiki/1982%E2%80%9383_Wolverhampton_Wanderers_F.C._season'), - ('en.wikipedia.org_wiki_1984%E2%80%9385_NHL_season.html', 'https://en.wikipedia.org/wiki/1984%E2%80%9385_NHL_season'), - ('en.wikipedia.org_wiki_1996%E2%80%9397_AC_Milan_season.html', - 'https://en.wikipedia.org/wiki/1996%E2%80%9397_AC_Milan_season'), - ('en.wikipedia.org_wiki_1996%E2%80%9397_Detroit_Red_Wings_season.html', - 'https://en.wikipedia.org/wiki/1996%E2%80%9397_Detroit_Red_Wings_season'), - ('en.wikipedia.org_wiki_1997_NFL_draft#Round_6.html', 'https://en.wikipedia.org/wiki/1997_NFL_draft#Round_6'), - ('en.wikipedia.org_wiki_1998_NFL_draft#Player_selections.html', - 'https://en.wikipedia.org/wiki/1998_NFL_draft#Player_selections'), - ('en.wikipedia.org_wiki_1999%E2%80%932000_Olympique_de_Marseille_season.html', - 'https://en.wikipedia.org/wiki/1999%E2%80%932000_Olympique_de_Marseille_season'), - ('en.wikipedia.org_wiki_19th_People%27s_Choice_Awards#Awards.html', - 'https://en.wikipedia.org/wiki/19th_People%27s_Choice_Awards#Awards'), - ('en.wikipedia.org_wiki_2002_Wimbledon_Championships_%E2%80%93_Men%27s_singles.html', - 'https://en.wikipedia.org/wiki/2002_Wimbledon_Championships_%E2%80%93_Men%27s_singles'), - ('en.wikipedia.org_wiki_2002_World_Series#Composite_box.html', - 'https://en.wikipedia.org/wiki/2002_World_Series#Composite_box'), - ('en.wikipedia.org_wiki_2003_US_Open_%E2%80%93_Men%27s_singles.html', - 'https://en.wikipedia.org/wiki/2003_US_Open_%E2%80%93_Men%27s_singles'), - ('en.wikipedia.org_wiki_2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in,clash%20and%20interlacing%20of%20cultures.%22.html', - 'https://en.wikipedia.org/wiki/2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in,clash%20and%20interlacing%20of%20cultures.%22'), - ('en.wikipedia.org_wiki_2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in.html', - 'https://en.wikipedia.org/wiki/2006_Nobel_Prize_in_Literature#_~_text=The%202006%20Nobel%20Prize%20in'), - ('en.wikipedia.org_wiki_2007_NHL_entry_draft#Round_one.html', - 'https://en.wikipedia.org/wiki/2007_NHL_entry_draft#Round_one'), - ('en.wikipedia.org_wiki_2011_T%C5%8Dhoku_earthquake_and_tsunami#Nuclear_power_plants.html', - 'https://en.wikipedia.org/wiki/2011_T%C5%8Dhoku_earthquake_and_tsunami#Nuclear_power_plants'), - ('en.wikipedia.org_wiki_2011_T%C5%8Dhoku_earthquake_and_tsunami.html', - 'https://en.wikipedia.org/wiki/2011_T%C5%8Dhoku_earthquake_and_tsunami'), - ('en.wikipedia.org_wiki_2013%E2%80%9314_Portland_Trail_Blazers_season#Playoffs.html', - 'https://en.wikipedia.org/wiki/2013%E2%80%9314_Portland_Trail_Blazers_season#Playoffs'), - ('en.wikipedia.org_wiki_2015_FIFA_Women%27s_World_Cup_knockout_stage.html', - 'https://en.wikipedia.org/wiki/2015_FIFA_Women%27s_World_Cup_knockout_stage'), - ('en.wikipedia.org_wiki_2015_World_Championships_in_Athletics_%E2%80%93_Women%27s_javelin_throw.html', - 'https://en.wikipedia.org/wiki/2015_World_Championships_in_Athletics_%E2%80%93_Women%27s_javelin_throw'), - ('en.wikipedia.org_wiki_2016_Australian_Open_%E2%80%93_Men%27s_singles.html', - 'https://en.wikipedia.org/wiki/2016_Australian_Open_%E2%80%93_Men%27s_singles'), - ('en.wikipedia.org_wiki_2016_CONCACAF_Women%27s_Olympic_Qualifying_Championship#Group_A.html', - 'https://en.wikipedia.org/wiki/2016_CONCACAF_Women%27s_Olympic_Qualifying_Championship#Group_A'), - ('en.wikipedia.org_wiki_2018%E2%80%9319_Brisbane_Roar_FC_season.html', - 'https://en.wikipedia.org/wiki/2018%E2%80%9319_Brisbane_Roar_FC_season'), - ('en.wikipedia.org_wiki_2018%E2%80%9319_Women%27s_Volleyball_Thailand_League.html', - 'https://en.wikipedia.org/wiki/2018%E2%80%9319_Women%27s_Volleyball_Thailand_League'), - ('en.wikipedia.org_wiki_2018_FIFA_World_Cup#Officiating.html', - 'https://en.wikipedia.org/wiki/2018_FIFA_World_Cup#Officiating'), - ('en.wikipedia.org_wiki_2019%E2%80%9320_NBA_season.html', 'https://en.wikipedia.org/wiki/2019%E2%80%9320_NBA_season'), - ('en.wikipedia.org_wiki_2020%E2%80%9321_Dallas_Mavericks_season.html', - 'https://en.wikipedia.org/wiki/2020%E2%80%9321_Dallas_Mavericks_season'), - ('en.wikipedia.org_wiki_2020%E2%80%9321_NBA_season.html', 'https://en.wikipedia.org/wiki/2020%E2%80%9321_NBA_season'), - ('en.wikipedia.org_wiki_2021_French_Open_%E2%80%93_Men%2527s_singles.html', - 'https://en.wikipedia.org/wiki/2021_French_Open_%E2%80%93_Men%27s_singles'), - ('en.wikipedia.org_wiki_2024_Yucat%C3%A1n_Open_%E2%80%93_Doubles.html', - 'https://en.wikipedia.org/wiki/2024_Yucat%C3%A1n_Open_%E2%80%93_Doubles'), - ('en.wikipedia.org_wiki_74th_Academy_Awards#Winners_and_nominees.html', - 'https://en.wikipedia.org/wiki/74th_Academy_Awards#Winners_and_nominees'), - ('en.wikipedia.org_wiki_95th_Academy_Awards#Awards.html', 'https://en.wikipedia.org/wiki/95th_Academy_Awards#Awards'), - ('en.wikipedia.org_wiki_AC_DC_discography#Live_albums.html', - 'https://en.wikipedia.org/wiki/AC/DC_discography#Live_albums'), - ('en.wikipedia.org_wiki_A_Brooklyn_State_of_Mind#Cast.html', - 'https://en.wikipedia.org/wiki/A_Brooklyn_State_of_Mind#Cast'), - ('en.wikipedia.org_wiki_Academy_Award_for_Best_Actor#1960s.html', - 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Actor#1960s'), - ('en.wikipedia.org_wiki_Academy_Award_for_Best_Animated_Feature#Multiple_wins_and_nominations.html', - 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Animated_Feature#Multiple_wins_and_nominations'), - ('en.wikipedia.org_wiki_Academy_Award_for_Best_Picture#1970s.html', - 'https://en.wikipedia.org/wiki/Academy_Award_for_Best_Picture#1970s'), - ('en.wikipedia.org_wiki_Academy_of_Science.html', 'https://en.wikipedia.org/wiki/Academy_of_Science'), - ('en.wikipedia.org_wiki_Albert_Bartholom%C3%A9#Main_works_(continued).html', - 'https://en.wikipedia.org/wiki/Albert_Bartholom%C3%A9#Main_works_(continued)'), - ('en.wikipedia.org_wiki_Alberto_Gin%C3%A9s_L%C3%B3pez.html', - 'https://en.wikipedia.org/wiki/Alberto_Gin%C3%A9s_L%C3%B3pez'), - ('en.wikipedia.org_wiki_Alex_Hern%C3%A1ndez_(tennis).html', - 'https://en.wikipedia.org/wiki/Alex_Hern%C3%A1ndez_(tennis)'), - ('en.wikipedia.org_wiki_Alexis_Clairaut#Focus_on_astronomical_motion.html', - 'https://en.wikipedia.org/wiki/Alexis_Clairaut#Focus_on_astronomical_motion'), - ('en.wikipedia.org_wiki_American_Athletic_Conference_Men%27s_Basketball_Player_of_the_Year.html', - 'https://en.wikipedia.org/wiki/American_Athletic_Conference_Men%27s_Basketball_Player_of_the_Year'), - ('en.wikipedia.org_wiki_Amplifier_(Dance_Exponents_album)#Track_listing.html', - 'https://en.wikipedia.org/wiki/Amplifier_(Dance_Exponents_album)#Track_listing'), - ('en.wikipedia.org_wiki_Andhra_Pradesh_(1956%E2%80%932014).html', - 'https://en.wikipedia.org/wiki/Andhra_Pradesh_(1956%E2%80%932014)'), - ('en.wikipedia.org_wiki_Andr%C3%A9_the_Giant.html', 'https://en.wikipedia.org/wiki/Andr%C3%A9_the_Giant'), - ('en.wikipedia.org_wiki_André_Silva_(footballer.html', 'https://en.wikipedia.org/wiki/Andr%C3%A9_Silva_(footballer)'), - ('en.wikipedia.org_wiki_Andy_Roddick_career_statistics#Singles__5_finals_(1–4).html', - 'https://en.wikipedia.org/wiki/Andy_Roddick_career_statistics#Singles__5_finals_(1%E2%80%934)'), - ('en.wikipedia.org_wiki_Anna_Stre%C5%BCy%C5%84ska.html', 'https://en.wikipedia.org/wiki/Anna_Stre%C5%BCy%C5%84ska'), - ('en.wikipedia.org_wiki_Anne.html', 'https://en.wikipedia.org/wiki/Anne'), - ('en.wikipedia.org_wiki_Apollo_11#Mission.html', 'https://en.wikipedia.org/wiki/Apollo_11#Mission'), - ('en.wikipedia.org_wiki_Archibald_Sinclair.html', 'https://en.wikipedia.org/wiki/Archibald_Sinclair'), - ('en.wikipedia.org_wiki_Arctic_Ocean#Climate.html', 'https://en.wikipedia.org/wiki/Arctic_Ocean#Climate'), - ('en.wikipedia.org_wiki_Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the,in%201930%2C%201990%20and%202014..html', - 'https://en.wikipedia.org/wiki/Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the,in%201930%2C%201990%20and%202014.'), - ('en.wikipedia.org_wiki_Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the.html', - 'https://en.wikipedia.org/wiki/Argentina_at_the_FIFA_World_Cup#_~_text=Argentina%20is%20one%20of%20the'), - ('en.wikipedia.org_wiki_Arlington.html', 'https://en.wikipedia.org/wiki/Arlington'), - ('en.wikipedia.org_wiki_Auckland_Island#Human_presence_on_the_island.html', - 'https://en.wikipedia.org/wiki/Auckland_Island#Human_presence_on_the_island'), - ('en.wikipedia.org_wiki_August_16#1901%E2%80%93present_2.html', - 'https://en.wikipedia.org/wiki/August_16#1901%E2%80%93present_2'), - ('en.wikipedia.org_wiki_Avalokite%C5%9Bvara.html', 'https://en.wikipedia.org/wiki/Avalokite%C5%9Bvara'), - ('en.wikipedia.org_wiki_Avukana_Buddha_statue#Location_and_appearance.html', - 'https://en.wikipedia.org/wiki/Avukana_Buddha_statue#Location_and_appearance'), - ('en.wikipedia.org_wiki_BTS#.html', 'https://en.wikipedia.org/wiki/BTS#'), - ('en.wikipedia.org_wiki_Baby_Vox#Studio_albums.html', 'https://en.wikipedia.org/wiki/Baby_Vox#Studio_albums'), - ('en.wikipedia.org_wiki_Baldur%27s_Gate_3#.html', 'https://en.wikipedia.org/wiki/Baldur%27s_Gate_3#'), - ('en.wikipedia.org_wiki_Balliol_College.html', 'https://en.wikipedia.org/wiki/Balliol_College'), - ('en.wikipedia.org_wiki_Bath,_Somerset#Culture.html', 'https://en.wikipedia.org/wiki/Bath,_Somerset#Culture'), - ('en.wikipedia.org_wiki_Bath.html', 'https://en.wikipedia.org/wiki/Bath'), - ('en.wikipedia.org_wiki_Bernhard_von_B%C3%BClow.html', 'https://en.wikipedia.org/wiki/Bernhard_von_B%C3%BClow'), - ('en.wikipedia.org_wiki_Bessemer.html', 'https://en.wikipedia.org/wiki/Bessemer'), - ('en.wikipedia.org_wiki_Beyonc%C3%A9#.html', 'https://en.wikipedia.org/wiki/Beyonc%C3%A9#'), - ('en.wikipedia.org_wiki_Big_Ben#Design.html', 'https://en.wikipedia.org/wiki/Big_Ben#Design'), - ('en.wikipedia.org_wiki_Big_Hit_Music#Groups.html', 'https://en.wikipedia.org/wiki/Big_Hit_Music#Groups'), - ('en.wikipedia.org_wiki_Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article,11.html', - 'https://en.wikipedia.org/wiki/Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article,11'), - ('en.wikipedia.org_wiki_Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article.html', - 'https://en.wikipedia.org/wiki/Billboard_Year-End_Hot_100_singles_of_1985#_~_text=Article'), - ('en.wikipedia.org_wiki_Bird#Anatomy_and_physiology.html', - 'https://en.wikipedia.org/wiki/Bird#Anatomy_and_physiology'), - ('en.wikipedia.org_wiki_Birut%C4%97_Galdikas.html', 'https://en.wikipedia.org/wiki/Birut%C4%97_Galdikas'), - ('en.wikipedia.org_wiki_Blink-182#Tours.html', 'https://en.wikipedia.org/wiki/Blink-182#Tours'), - ('en.wikipedia.org_wiki_Blue_moon#Blue_moon_dates.html', 'https://en.wikipedia.org/wiki/Blue_moon#Blue_moon_dates'), - ('en.wikipedia.org_wiki_Blyde_River_Canyon_Nature_Reserve#God.27s_Window.html', - 'https://en.wikipedia.org/wiki/Blyde_River_Canyon_Nature_Reserve#God.27s_Window'), - ('en.wikipedia.org_wiki_BoA#Discography.html', 'https://en.wikipedia.org/wiki/BoA#Discography'), - ('en.wikipedia.org_wiki_Boston_Marathon#Rosie_Ruiz,_the_impostor.html', - 'https://en.wikipedia.org/wiki/Boston_Marathon#Rosie_Ruiz,_the_impostor'), - ('en.wikipedia.org_wiki_Boston_Marathon#Rosie_Ruiz.html', 'https://en.wikipedia.org/wiki/Boston_Marathon#Rosie_Ruiz'), - ("en.wikipedia.org_wiki_Bowling_at_the_2011_Pan_American_Games_%E2%80%93_Women's_individual.html", - "https://en.wikipedia.org/wiki/Bowling_at_the_2011_Pan_American_Games_%E2%80%93_Women's_individual"), - ('en.wikipedia.org_wiki_Brazil_national_football_team#FIFA_World_Cup.html', - 'https://en.wikipedia.org/wiki/Brazil_national_football_team#FIFA_World_Cup'), - ('en.wikipedia.org_wiki_Bridgeton.html', 'https://en.wikipedia.org/wiki/Bridgeton'), - ('en.wikipedia.org_wiki_Britain%27s_Got_Talent.html', 'https://en.wikipedia.org/wiki/Britain%27s_Got_Talent'), - ('en.wikipedia.org_wiki_Bro%27Town.html', 'https://en.wikipedia.org/wiki/Bro%27Town'), - ('en.wikipedia.org_wiki_Broken_Arrow.html', 'https://en.wikipedia.org/wiki/Broken_Arrow'), - ('en.wikipedia.org_wiki_Bront%C3%AB_family.html', 'https://en.wikipedia.org/wiki/Bront%C3%AB_family'), - ('en.wikipedia.org_wiki_Brown_County.html', 'https://en.wikipedia.org/wiki/Brown_County'), - ('en.wikipedia.org_wiki_C%27%C3%A8_la_luna_mezzo_mare#Notable_recordings.html', - 'https://en.wikipedia.org/wiki/C%27%C3%A8_la_luna_mezzo_mare#Notable_recordings'), - ('en.wikipedia.org_wiki_Caldecott_Medal#Recipients.html', 'https://en.wikipedia.org/wiki/Caldecott_Medal#Recipients'), - ('en.wikipedia.org_wiki_Call_Me_Maybe#Year-end_charts.html', - 'https://en.wikipedia.org/wiki/Call_Me_Maybe#Year-end_charts'), - ('en.wikipedia.org_wiki_Calton.html', 'https://en.wikipedia.org/wiki/Calton'), - ('en.wikipedia.org_wiki_Capybara#.html', 'https://en.wikipedia.org/wiki/Capybara#'), - ('en.wikipedia.org_wiki_Carola_H%C3%A4ggkvist.html', 'https://en.wikipedia.org/wiki/Carola_H%C3%A4ggkvist'), - ('en.wikipedia.org_wiki_Charley%27s_Aunt.html', 'https://en.wikipedia.org/wiki/Charley%27s_Aunt'), - ('en.wikipedia.org_wiki_Charlotte_Bront%C3%AB.html', 'https://en.wikipedia.org/wiki/Charlotte_Bront%C3%AB'), - ('en.wikipedia.org_wiki_Charlotte_Checkers_(1956%E2%80%931977).html', - 'https://en.wikipedia.org/wiki/Charlotte_Checkers_(1956%E2%80%931977)'), - ('en.wikipedia.org_wiki_Chemotherapy#History.html', 'https://en.wikipedia.org/wiki/Chemotherapy#History'), - ('en.wikipedia.org_wiki_Cheyenne.html', 'https://en.wikipedia.org/wiki/Cheyenne'), - ('en.wikipedia.org_wiki_Chikhali.html', 'https://en.wikipedia.org/wiki/Chikhali'), - ('en.wikipedia.org_wiki_Chris_Benoit#Death.html', 'https://en.wikipedia.org/wiki/Chris_Benoit#Death'), - ('en.wikipedia.org_wiki_Chris_Columbus_(filmmaker)#Filmography.html', - 'https://en.wikipedia.org/wiki/Chris_Columbus_(filmmaker)#Filmography'), - ('en.wikipedia.org_wiki_Christie%27s.html', 'https://en.wikipedia.org/wiki/Christie%27s'), - ('en.wikipedia.org_wiki_Cinthia_Pi%C3%B1eiro#.html', 'https://en.wikipedia.org/wiki/Cinthia_Pi%C3%B1eiro#'), - ('en.wikipedia.org_wiki_Confucius#.html', 'https://en.wikipedia.org/wiki/Confucius#'), - ('en.wikipedia.org_wiki_Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton,productivity%20than%20manual%20cotton%20separation..html', - 'https://en.wikipedia.org/wiki/Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton,productivity%20than%20manual%20cotton%20separation.'), - ('en.wikipedia.org_wiki_Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton.html', - 'https://en.wikipedia.org/wiki/Cotton_gin#_~_text=A%20cotton%20gin%E2%80%94meaning%20%22cotton'), - ('en.wikipedia.org_wiki_Cowboy_Bebop__Knockin%27_on_Heaven%27s_Door.html', - 'https://en.wikipedia.org/wiki/Cowboy_Bebop:_Knockin%27_on_Heaven%27s_Door'), - ('en.wikipedia.org_wiki_Cricket#Governance.html', 'https://en.wikipedia.org/wiki/Cricket#Governance'), - ('en.wikipedia.org_wiki_Croix_de_guerre_1914%E2%80%931918_(France)#Award_description.html', - 'https://en.wikipedia.org/wiki/Croix_de_guerre_1914%E2%80%931918_(France)#Award_description'), - ('en.wikipedia.org_wiki_Crossroads_(2002_film)#Reception.html', - 'https://en.wikipedia.org/wiki/Crossroads_(2002_film)#Reception'), - ('en.wikipedia.org_wiki_Danny_DeVito#Acting_credits_and_accolades.html', - 'https://en.wikipedia.org/wiki/Danny_DeVito#Acting_credits_and_accolades'), - ('en.wikipedia.org_wiki_Dax_Shepard#Film.html', 'https://en.wikipedia.org/wiki/Dax_Shepard#Film'), - ('en.wikipedia.org_wiki_Derek_Smith_(footballer.html', 'https://en.wikipedia.org/wiki/Derek_Smith_(footballer)'), - ('en.wikipedia.org_wiki_Devil%27s_Pass.html', 'https://en.wikipedia.org/wiki/Devil%27s_Pass'), - ('en.wikipedia.org_wiki_Dhatusena_of_Anuradhapura#Early_life_and_becoming_king.html', - 'https://en.wikipedia.org/wiki/Dhatusena_of_Anuradhapura#Early_life_and_becoming_king'), - ('en.wikipedia.org_wiki_Diana.html', 'https://en.wikipedia.org/wiki/Diana'), - ('en.wikipedia.org_wiki_Didier_Drogba#Honours.html', 'https://en.wikipedia.org/wiki/Didier_Drogba#Honours'), - ('en.wikipedia.org_wiki_Diego_Costa#Spain.html', 'https://en.wikipedia.org/wiki/Diego_Costa#Spain'), - ('en.wikipedia.org_wiki_Diego_L%C3%B3pez_V_de_Haro.html', 'https://en.wikipedia.org/wiki/Diego_L%C3%B3pez_V_de_Haro'), - ('en.wikipedia.org_wiki_Dippin%27_Dots.html', 'https://en.wikipedia.org/wiki/Dippin%27_Dots'), - ('en.wikipedia.org_wiki_Disneynature#Filmography.html', 'https://en.wikipedia.org/wiki/Disneynature#Filmography'), - ('en.wikipedia.org_wiki_Djokovic%E2%80%93Murray_rivalry.html', - 'https://en.wikipedia.org/wiki/Djokovic%E2%80%93Murray_rivalry'), - ('en.wikipedia.org_wiki_Donahoe.html', 'https://en.wikipedia.org/wiki/Donahoe'), - ('en.wikipedia.org_wiki_Dr._Quinn.html', 'https://en.wikipedia.org/wiki/Dr._Quinn'), - ('en.wikipedia.org_wiki_Elon_Musk#Personal_life.html', 'https://en.wikipedia.org/wiki/Elon_Musk#Personal_life'), - ('en.wikipedia.org_wiki_Emirates_(airline)#Services.html', - 'https://en.wikipedia.org/wiki/Emirates_(airline)#Services'), - ('en.wikipedia.org_wiki_Emmanuel_Lubezki#Feature_film.html', - 'https://en.wikipedia.org/wiki/Emmanuel_Lubezki#Feature_film'), - ('en.wikipedia.org_wiki_Emmy_Rossum#Awards_and_nominations.html', - 'https://en.wikipedia.org/wiki/Emmy_Rossum#Awards_and_nominations'), - ('en.wikipedia.org_wiki_England#Geography.html', 'https://en.wikipedia.org/wiki/England#Geography'), - ('en.wikipedia.org_wiki_Estadio_Jos%C3%A9_Mart%C3%ADn_Olaeta.html', - 'https://en.wikipedia.org/wiki/Estadio_Jos%C3%A9_Mart%C3%ADn_Olaeta'), - ('en.wikipedia.org_wiki_Ethereum#Ether.html', 'https://en.wikipedia.org/wiki/Ethereum#Ether'), - ('en.wikipedia.org_wiki_Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin,restrictions%20were%20overwhelmed%20and%20discarded..html', - 'https://en.wikipedia.org/wiki/Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin,restrictions%20were%20overwhelmed%20and%20discarded.'), - ('en.wikipedia.org_wiki_Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin.html', - 'https://en.wikipedia.org/wiki/Fall_of_the_Berlin_Wall#_~_text=The%20fall%20of%20the%20Berlin'), - ('en.wikipedia.org_wiki_Fastest_animals#Fish.html', 'https://en.wikipedia.org/wiki/Fastest_animals#Fish'), - ('en.wikipedia.org_wiki_Federative_units_of_Brazil#_media_File_Brazil,_administrative_divisions_(states)_-_en_-_colored.svg.html', - 'https://en.wikipedia.org/wiki/Federative_units_of_Brazil#_media_File_Brazil,_administrative_divisions_(states)_-_en_-_colored.svg'), - ('en.wikipedia.org_wiki_Federative_units_of_Brazil#_media_File_Brazil.html', - 'https://en.wikipedia.org/wiki/Federative_units_of_Brazil#_media_File_Brazil'), - ('en.wikipedia.org_wiki_Five_Nights_at_Freddy%27s.html', 'https://en.wikipedia.org/wiki/Five_Nights_at_Freddy%27s'), - ('en.wikipedia.org_wiki_Fleetwood_Mac#Grammy_Awards.html', - 'https://en.wikipedia.org/wiki/Fleetwood_Mac#Grammy_Awards'), - ('en.wikipedia.org_wiki_Flight_of_the_Earls#Journey.html', - 'https://en.wikipedia.org/wiki/Flight_of_the_Earls#Journey'), - ('en.wikipedia.org_wiki_Florida_A%26M_University.html', 'https://en.wikipedia.org/wiki/Florida_A%26M_University'), - ('en.wikipedia.org_wiki_Florida_Atlantic_Owls_men%27s_basketball.html', - 'https://en.wikipedia.org/wiki/Florida_Atlantic_Owls_men%27s_basketball'), - ('en.wikipedia.org_wiki_For_Whom_the_Bell_Tolls#Pulitzer_Prize_snub.html', - 'https://en.wikipedia.org/wiki/For_Whom_the_Bell_Tolls#Pulitzer_Prize_snub'), - ('en.wikipedia.org_wiki_Ford_Motor_Company#Sales_numbers.html', - 'https://en.wikipedia.org/wiki/Ford_Motor_Company#Sales_numbers'), - ('en.wikipedia.org_wiki_Franz_Kafka#Stories.html', 'https://en.wikipedia.org/wiki/Franz_Kafka#Stories'), - ('en.wikipedia.org_wiki_Friends_%26_Relatives.html', 'https://en.wikipedia.org/wiki/Friends_%26_Relatives'), - ('en.wikipedia.org_wiki_G%C3%A9za_K%C3%A1das.html', 'https://en.wikipedia.org/wiki/G%C3%A9za_K%C3%A1das'), - ('en.wikipedia.org_wiki_Galileo%27s_middle_finger#Exhibition_history.html', - 'https://en.wikipedia.org/wiki/Galileo%27s_middle_finger#Exhibition_history'), - ('en.wikipedia.org_wiki_Garypidae#Genera.html', 'https://en.wikipedia.org/wiki/Garypidae#Genera'), - ('en.wikipedia.org_wiki_Geffen_Records#History.html', 'https://en.wikipedia.org/wiki/Geffen_Records#History'), - ('en.wikipedia.org_wiki_George_Harrison#Discography.html', - 'https://en.wikipedia.org/wiki/George_Harrison#Discography'), - ('en.wikipedia.org_wiki_George_Howell_(entrepreneur)#The_Coffee_Connection.html', - 'https://en.wikipedia.org/wiki/George_Howell_(entrepreneur)#The_Coffee_Connection'), - ('en.wikipedia.org_wiki_George_Talbot.html', 'https://en.wikipedia.org/wiki/George_Talbot'), - ('en.wikipedia.org_wiki_George_Washington#Personal_life.html', - 'https://en.wikipedia.org/wiki/George_Washington#Personal_life'), - ('en.wikipedia.org_wiki_Girl_Scouts_of_the_USA#Presidents.html', - 'https://en.wikipedia.org/wiki/Girl_Scouts_of_the_USA#Presidents'), - ('en.wikipedia.org_wiki_Golden_Globe_Award_for_Best_Actor_in_a_Motion_Picture_%E2%80%93_Drama.html', - 'https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_in_a_Motion_Picture_%E2%80%93_Drama'), - ('en.wikipedia.org_wiki_Golden_Retriever#Health.html', 'https://en.wikipedia.org/wiki/Golden_Retriever#Health'), - ('en.wikipedia.org_wiki_Goodbye.html', 'https://en.wikipedia.org/wiki/Goodbye'), - ('en.wikipedia.org_wiki_Gothic_Revival_architecture#Roots.html', - 'https://en.wikipedia.org/wiki/Gothic_Revival_architecture#Roots'), - ('en.wikipedia.org_wiki_Grace%27s_High_Falls.html', 'https://en.wikipedia.org/wiki/Grace%27s_High_Falls'), - ('en.wikipedia.org_wiki_Grammy_Award_for_Best_New_Artist#1960s.html', - 'https://en.wikipedia.org/wiki/Grammy_Award_for_Best_New_Artist#1960s'), - ('en.wikipedia.org_wiki_Grammy_Award_for_Song_of_the_Year#2000s.html', - 'https://en.wikipedia.org/wiki/Grammy_Award_for_Song_of_the_Year#2000s'), - ('en.wikipedia.org_wiki_Grand_Slam_(tennis)#.html', 'https://en.wikipedia.org/wiki/Grand_Slam_(tennis)#'), - ('en.wikipedia.org_wiki_Grateful_Dead#Main_career_(1967–1995).html', - 'https://en.wikipedia.org/wiki/Grateful_Dead#Main_career_(1967%E2%80%931995)'), - ('en.wikipedia.org_wiki_Grenoble#Population.html', 'https://en.wikipedia.org/wiki/Grenoble#Population'), - ('en.wikipedia.org_wiki_Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018,serve%20non%2Dconsecutive%20presidential%20terms..html', - 'https://en.wikipedia.org/wiki/Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018,serve%20non%2Dconsecutive%20presidential%20terms.'), - ('en.wikipedia.org_wiki_Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018.html', - 'https://en.wikipedia.org/wiki/Grover_Cleveland#_~_text=Stephen%20Grover%20Cleveland%20(March%2018'), - ('en.wikipedia.org_wiki_Gymnastics_at_the_1992_Summer_Olympics_%E2%80%93_Women%27s_artistic_team_all-around.html', - 'https://en.wikipedia.org/wiki/Gymnastics_at_the_1992_Summer_Olympics_%E2%80%93_Women%27s_artistic_team_all-around'), - ('en.wikipedia.org_wiki_H%C3%A4n.html', 'https://en.wikipedia.org/wiki/H%C3%A4n'), - ('en.wikipedia.org_wiki_H%C3%B4tel_Matignon.html', 'https://en.wikipedia.org/wiki/H%C3%B4tel_Matignon'), - ('en.wikipedia.org_wiki_Halley%27s_Comet.html', 'https://en.wikipedia.org/wiki/Halley%27s_Comet'), - ('en.wikipedia.org_wiki_Hans_Zimmer#Grammy_Awards.html', 'https://en.wikipedia.org/wiki/Hans_Zimmer#Grammy_Awards'), - ('en.wikipedia.org_wiki_Happy_Days#Characters.html', 'https://en.wikipedia.org/wiki/Happy_Days#Characters'), - ('en.wikipedia.org_wiki_Harper%27s_Magazine.html', 'https://en.wikipedia.org/wiki/Harper%27s_Magazine'), - ('en.wikipedia.org_wiki_Harry_Potter_and_the_Philosopher%27s_Stone.html', - 'https://en.wikipedia.org/wiki/Harry_Potter_and_the_Philosopher%27s_Stone'), - ('en.wikipedia.org_wiki_Hatfield_College.html', 'https://en.wikipedia.org/wiki/Hatfield_College'), - ('en.wikipedia.org_wiki_Hayao_Miyazaki#Views.html', 'https://en.wikipedia.org/wiki/Hayao_Miyazaki#Views'), - ('en.wikipedia.org_wiki_Heaven%27s_Gate_(religious_group)#Nike_Decades.html', - 'https://en.wikipedia.org/wiki/Heaven%27s_Gate_(religious_group)#Nike_Decades'), - ('en.wikipedia.org_wiki_Hemangiosarcoma#Treatments.html', 'https://en.wikipedia.org/wiki/Hemangiosarcoma#Treatments'), - ('en.wikipedia.org_wiki_Hermann_G%C3%B6ring.html', 'https://en.wikipedia.org/wiki/Hermann_G%C3%B6ring'), - ('en.wikipedia.org_wiki_Herzog_%26_de_Meuron.html', 'https://en.wikipedia.org/wiki/Herzog_%26_de_Meuron'), - ('en.wikipedia.org_wiki_Hind%27s_Hall.html', 'https://en.wikipedia.org/wiki/Hind%27s_Hall'), - ('en.wikipedia.org_wiki_Home_Alone_2__Lost_in_New_York#Cast.html', - 'https://en.wikipedia.org/wiki/Home_Alone_2:_Lost_in_New_York#Cast'), - ('en.wikipedia.org_wiki_Hot_Rock_%26_Alternative_Songs.html', - 'https://en.wikipedia.org/wiki/Hot_Rock_%26_Alternative_Songs'), - ('en.wikipedia.org_wiki_Hoxie.html', 'https://en.wikipedia.org/wiki/Hoxie'), - ('en.wikipedia.org_wiki_Hudson%27s_Bay_Company.html', 'https://en.wikipedia.org/wiki/Hudson%27s_Bay_Company'), - ('en.wikipedia.org_wiki_Hugh_Hefner#Early_life_and_education.html', - 'https://en.wikipedia.org/wiki/Hugh_Hefner#Early_life_and_education'), - ('en.wikipedia.org_wiki_ICC_men%27s_player_rankings#Top_10_Test_batsmen.html', - 'https://en.wikipedia.org/wiki/ICC_men%27s_player_rankings#Top_10_Test_batsmen'), - ('en.wikipedia.org_wiki_IPad#.html', 'https://en.wikipedia.org/wiki/IPad#'), - ('en.wikipedia.org_wiki_IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral,released%20on%20November%203%2C%202017..html', - 'https://en.wikipedia.org/wiki/IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral,released%20on%20November%203%2C%202017.'), - ('en.wikipedia.org_wiki_IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral.html', - 'https://en.wikipedia.org/wiki/IPhone_X#_~_text=The%20iPhone%20X%20(Roman%20numeral'), - ('en.wikipedia.org_wiki_Ice_hockey_at_the_2010_Winter_Olympics_%E2%80%93_Men%27s_tournament.html', - 'https://en.wikipedia.org/wiki/Ice_hockey_at_the_2010_Winter_Olympics_%E2%80%93_Men%27s_tournament'), - ('en.wikipedia.org_wiki_If_You_Want_Blood_You%27ve_Got_It.html', - 'https://en.wikipedia.org/wiki/If_You_Want_Blood_You%27ve_Got_It'), - ('en.wikipedia.org_wiki_Iga_%C5%9Awi%C4%85tek.html', 'https://en.wikipedia.org/wiki/Iga_%C5%9Awi%C4%85tek'), - ('en.wikipedia.org_wiki_Infinite_Jest#Adaptations.html', 'https://en.wikipedia.org/wiki/Infinite_Jest#Adaptations'), - ('en.wikipedia.org_wiki_Interstate_94_in_Michigan#.html', 'https://en.wikipedia.org/wiki/Interstate_94_in_Michigan#'), - ('en.wikipedia.org_wiki_Is%C3%A8re#Principal_towns.html', 'https://en.wikipedia.org/wiki/Is%C3%A8re#Principal_towns'), - ('en.wikipedia.org_wiki_Istiklal_Mosque.html', 'https://en.wikipedia.org/wiki/Istiklal_Mosque'), - ('en.wikipedia.org_wiki_It%27s_Always_Sunny_in_Philadelphia.html', - 'https://en.wikipedia.org/wiki/It%27s_Always_Sunny_in_Philadelphia'), - ('en.wikipedia.org_wiki_Iv%C3%A1n_Rodr%C3%ADguez.html', 'https://en.wikipedia.org/wiki/Iv%C3%A1n_Rodr%C3%ADguez'), - ('en.wikipedia.org_wiki_J%C3%BCrgen_Warnke.html', 'https://en.wikipedia.org/wiki/J%C3%BCrgen_Warnke'), - ('en.wikipedia.org_wiki_Jackie_Robinson#.html', 'https://en.wikipedia.org/wiki/Jackie_Robinson#'), - ('en.wikipedia.org_wiki_James_%22J.T.%22_Taylor.html', 'https://en.wikipedia.org/wiki/James_%22J.T.%22_Taylor'), - ('en.wikipedia.org_wiki_James_Cameron#Filmography.html', 'https://en.wikipedia.org/wiki/James_Cameron#Filmography'), - ('en.wikipedia.org_wiki_Jane_Austen#List_of_works.html', 'https://en.wikipedia.org/wiki/Jane_Austen#List_of_works'), - ('en.wikipedia.org_wiki_Japanese_aircraft_carrier_Hiy%C5%8D.html', - 'https://en.wikipedia.org/wiki/Japanese_aircraft_carrier_Hiy%C5%8D'), - ('en.wikipedia.org_wiki_Ji%C5%99%C3%AD_Bro%C5%BEek.html', 'https://en.wikipedia.org/wiki/Ji%C5%99%C3%AD_Bro%C5%BEek'), - ('en.wikipedia.org_wiki_Jo%C3%ABl_Retornaz.html', 'https://en.wikipedia.org/wiki/Jo%C3%ABl_Retornaz'), - ('en.wikipedia.org_wiki_Joe_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Joe_Jonas#Personal_life'), - ('en.wikipedia.org_wiki_Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November,actor%2C%20comedian%20and%20television%20presenter..html', - 'https://en.wikipedia.org/wiki/Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November,actor%2C%20comedian%20and%20television%20presenter.'), - ('en.wikipedia.org_wiki_Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November.html', - 'https://en.wikipedia.org/wiki/Joel_McHale#_~_text=Joel%20Edward%20McHale%20(born%20November'), - ('en.wikipedia.org_wiki_John_Snow#.html', 'https://en.wikipedia.org/wiki/John_Snow#'), - ('en.wikipedia.org_wiki_John_of_Lancaster.html', 'https://en.wikipedia.org/wiki/John_of_Lancaster'), - ('en.wikipedia.org_wiki_Jonas_Brothers#Members.html', 'https://en.wikipedia.org/wiki/Jonas_Brothers#Members'), - ('en.wikipedia.org_wiki_Jos%C3%A9_Loiola.html', 'https://en.wikipedia.org/wiki/Jos%C3%A9_Loiola'), - ('en.wikipedia.org_wiki_Jos%C3%A9_Mar%C3%ADa_Arizmendiarrieta.html', - 'https://en.wikipedia.org/wiki/Jos%C3%A9_Mar%C3%ADa_Arizmendiarrieta'), - ('en.wikipedia.org_wiki_José_Saramago#.html', 'https://en.wikipedia.org/wiki/Jos%C3%A9_Saramago#'), - ('en.wikipedia.org_wiki_Juan_Gonz%C3%A1lez_(baseball).html', - 'https://en.wikipedia.org/wiki/Juan_Gonz%C3%A1lez_(baseball)'), - ('en.wikipedia.org_wiki_Juneau.html', 'https://en.wikipedia.org/wiki/Juneau'), - ('en.wikipedia.org_wiki_K%C5%8Dnan_Railway_K%C5%8Dnan_Line.html', - 'https://en.wikipedia.org/wiki/K%C5%8Dnan_Railway_K%C5%8Dnan_Line'), - ('en.wikipedia.org_wiki_Kala_Wewa#History.html', 'https://en.wikipedia.org/wiki/Kala_Wewa#History'), - ('en.wikipedia.org_wiki_Kelley_O%27Hara.html', 'https://en.wikipedia.org/wiki/Kelley_O%27Hara'), - ('en.wikipedia.org_wiki_Kennebec_County.html', 'https://en.wikipedia.org/wiki/Kennebec_County'), - ('en.wikipedia.org_wiki_Kente_cloth#cite_note-CNN-2020-06-08-18.html', - 'https://en.wikipedia.org/wiki/Kente_cloth#cite_note-CNN-2020-06-08-18'), - ('en.wikipedia.org_wiki_Kevin_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Kevin_Jonas#Personal_life'), - ('en.wikipedia.org_wiki_Key_West#_~_text=The%20southernmost%20location%20that%20the,apart%20at%20their%20closest%20points..html', - 'https://en.wikipedia.org/wiki/Key_West#_~_text=The%20southernmost%20location%20that%20the,apart%20at%20their%20closest%20points.'), - ('en.wikipedia.org_wiki_Key_West#_~_text=The%20southernmost%20location%20that%20the.html', - 'https://en.wikipedia.org/wiki/Key_West#_~_text=The%20southernmost%20location%20that%20the'), - ('en.wikipedia.org_wiki_Kill_%27Em_All.html', 'https://en.wikipedia.org/wiki/Kill_%27Em_All'), - ('en.wikipedia.org_wiki_King%27s_Service_Medal.html', 'https://en.wikipedia.org/wiki/King%27s_Service_Medal'), - ('en.wikipedia.org_wiki_Kundavai_Pir%C4%81ttiy%C4%81r.html', - 'https://en.wikipedia.org/wiki/Kundavai_Pir%C4%81ttiy%C4%81r'), - ('en.wikipedia.org_wiki_LaMarcus_Aldridge#2013%E2%80%9314_season.html', - 'https://en.wikipedia.org/wiki/LaMarcus_Aldridge#2013%E2%80%9314_season'), - ('en.wikipedia.org_wiki_La_Casita-Garciasville.html', 'https://en.wikipedia.org/wiki/La_Casita-Garciasville'), - ('en.wikipedia.org_wiki_La_La_Land#Cast.html', 'https://en.wikipedia.org/wiki/La_La_Land#Cast'), - ('en.wikipedia.org_wiki_La_Llorona#Literature.html', 'https://en.wikipedia.org/wiki/La_Llorona#Literature'), - ('en.wikipedia.org_wiki_La_boh%C3%A8me.html', 'https://en.wikipedia.org/wiki/La_boh%C3%A8me'), - ('en.wikipedia.org_wiki_Latrobe.html', 'https://en.wikipedia.org/wiki/Latrobe'), - ('en.wikipedia.org_wiki_Laurentian_Library#Architecture.html', - 'https://en.wikipedia.org/wiki/Laurentian_Library#Architecture'), - ('en.wikipedia.org_wiki_Lauryn_Hill#Early_life.html', 'https://en.wikipedia.org/wiki/Lauryn_Hill#Early_life'), - ('en.wikipedia.org_wiki_Law_%26_Order__Special_Victims_Unit.html', - 'https://en.wikipedia.org/wiki/Law_%26_Order__Special_Victims_Unit'), - ('en.wikipedia.org_wiki_Legion_of_Honour#Legal_status_and_leadership.html', - 'https://en.wikipedia.org/wiki/Legion_of_Honour#Legal_status_and_leadership'), - ('en.wikipedia.org_wiki_Levi_Strauss_%26_Co.#Blue_jeans_era_(1960s%E2%80%931980s).html', - 'https://en.wikipedia.org/wiki/Levi_Strauss_%26_Co.#Blue_jeans_era_(1960s%E2%80%931980s)'), - ('en.wikipedia.org_wiki_Life_%26_Casualty_Tower.html', 'https://en.wikipedia.org/wiki/Life_%26_Casualty_Tower'), - ('en.wikipedia.org_wiki_Lincoln.html', 'https://en.wikipedia.org/wiki/Lincoln'), - ('en.wikipedia.org_wiki_List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became,major%20party%2C%20namely%20the%20Democr.html', - 'https://en.wikipedia.org/wiki/List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became,major%20party%2C%20namely%20the%20Democr'), - ('en.wikipedia.org_wiki_List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became.html', - 'https://en.wikipedia.org/wiki/List_of_African-American_United_States_presidential_and_vice_presidential_candidates#_~_text=In%201848%2C%20Frederick%20Douglass%20became'), - ('en.wikipedia.org_wiki_List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling,nomination%20at%20the%20Grammy%20Awards..html', - 'https://en.wikipedia.org/wiki/List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling,nomination%20at%20the%20Grammy%20Awards.'), - ('en.wikipedia.org_wiki_List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling.html', - 'https://en.wikipedia.org/wiki/List_of_Billboard_200_number-one_albums_of_1999#_~_text=Millennium%20became%20the%20best%2Dselling'), - ('en.wikipedia.org_wiki_List_of_Copa_Am%C3%A9rica_finals#Finals.html', - 'https://en.wikipedia.org/wiki/List_of_Copa_Am%C3%A9rica_finals#Finals'), - ('en.wikipedia.org_wiki_List_of_Hot_Ones_episodes#Season_23_(2024).html', - 'https://en.wikipedia.org/wiki/List_of_Hot_Ones_episodes#Season_23_(2024)'), - ('en.wikipedia.org_wiki_List_of_How_I_Met_Your_Mother_episodes#Season_3_(2007%E2%80%9308).html', - 'https://en.wikipedia.org/wiki/List_of_How_I_Met_Your_Mother_episodes#Season_3_(2007%E2%80%9308)'), - ('en.wikipedia.org_wiki_List_of_Liverpool_F.C._managers#Managers.html', - 'https://en.wikipedia.org/wiki/List_of_Liverpool_F.C._managers#Managers'), - ('en.wikipedia.org_wiki_List_of_MLS_Cup_finals#Results_by_team.html', - 'https://en.wikipedia.org/wiki/List_of_MLS_Cup_finals#Results_by_team'), - ('en.wikipedia.org_wiki_List_of_New_Girl_episodes#Season_2_(2012%E2%80%9313).html', - 'https://en.wikipedia.org/wiki/List_of_New_Girl_episodes#Season_2_(2012%E2%80%9313)'), - ('en.wikipedia.org_wiki_List_of_New_Zealand_Test_cricket_records#Most_career_runs.html', - 'https://en.wikipedia.org/wiki/List_of_New_Zealand_Test_cricket_records#Most_career_runs'), - ('en.wikipedia.org_wiki_List_of_Nobel_laureates_in_Literature#1960.html', - 'https://en.wikipedia.org/wiki/List_of_Nobel_laureates_in_Literature#1960'), - ('en.wikipedia.org_wiki_List_of_Northern_Ireland_international_footballers#List_of_players.html', - 'https://en.wikipedia.org/wiki/List_of_Northern_Ireland_international_footballers#List_of_players'), - ('en.wikipedia.org_wiki_List_of_S%26P_500_companies.html', - 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'), - ('en.wikipedia.org_wiki_List_of_Star_Wars_film_actors#Introduced_in_The_Skywalker_Saga.html', - 'https://en.wikipedia.org/wiki/List_of_Star_Wars_film_actors#Introduced_in_The_Skywalker_Saga'), - ('en.wikipedia.org_wiki_List_of_Walt_Disney_Animation_Studios_films#ep29.html', - 'https://en.wikipedia.org/wiki/List_of_Walt_Disney_Animation_Studios_films#ep29'), - ('en.wikipedia.org_wiki_List_of_Wildlife_Species_at_Risk_(Canada)#Amphibians.html', - 'https://en.wikipedia.org/wiki/List_of_Wildlife_Species_at_Risk_(Canada)#Amphibians'), - ('en.wikipedia.org_wiki_List_of_World_Series_champions#World_Series_results.html', - 'https://en.wikipedia.org/wiki/List_of_World_Series_champions#World_Series_results'), - ('en.wikipedia.org_wiki_List_of_awards_and_nominations_received_by_Katy_Perry#Awards_and_nominations.html', - 'https://en.wikipedia.org/wiki/List_of_awards_and_nominations_received_by_Katy_Perry#Awards_and_nominations'), - ('en.wikipedia.org_wiki_List_of_awards_and_nominations_received_by_Taylor_Swift#Honorary_degree.html', - 'https://en.wikipedia.org/wiki/List_of_awards_and_nominations_received_by_Taylor_Swift#Honorary_degree'), - ('en.wikipedia.org_wiki_List_of_minor_planets__8001%E2%80%939000#323c.html', - 'https://en.wikipedia.org/wiki/List_of_minor_planets:_8001%E2%80%939000#323c'), - ('en.wikipedia.org_wiki_List_of_presidents_of_France#Presidents_2.html', - 'https://en.wikipedia.org/wiki/List_of_presidents_of_France#Presidents_2'), - ('en.wikipedia.org_wiki_List_of_presidents_of_the_United_States#Presidents.html', - 'https://en.wikipedia.org/wiki/List_of_presidents_of_the_United_States#Presidents'), - ('en.wikipedia.org_wiki_List_of_prime_ministers_of_Australia#.html', - 'https://en.wikipedia.org/wiki/List_of_prime_ministers_of_Australia#'), - ('en.wikipedia.org_wiki_List_of_prime_ministers_of_Canada#Prime_ministers.html', - 'https://en.wikipedia.org/wiki/List_of_prime_ministers_of_Canada#Prime_ministers'), - ('en.wikipedia.org_wiki_List_of_vetoed_United_Nations_Security_Council_resolutions#Resolutions.html', - 'https://en.wikipedia.org/wiki/List_of_vetoed_United_Nations_Security_Council_resolutions#Resolutions'), - ('en.wikipedia.org_wiki_List_of_waterfalls_by_height#By_overall_height.html', - 'https://en.wikipedia.org/wiki/List_of_waterfalls_by_height#By_overall_height'), - ('en.wikipedia.org_wiki_Lists_of_airports#By_code.html', 'https://en.wikipedia.org/wiki/Lists_of_airports#By_code'), - ('en.wikipedia.org_wiki_Lists_of_earthquakes#Largest_earthquakes_by_magnitude.html', - 'https://en.wikipedia.org/wiki/Lists_of_earthquakes#Largest_earthquakes_by_magnitude'), - ('en.wikipedia.org_wiki_Literary_and_Philosophical_Society_of_Newcastle_upon_Tyne#.html', - 'https://en.wikipedia.org/wiki/Literary_and_Philosophical_Society_of_Newcastle_upon_Tyne#'), - ('en.wikipedia.org_wiki_London_Broncos#1994%E2%80%932005__Broncos_and_Super_League.html', - 'https://en.wikipedia.org/wiki/London_Broncos#1994%E2%80%932005__Broncos_and_Super_League'), - ('en.wikipedia.org_wiki_Louis_XVI#.html', 'https://en.wikipedia.org/wiki/Louis_XVI#'), - ('en.wikipedia.org_wiki_Lynn_Rogers_(politician)#Career.html', - 'https://en.wikipedia.org/wiki/Lynn_Rogers_(politician)#Career'), - ('en.wikipedia.org_wiki_Ma%C5%82gorzata_Ro%C5%BCniecka.html', - 'https://en.wikipedia.org/wiki/Ma%C5%82gorzata_Ro%C5%BCniecka'), - ('en.wikipedia.org_wiki_MacArthur_Fellows_Program#.html', 'https://en.wikipedia.org/wiki/MacArthur_Fellows_Program#'), - ('en.wikipedia.org_wiki_Mali#Ethnic_groups.html', 'https://en.wikipedia.org/wiki/Mali#Ethnic_groups'), - ('en.wikipedia.org_wiki_Marathon_County.html', 'https://en.wikipedia.org/wiki/Marathon_County'), - ('en.wikipedia.org_wiki_Margaret_Mansfield.html', 'https://en.wikipedia.org/wiki/Margaret_Mansfield'), - ('en.wikipedia.org_wiki_Mark_Harvey_(arachnologist)#Achievements,_awards_and_recognition.html', - 'https://en.wikipedia.org/wiki/Mark_Harvey_(arachnologist)#Achievements,_awards_and_recognition'), - ('en.wikipedia.org_wiki_Mark_Harvey_(arachnologist)#Achievements.html', - 'https://en.wikipedia.org/wiki/Mark_Harvey_(arachnologist)#Achievements'), - ('en.wikipedia.org_wiki_Mark_O%27Halloran_(rugby_league).html', - 'https://en.wikipedia.org/wiki/Mark_O%27Halloran_(rugby_league)'), - ('en.wikipedia.org_wiki_McDonald%27s.html', 'https://en.wikipedia.org/wiki/McDonald%27s'), - ('en.wikipedia.org_wiki_Meat_Loaf#Personal_life.html', 'https://en.wikipedia.org/wiki/Meat_Loaf#Personal_life'), - ('en.wikipedia.org_wiki_Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as,former%20members%20of%20the%20Commonwealth..html', - 'https://en.wikipedia.org/wiki/Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as,former%20members%20of%20the%20Commonwealth.'), - ('en.wikipedia.org_wiki_Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as.html', - 'https://en.wikipedia.org/wiki/Member_states_of_the_Commonwealth_of_Nations#_~_text=The%20Republic%20of%20Ireland%20(as'), - ('en.wikipedia.org_wiki_Miasma_theory#.html', 'https://en.wikipedia.org/wiki/Miasma_theory#'), - ('en.wikipedia.org_wiki_Michelson%E2%80%93Morley_experiment.html', - 'https://en.wikipedia.org/wiki/Michelson%E2%80%93Morley_experiment'), - ('en.wikipedia.org_wiki_Milo%C5%A1_Beleslin.html', 'https://en.wikipedia.org/wiki/Milo%C5%A1_Beleslin'), - ('en.wikipedia.org_wiki_Minamidait%C5%8Djima.html', 'https://en.wikipedia.org/wiki/Minamidait%C5%8Djima'), - ('en.wikipedia.org_wiki_Mona_Lisa#Refuge,_theft,_and_vandalism.html', - 'https://en.wikipedia.org/wiki/Mona_Lisa#Refuge,_theft,_and_vandalism'), - ('en.wikipedia.org_wiki_Mona_Lisa#Refuge.html', 'https://en.wikipedia.org/wiki/Mona_Lisa#Refuge'), - ('en.wikipedia.org_wiki_Morgana_King#Film_debut.html', 'https://en.wikipedia.org/wiki/Morgana_King#Film_debut'), - ('en.wikipedia.org_wiki_Mount_Rushmore#History.html', 'https://en.wikipedia.org/wiki/Mount_Rushmore#History'), - ('en.wikipedia.org_wiki_Murray_State_University#Former_Presidents_of_the_University.html', - 'https://en.wikipedia.org/wiki/Murray_State_University#Former_Presidents_of_the_University'), - ('en.wikipedia.org_wiki_NCAA_Division_III_men%27s_cross_country_championships.html', - 'https://en.wikipedia.org/wiki/NCAA_Division_III_men%27s_cross_country_championships'), - ('en.wikipedia.org_wiki_NCT_(group)#Members.html', 'https://en.wikipedia.org/wiki/NCT_(group)#Members'), - ('en.wikipedia.org_wiki_Nancy_Farmer#Bibliography.html', 'https://en.wikipedia.org/wiki/Nancy_Farmer#Bibliography'), - ('en.wikipedia.org_wiki_Nashville.html', 'https://en.wikipedia.org/wiki/Nashville'), - ('en.wikipedia.org_wiki_Navy_%E2%80%93_Merchant_Marine_Memorial.html', - 'https://en.wikipedia.org/wiki/Navy_%E2%80%93_Merchant_Marine_Memorial'), - ('en.wikipedia.org_wiki_Nelson_Mandela#Imprisonment.html', - 'https://en.wikipedia.org/wiki/Nelson_Mandela#Imprisonment'), - ('en.wikipedia.org_wiki_New_Girl#Episodes.html', 'https://en.wikipedia.org/wiki/New_Girl#Episodes'), - ('en.wikipedia.org_wiki_Nick_Jonas#Personal_life.html', 'https://en.wikipedia.org/wiki/Nick_Jonas#Personal_life'), - ('en.wikipedia.org_wiki_Nike.html', 'https://en.wikipedia.org/wiki/Nike'), - ('en.wikipedia.org_wiki_Nim_Chimpsky#Quotations.html', 'https://en.wikipedia.org/wiki/Nim_Chimpsky#Quotations'), - ('en.wikipedia.org_wiki_Nineteenth_Dynasty_of_Egypt#Pharaohs_of_the_19th_Dynasty.html', - 'https://en.wikipedia.org/wiki/Nineteenth_Dynasty_of_Egypt#Pharaohs_of_the_19th_Dynasty'), - ('en.wikipedia.org_wiki_Noam_Chomsky#In_academia.html', 'https://en.wikipedia.org/wiki/Noam_Chomsky#In_academia'), - ('en.wikipedia.org_wiki_Nobel_Prize#Multiple_laureates.html', - 'https://en.wikipedia.org/wiki/Nobel_Prize#Multiple_laureates'), - ('en.wikipedia.org_wiki_Norman.html', 'https://en.wikipedia.org/wiki/Norman'), - ('en.wikipedia.org_wiki_Northern_California#Cities.html', 'https://en.wikipedia.org/wiki/Northern_California#Cities'), - ('en.wikipedia.org_wiki_Novake,_Polj%C4%8Dane.html', 'https://en.wikipedia.org/wiki/Novake,_Polj%C4%8Dane'), - ('en.wikipedia.org_wiki_Ol%27_Dirty_Bastard.html', 'https://en.wikipedia.org/wiki/Ol%27_Dirty_Bastard'), - ('en.wikipedia.org_wiki_Ospedale_della_Piet%C3%A0.html', 'https://en.wikipedia.org/wiki/Ospedale_della_Piet%C3%A0'), - ('en.wikipedia.org_wiki_Paducah.html', 'https://en.wikipedia.org/wiki/Paducah'), - ('en.wikipedia.org_wiki_Pain_%26_Gain.html', 'https://en.wikipedia.org/wiki/Pain_%26_Gain'), - ('en.wikipedia.org_wiki_Panic_Room#Theatrical_run.html', 'https://en.wikipedia.org/wiki/Panic_Room#Theatrical_run'), - ('en.wikipedia.org_wiki_Patrick_Kane#Early_life.html', 'https://en.wikipedia.org/wiki/Patrick_Kane#Early_life'), - ('en.wikipedia.org_wiki_Pelee.html', 'https://en.wikipedia.org/wiki/Pelee'), - ('en.wikipedia.org_wiki_Phil_Quartararo#Warner_Bros._Records.html', - 'https://en.wikipedia.org/wiki/Phil_Quartararo#Warner_Bros._Records'), - ('en.wikipedia.org_wiki_Philadelphia_Waterdogs#Season_results.html', - 'https://en.wikipedia.org/wiki/Philadelphia_Waterdogs#Season_results'), - ('en.wikipedia.org_wiki_Philip_K._Dick#Career.html', 'https://en.wikipedia.org/wiki/Philip_K._Dick#Career'), - ('en.wikipedia.org_wiki_Philosophi%C3%A6_Naturalis_Principia_Mathematica.html', - 'https://en.wikipedia.org/wiki/Philosophi%C3%A6_Naturalis_Principia_Mathematica'), - ('en.wikipedia.org_wiki_Pok%C3%A9mon.html', 'https://en.wikipedia.org/wiki/Pok%C3%A9mon'), - ('en.wikipedia.org_wiki_Pok%C3%A9mon_Diamond_and_Pearl.html', - 'https://en.wikipedia.org/wiki/Pok%C3%A9mon_Diamond_and_Pearl'), - ('en.wikipedia.org_wiki_Pok%C3%A9mon_World_Championships.html', - 'https://en.wikipedia.org/wiki/Pok%C3%A9mon_World_Championships'), - ('en.wikipedia.org_wiki_Portage_County.html', 'https://en.wikipedia.org/wiki/Portage_County'), - ('en.wikipedia.org_wiki_Portland.html', 'https://en.wikipedia.org/wiki/Portland'), - ('en.wikipedia.org_wiki_Prague#_~_text=Prague%20is%20located%20approximately%20at,N%2014%C2%B025%E2%80%B2E..html', - 'https://en.wikipedia.org/wiki/Prague#_~_text=Prague%20is%20located%20approximately%20at,N%2014%C2%B025%E2%80%B2E.'), - ('en.wikipedia.org_wiki_Prague#_~_text=Prague%20is%20located%20approximately%20at.html', - 'https://en.wikipedia.org/wiki/Prague#_~_text=Prague%20is%20located%20approximately%20at'), - ('en.wikipedia.org_wiki_President_of_the_United_States#History_and_development.html', - 'https://en.wikipedia.org/wiki/President_of_the_United_States#History_and_development'), - ('en.wikipedia.org_wiki_Pride_%26_Prejudice_(2005_film).html', - 'https://en.wikipedia.org/wiki/Pride_%26_Prejudice_(2005_film)'), - ('en.wikipedia.org_wiki_Pride_and_Prejudice#Film,_television_and_theatre.html', - 'https://en.wikipedia.org/wiki/Pride_and_Prejudice#Film,_television_and_theatre'), - ('en.wikipedia.org_wiki_Pride_and_Prejudice#Film.html', 'https://en.wikipedia.org/wiki/Pride_and_Prejudice#Film'), - ('en.wikipedia.org_wiki_Prince_Andrew.html', 'https://en.wikipedia.org/wiki/Prince_Andrew'), - ('en.wikipedia.org_wiki_Prince_Edward.html', 'https://en.wikipedia.org/wiki/Prince_Edward'), - ('en.wikipedia.org_wiki_Prince_Philip.html', 'https://en.wikipedia.org/wiki/Prince_Philip'), - ('en.wikipedia.org_wiki_Prinsjesdag#History.html', 'https://en.wikipedia.org/wiki/Prinsjesdag#History'), - ('en.wikipedia.org_wiki_Pro_Bowl#Players_with_most_invitations.html', - 'https://en.wikipedia.org/wiki/Pro_Bowl#Players_with_most_invitations'), - ('en.wikipedia.org_wiki_Professional_wrestling#Occupational_hazards.html', - 'https://en.wikipedia.org/wiki/Professional_wrestling#Occupational_hazards'), - ('en.wikipedia.org_wiki_Province_of_A_Coru%C3%B1a.html', 'https://en.wikipedia.org/wiki/Province_of_A_Coru%C3%B1a'), - ('en.wikipedia.org_wiki_Provo.html', 'https://en.wikipedia.org/wiki/Provo'), - ('en.wikipedia.org_wiki_Pseudoscorpion#Classification.html', - 'https://en.wikipedia.org/wiki/Pseudoscorpion#Classification'), - ('en.wikipedia.org_wiki_Pterostylis_aestiva#Description.html', - 'https://en.wikipedia.org/wiki/Pterostylis_aestiva#Description'), - ('en.wikipedia.org_wiki_Pulitzer_Prize_for_Fiction#Repeat_winners.html', - 'https://en.wikipedia.org/wiki/Pulitzer_Prize_for_Fiction#Repeat_winners'), - ('en.wikipedia.org_wiki_Qufu#Geography.html', 'https://en.wikipedia.org/wiki/Qufu#Geography'), - ('en.wikipedia.org_wiki_Red_Hot_Chili_Peppers#Discography.html', - 'https://en.wikipedia.org/wiki/Red_Hot_Chili_Peppers#Discography'), - ('en.wikipedia.org_wiki_Remy%27s_Ratatouille_Adventure.html', - 'https://en.wikipedia.org/wiki/Remy%27s_Ratatouille_Adventure'), - ('en.wikipedia.org_wiki_Ren%C4%8De.html', 'https://en.wikipedia.org/wiki/Ren%C4%8De'), - ('en.wikipedia.org_wiki_Resident_Evil__Revelations_2#.html', - 'https://en.wikipedia.org/wiki/Resident_Evil__Revelations_2#'), - ('en.wikipedia.org_wiki_Revive_%26_Restore.html', 'https://en.wikipedia.org/wiki/Revive_%26_Restore'), - ('en.wikipedia.org_wiki_River_Avon,_Bristol#Hydrology_and_water_quality.html', - 'https://en.wikipedia.org/wiki/River_Avon,_Bristol#Hydrology_and_water_quality'), - ('en.wikipedia.org_wiki_River_Avon.html', 'https://en.wikipedia.org/wiki/River_Avon'), - ('en.wikipedia.org_wiki_Roberto_%C3%81lamo.html', 'https://en.wikipedia.org/wiki/Roberto_%C3%81lamo'), - ('en.wikipedia.org_wiki_Rodrigo_Pacheco_M%C3%A9ndez.html', - 'https://en.wikipedia.org/wiki/Rodrigo_Pacheco_M%C3%A9ndez'), - ('en.wikipedia.org_wiki_Rubik%27s_Cube.html', 'https://en.wikipedia.org/wiki/Rubik%27s_Cube'), - ('en.wikipedia.org_wiki_Russian_Bishop%27s_House.html', 'https://en.wikipedia.org/wiki/Russian_Bishop%27s_House'), - ('en.wikipedia.org_wiki_Sailing_at_the_1984_Summer_Olympics_%E2%80%93_470.html', - 'https://en.wikipedia.org/wiki/Sailing_at_the_1984_Summer_Olympics_%E2%80%93_470'), - ('en.wikipedia.org_wiki_San_Jose.html', 'https://en.wikipedia.org/wiki/San_Jose'), - ('en.wikipedia.org_wiki_Saturday_Night_Live#Controversies.html', - 'https://en.wikipedia.org/wiki/Saturday_Night_Live#Controversies'), - ('en.wikipedia.org_wiki_Saving_Private_Ryan#Reception.html', - 'https://en.wikipedia.org/wiki/Saving_Private_Ryan#Reception'), - ('en.wikipedia.org_wiki_Schindler%27s_List.html', 'https://en.wikipedia.org/wiki/Schindler%27s_List'), - ('en.wikipedia.org_wiki_Se%C3%A1n_Mac_Diarmada.html', 'https://en.wikipedia.org/wiki/Se%C3%A1n_Mac_Diarmada'), - ('en.wikipedia.org_wiki_Sea_Cliff.html', 'https://en.wikipedia.org/wiki/Sea_Cliff'), - ('en.wikipedia.org_wiki_Sens%C5%8D-ji.html', 'https://en.wikipedia.org/wiki/Sens%C5%8D-ji'), - ('en.wikipedia.org_wiki_Seven_Wonders_of_the_Ancient_World#Wonders.html', - 'https://en.wikipedia.org/wiki/Seven_Wonders_of_the_Ancient_World#Wonders'), - ('en.wikipedia.org_wiki_She%27s_All_That.html', 'https://en.wikipedia.org/wiki/She%27s_All_That'), - ('en.wikipedia.org_wiki_Shrek#Accolades.html', 'https://en.wikipedia.org/wiki/Shrek#Accolades'), - ('en.wikipedia.org_wiki_Shreve,_Lamb_%26_Harmon.html', 'https://en.wikipedia.org/wiki/Shreve,_Lamb_%26_Harmon'), - ('en.wikipedia.org_wiki_Shreve.html', 'https://en.wikipedia.org/wiki/Shreve'), - ('en.wikipedia.org_wiki_Solomon_Sea#Deepest_point.html', 'https://en.wikipedia.org/wiki/Solomon_Sea#Deepest_point'), - ('en.wikipedia.org_wiki_Speech_from_the_throne#Netherlands.html', - 'https://en.wikipedia.org/wiki/Speech_from_the_throne#Netherlands'), - ('en.wikipedia.org_wiki_Spokane_University#Notable_alumni.html', - 'https://en.wikipedia.org/wiki/Spokane_University#Notable_alumni'), - ('en.wikipedia.org_wiki_SpongeBob_SquarePants#Franchise.html', - 'https://en.wikipedia.org/wiki/SpongeBob_SquarePants#Franchise'), - ('en.wikipedia.org_wiki_Spyro_2__Ripto%27s_Rage!.html', 'https://en.wikipedia.org/wiki/Spyro_2:_Ripto%27s_Rage!'), - ('en.wikipedia.org_wiki_St._Philip%27s_College_(United_States).html', - 'https://en.wikipedia.org/wiki/St._Philip%27s_College_(United_States)'), - ('en.wikipedia.org_wiki_Stephen_Hawking#1975%E2%80%931990.html', - 'https://en.wikipedia.org/wiki/Stephen_Hawking#1975%E2%80%931990'), - ('en.wikipedia.org_wiki_Steve_Carell#2004%E2%80%932013__The_Office_and_comedic_roles.html', - 'https://en.wikipedia.org/wiki/Steve_Carell#2004%E2%80%932013__The_Office_and_comedic_roles'), - ('en.wikipedia.org_wiki_Strong_Love_Affair#Track_listing.html', - 'https://en.wikipedia.org/wiki/Strong_Love_Affair#Track_listing'), - ('en.wikipedia.org_wiki_Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers,ravaged%20the%20city%20in%201992..html', - 'https://en.wikipedia.org/wiki/Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers,ravaged%20the%20city%20in%201992.'), - ('en.wikipedia.org_wiki_Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers.html', - 'https://en.wikipedia.org/wiki/Super_Bowl_XXIX#_~_text=The%2049ers%20defeated%20the%20Chargers'), - ('en.wikipedia.org_wiki_Superman_%26_Bugs_Bunny.html', 'https://en.wikipedia.org/wiki/Superman_%26_Bugs_Bunny'), - ('en.wikipedia.org_wiki_Swimming_at_the_1948_Summer_Olympics_%E2%80%93_Men%27s_100_metre_freestyle.html', - 'https://en.wikipedia.org/wiki/Swimming_at_the_1948_Summer_Olympics_%E2%80%93_Men%27s_100_metre_freestyle'), - ('en.wikipedia.org_wiki_Synsphyronus#Species.html', 'https://en.wikipedia.org/wiki/Synsphyronus#Species'), - ('en.wikipedia.org_wiki_Takarazuka,_Hy%C5%8Dgo.html', 'https://en.wikipedia.org/wiki/Takarazuka,_Hy%C5%8Dgo'), - ('en.wikipedia.org_wiki_Takarazuka.html', 'https://en.wikipedia.org/wiki/Takarazuka'), - ('en.wikipedia.org_wiki_Target_House.html', 'https://en.wikipedia.org/wiki/Target_House'), - ('en.wikipedia.org_wiki_Taylor_Swift#.html', 'https://en.wikipedia.org/wiki/Taylor_Swift#'), - ('en.wikipedia.org_wiki_Tennessee_Coal.html', 'https://en.wikipedia.org/wiki/Tennessee_Coal'), - ('en.wikipedia.org_wiki_Tennis_at_the_1988_Summer_Olympics_%E2%80%93_Women%27s_singles.html', - 'https://en.wikipedia.org/wiki/Tennis_at_the_1988_Summer_Olympics_%E2%80%93_Women%27s_singles'), - ('en.wikipedia.org_wiki_Terrorism_in_the_United_States#Deadliest_attacks.html', - 'https://en.wikipedia.org/wiki/Terrorism_in_the_United_States#Deadliest_attacks'), - ('en.wikipedia.org_wiki_Thailand_national_football_team#Coaching_history.html', - 'https://en.wikipedia.org/wiki/Thailand_national_football_team#Coaching_history'), - ('en.wikipedia.org_wiki_That%27s_What_Friends_Are_For.html', - 'https://en.wikipedia.org/wiki/That%27s_What_Friends_Are_For'), - ('en.wikipedia.org_wiki_The_Apollo.html', 'https://en.wikipedia.org/wiki/The_Apollo'), - ('en.wikipedia.org_wiki_The_Band#Discography.html', 'https://en.wikipedia.org/wiki/The_Band#Discography'), - ('en.wikipedia.org_wiki_The_Beach_Boys#History.html', 'https://en.wikipedia.org/wiki/The_Beach_Boys#History'), - ('en.wikipedia.org_wiki_The_Duke_of_Edinburgh%27s_Award.html', - 'https://en.wikipedia.org/wiki/The_Duke_of_Edinburgh%27s_Award'), - ('en.wikipedia.org_wiki_The_Old_Man_and_the_Sea#Reception_and_legacy.html', - 'https://en.wikipedia.org/wiki/The_Old_Man_and_the_Sea#Reception_and_legacy'), - ('en.wikipedia.org_wiki_The_Seduction_of_Joe_Tynan#Awards.html', - 'https://en.wikipedia.org/wiki/The_Seduction_of_Joe_Tynan#Awards'), - ('en.wikipedia.org_wiki_Thirteen_Years%27_War_(1454%E2%80%931466).html', - 'https://en.wikipedia.org/wiki/Thirteen_Years%27_War_(1454%E2%80%931466)'), - ('en.wikipedia.org_wiki_Tiny_Tina%27s_Wonderlands.html', 'https://en.wikipedia.org/wiki/Tiny_Tina%27s_Wonderlands'), - ('en.wikipedia.org_wiki_Tom%C3%A1%C5%A1_Pekhart.html', 'https://en.wikipedia.org/wiki/Tom%C3%A1%C5%A1_Pekhart'), - ('en.wikipedia.org_wiki_Tony_Hawk%27s_Pro_Skater.html', 'https://en.wikipedia.org/wiki/Tony_Hawk%27s_Pro_Skater'), - ('en.wikipedia.org_wiki_Toronto_Maple_Leafs#Season-by-season_record.html', - 'https://en.wikipedia.org/wiki/Toronto_Maple_Leafs#Season-by-season_record'), - ('en.wikipedia.org_wiki_Trans%E2%80%93West_African_Coastal_Highway.html', - 'https://en.wikipedia.org/wiki/Trans%E2%80%93West_African_Coastal_Highway'), - ('en.wikipedia.org_wiki_Triple_Crown_of_Thoroughbred_Racing_(United_States)#Winners_of_the_Triple_Crown.html', - 'https://en.wikipedia.org/wiki/Triple_Crown_of_Thoroughbred_Racing_(United_States)#Winners_of_the_Triple_Crown'), - ('en.wikipedia.org_wiki_Tulsa.html', 'https://en.wikipedia.org/wiki/Tulsa'), - ('en.wikipedia.org_wiki_United_States_federal_executive_departments#Former_departments.html', - 'https://en.wikipedia.org/wiki/United_States_federal_executive_departments#Former_departments'), - ('en.wikipedia.org_wiki_University_of_Illinois_Urbana-Champaign#Notable_alumni_and_faculty.html', - 'https://en.wikipedia.org/wiki/University_of_Illinois_Urbana-Champaign#Notable_alumni_and_faculty'), - ('en.wikipedia.org_wiki_University_of_Maryland,_College_Park#Notable_alumni.html', - 'https://en.wikipedia.org/wiki/University_of_Maryland,_College_Park#Notable_alumni'), - ('en.wikipedia.org_wiki_University_of_Maryland.html', 'https://en.wikipedia.org/wiki/University_of_Maryland'), - ('en.wikipedia.org_wiki_University_of_Oxford#Mathematics_and_sciences.html', - 'https://en.wikipedia.org/wiki/University_of_Oxford#Mathematics_and_sciences'), - ('en.wikipedia.org_wiki_Upper_Franconia#Coat_of_arms.html', - 'https://en.wikipedia.org/wiki/Upper_Franconia#Coat_of_arms'), - ('en.wikipedia.org_wiki_Utah_County.html', 'https://en.wikipedia.org/wiki/Utah_County'), - ('en.wikipedia.org_wiki_Warner_Records#End_of_an_era__Ostin_and_Waronker_depart.html', - 'https://en.wikipedia.org/wiki/Warner_Records#End_of_an_era__Ostin_and_Waronker_depart'), - ('en.wikipedia.org_wiki_Waterville.html', 'https://en.wikipedia.org/wiki/Waterville'), - ('en.wikipedia.org_wiki_Weekly_Sh%C5%8Dnen_Jump.html', 'https://en.wikipedia.org/wiki/Weekly_Sh%C5%8Dnen_Jump'), - ('en.wikipedia.org_wiki_Wendy%27s.html', 'https://en.wikipedia.org/wiki/Wendy%27s'), - ('en.wikipedia.org_wiki_Westfield.html', 'https://en.wikipedia.org/wiki/Westfield'), - ('en.wikipedia.org_wiki_Wheatland.html', 'https://en.wikipedia.org/wiki/Wheatland'), - ('en.wikipedia.org_wiki_Whiplash_(2014_film)#.html', 'https://en.wikipedia.org/wiki/Whiplash_(2014_film)#'), - ('en.wikipedia.org_wiki_William.html', 'https://en.wikipedia.org/wiki/William'), - ('en.wikipedia.org_wiki_William_Mansfield.html', 'https://en.wikipedia.org/wiki/William_Mansfield'), - ('en.wikipedia.org_wiki_Willie_Mays#.html', 'https://en.wikipedia.org/wiki/Willie_Mays#'), - ('en.wikipedia.org_wiki_Women%27s_Tennis_Association.html', - 'https://en.wikipedia.org/wiki/Women%27s_Tennis_Association'), - ('en.wikipedia.org_wiki_Women_in_Film_Crystal_%2B_Lucy_Awards#Dorothy_Arzner_Directors_award.html', - 'https://en.wikipedia.org/wiki/Women_in_Film_Crystal_%2B_Lucy_Awards#Dorothy_Arzner_Directors_award'), - ('en.wikipedia.org_wiki_Wood_County.html', 'https://en.wikipedia.org/wiki/Wood_County'), - ('en.wikipedia.org_wiki_World%27s_Columbian_Exposition.html', - 'https://en.wikipedia.org/wiki/World%27s_Columbian_Exposition'), - ('en.wikipedia.org_wiki_World_War_I#.html', 'https://en.wikipedia.org/wiki/World_War_I#'), - ('en.wikipedia.org_wiki_WrestleMania_13#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_13#Results'), - ('en.wikipedia.org_wiki_WrestleMania_21#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_21#Results'), - ('en.wikipedia.org_wiki_WrestleMania_22#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_22#Results'), - ('en.wikipedia.org_wiki_WrestleMania_23#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_23#Results'), - ('en.wikipedia.org_wiki_WrestleMania_25#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_25#Results'), - ('en.wikipedia.org_wiki_WrestleMania_29#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_29#Results'), - ('en.wikipedia.org_wiki_WrestleMania_IX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_IX#Results'), - ('en.wikipedia.org_wiki_WrestleMania_VII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_VII#Results'), - ('en.wikipedia.org_wiki_WrestleMania_VIII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_VIII#Results'), - ('en.wikipedia.org_wiki_WrestleMania_X-Seven#Results.html', - 'https://en.wikipedia.org/wiki/WrestleMania_X-Seven#Results'), - ('en.wikipedia.org_wiki_WrestleMania_X8#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_X8#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XI#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XI#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XII#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XIV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XIV#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XIX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XIX#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XV#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XX#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XX#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XXIV#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXIV#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XXVI#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXVI#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XXVII#Results.html', 'https://en.wikipedia.org/wiki/WrestleMania_XXVII#Results'), - ('en.wikipedia.org_wiki_WrestleMania_XXVIII#Results.html', - 'https://en.wikipedia.org/wiki/WrestleMania_XXVIII#Results'), - ('en.wikipedia.org_wiki_Wrestling_at_the_1960_Summer_Olympics_%E2%80%93_Men%27s_Greco-Roman_bantamweight.html', - 'https://en.wikipedia.org/wiki/Wrestling_at_the_1960_Summer_Olympics_%E2%80%93_Men%27s_Greco-Roman_bantamweight'), - ('en.wikipedia.org_wiki_Wrestling_at_the_2020_Summer_Olympics#Medalists.html', - 'https://en.wikipedia.org/wiki/Wrestling_at_the_2020_Summer_Olympics#Medalists'), - ('en.wikipedia.org_wiki_Wrestling_at_the_2020_Summer_Olympics_%E2%80%93_Men%27s_freestyle_86_kg.html', - 'https://en.wikipedia.org/wiki/Wrestling_at_the_2020_Summer_Olympics_%E2%80%93_Men%27s_freestyle_86_kg'), - ('en.wikipedia.org_wiki_Wroc%C5%82aw.html', 'https://en.wikipedia.org/wiki/Wroc%C5%82aw'), - ('en.wikipedia.org_wiki_Wroc%C5%82aw_Dwarfs.html', 'https://en.wikipedia.org/wiki/Wroc%C5%82aw_Dwarfs'), - ('en.wikipedia.org_wiki_Wyandotte.html', 'https://en.wikipedia.org/wiki/Wyandotte'), - ('en.wikipedia.org_wiki_You_(TV_series)#Season_2_(2019).html', - 'https://en.wikipedia.org/wiki/You_(TV_series)#Season_2_(2019)'), - ('en.wikipedia.org_wiki_Ypsilanti.html', 'https://en.wikipedia.org/wiki/Ypsilanti'), - ('en.wikipedia.org_wiki_Zac_Efron#Awards_and_nominations.html', - 'https://en.wikipedia.org/wiki/Zac_Efron#Awards_and_nominations'), - ('en.wikipedia.org_wiki_Zack_Wheat#.html', 'https://en.wikipedia.org/wiki/Zack_Wheat#')] - - -def download_one(filename: str, url: str, out_dir: Path) -> dict: - out_path = out_dir / filename - result = {"filename": filename, "url": url, "ok": False, "status": None, "bytes": 0, "error": None} - headers = {"User-Agent": USER_AGENT} - - for attempt in range(1, 4): - try: - request = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(request, timeout=30) as response: - data = response.read() - result["status"] = getattr(response, "status", None) - result["bytes"] = len(data) - if result["status"] and 200 <= result["status"] < 300 and data: - out_path.write_bytes(data) - result["ok"] = True - return result - result["error"] = f"unexpected status/content: status={result['status']} bytes={len(data)}" - except urllib.error.HTTPError as exc: - result["status"] = exc.code - result["error"] = f"HTTPError: {exc.code} {exc.reason}" - if exc.code == 429: - time.sleep(15 * attempt) - continue - if exc.code in (404, 410): - break - except Exception as exc: - result["error"] = f"{type(exc).__name__}: {exc}" - - time.sleep(0.75 * attempt) - - return result - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Download the embedded 470 additional Wikipedia HTML files." - ) - parser.add_argument("output_folder", type=Path, help="Folder where HTML files and manifests will be written.") - args = parser.parse_args() - - out_dir = args.output_folder - out_dir.mkdir(parents=True, exist_ok=True) - manifest_path = out_dir / "manifest.jsonl" - - results = [] - with ThreadPoolExecutor(max_workers=WORKERS) as pool: - futures = [pool.submit(download_one, filename, url, out_dir) for filename, url in DOWNLOADS] - with manifest_path.open("w", encoding="utf-8") as manifest: - for future in as_completed(futures): - result = future.result() - results.append(result) - manifest.write(json.dumps(result, ensure_ascii=False) + "\n") - manifest.flush() - - failures = [result for result in results if not result["ok"]] - if failures: - failures_path = out_dir / "failures.txt" - with failures_path.open("w", encoding="utf-8") as failure_file: - for result in sorted(failures, key=lambda item: item["filename"]): - failure_file.write( - f"{result['filename']}\t{result['url']}\t{result['status']}\t{result['error']}\n" - ) - else: - failures_path = None - - summary = { - "output_folder": str(out_dir), - "requested": len(DOWNLOADS), - "downloaded": sum(1 for result in results if result["ok"]), - "failed": len(failures), - "manifest": str(manifest_path), - "failures": str(failures_path) if failures_path else None, - } - (out_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/e2e-rag/evaluate.py b/e2e-rag/evaluate.py index 9c94ce8890..36b9a1258a 100644 --- a/e2e-rag/evaluate.py +++ b/e2e-rag/evaluate.py @@ -40,8 +40,8 @@ import requests # LLM judge configuration (defaults to local vLLM) -DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions" -DEFAULT_JUDGE_MODEL = "gpt-oss-20b" +DEFAULT_JUDGE_URL = "http://127.0.0.1:8192/v1/chat/completions" +DEFAULT_JUDGE_MODEL = "gpt-oss-20b-mxfp4" OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', '') diff --git a/e2e-rag/oracle_single_shot.py b/e2e-rag/oracle_single_shot.py index 1a39f7fd65..43bd0e29f1 100644 --- a/e2e-rag/oracle_single_shot.py +++ b/e2e-rag/oracle_single_shot.py @@ -46,9 +46,7 @@ DEFAULT_CHECKPOINT_FILE = "oracle_checkpoint.pkl" DEFAULT_SERVICE_URL = "http://localhost:8123/v1/chat/completions" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.3/Meta-Llama-3.3-70B-Instruct" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.1/Meta-Llama-3.1-405B-Instruct-v2" -DEFAULT_MODEL_NAME = "/model/gpt-oss-120b-mxfp4" +DEFAULT_MODEL_NAME = "gpt-oss-120b-mxfp4" DEFAULT_BATCH_SIZE = 1 DEFAULT_TIMEOUT = 2400 # For reasoning model, it should be large enough diff --git a/e2e-rag/params.py b/e2e-rag/params.py index b59e573937..16a62f8553 100644 --- a/e2e-rag/params.py +++ b/e2e-rag/params.py @@ -189,7 +189,7 @@ def suggest_value(self, trial): name="retriever_model", arg_names=["--retriever_model"], type=str, - default="intfloat/e5-base-v2", + default="intfloat_e5-base-v2/e5-base-v2", help="Model to use for embedding-based retrieval", category="common", applies_to=["vector"] @@ -198,7 +198,7 @@ def suggest_value(self, trial): name="reranker_model", arg_names=["--reranker_model"], type=str, - default="colbert-ir/colbertv2.0", + default="colbert-ir_colbertv2.0/colbertv2.0", help="Model to use for reranking (currently unused)", category="common", applies_to=["both"] @@ -320,8 +320,8 @@ def suggest_value(self, trial): name="llm_service_url", arg_names=["--llm_service_url"], type=str, - default="http://127.0.0.1:8123/v1/chat/completions", - help="URL for the LLM service endpoint", + default="http://127.0.0.1:8192/v1/chat/completions", + help="URL for the LLM service endpoint (20B model on port 8192)", category="general", applies_to=["both"] ), @@ -338,14 +338,14 @@ def suggest_value(self, trial): name="query_model", arg_names=["--query_model"], type=str, - default=None, - help="LLM model name/path for query generation (generate_search_queries). Defaults to --llm_model if not set. Example: /model/gpt-oss-120b", + default="gpt-oss-120b-mxfp4", + help="LLM model name/path for query generation (generate_search_queries). Defaults to --llm_model if not set. Example: gpt-oss-120b-mxfp4", category="general", applies_to=["both"] ), # Per-component endpoint overrides. Each defaults to --llm_service_url when # not set; same for the model. Lets you split components across separate - # vLLM servers (e.g. 20B grader on :8124, 120B query/sufficiency on :8123). + # vLLM servers (e.g. 20B grader on :8192, 120B query/sufficiency on :8123). ParamDef( name="grader_service_url", arg_names=["--grader-service-url"], @@ -368,8 +368,8 @@ def suggest_value(self, trial): name="query_service_url", arg_names=["--query-service-url"], type=str, - default=None, - help="LLM service URL for query generation (default: --llm_service_url).", + default="http://127.0.0.1:8123/v1/chat/completions", + help="LLM service URL for query generation (120B model on port 8123, default: --llm_service_url).", category="general", applies_to=["both"] ), diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index 8e1a50f19d..f9c424e3b8 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -48,13 +48,18 @@ export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} export OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-sk-or-v1-****} # Default to local vLLM server -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} + +# Query and sufficiency use 120B model on port 8123 +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} # Judge LLM configuration (for accuracy evaluation) -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} echo " LLM Service URL: ${LLM_SERVICE_URL}" echo " Judge Service URL: ${JUDGE_SERVICE_URL}" @@ -98,6 +103,9 @@ python3 reference_mlperf.py \ --llm_service_url ${LLM_SERVICE_URL} \ --llm_model ${LLM_MODEL} \ --query_model ${QUERY_MODEL} \ + --query-service-url ${QUERY_SERVICE_URL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ --accuracy diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index eb38215279..21e1459de3 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -51,14 +51,17 @@ export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} export OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-sk-or-v1-****} # Separate service endpoints for each LLM component -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} -export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8124/v1/chat/completions} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} + +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} echo "Configuration:" echo " DATASET_PATH: ${DATASET_PATH}" @@ -109,6 +112,8 @@ python3 reference_mlperf.py \ --llm_model ${LLM_MODEL} \ --query_service_url ${QUERY_SERVICE_URL} \ --query_model ${QUERY_MODEL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ ${PERF_CACHE_ARG} diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index 483d6590bc..9eb63ae6f2 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -55,16 +55,18 @@ export MAX_SUB_QUERIES=${MAX_SUB_QUERIES:-3} export TOP_K_RETRIEVER=${TOP_K_RETRIEVER:-10} # Model paths -export RETRIEVER_MODEL=${RETRIEVER_MODEL:-/data/model/e5-base-v2} -export RERANKER_MODEL=${RERANKER_MODEL:-/data/model/colbertv2.0} +export RETRIEVER_MODEL=${RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2} +export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} # LLM service configuration -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} -export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8124/v1/chat/completions} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} # Performance cache file (optional - for faster testing) export PERF_CACHE_FILE=${PERF_CACHE_FILE:-""} @@ -147,6 +149,8 @@ python3 reference_mlperf.py \ --llm_model ${LLM_MODEL} \ --query_service_url ${QUERY_SERVICE_URL} \ --query_model ${QUERY_MODEL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ ${PERF_CACHE_ARG} diff --git a/e2e-rag/scripts/run_ingestion.sh b/e2e-rag/scripts/run_ingestion.sh index 7b3348bbcd..f4c21845fc 100644 --- a/e2e-rag/scripts/run_ingestion.sh +++ b/e2e-rag/scripts/run_ingestion.sh @@ -27,7 +27,7 @@ INGESTION_EMBEDDING_DEVICE="${INGESTION_EMBEDDING_DEVICE:-${INGESTION_DEVICE}}" INGESTION_NUM_EMBEDDING_DEVICES="${INGESTION_NUM_EMBEDDING_DEVICES:-4}" INGESTION_CHUNK_LEN="${INGESTION_CHUNK_LEN:-768}" INGESTION_CHUNK_OVERLAP="${INGESTION_CHUNK_OVERLAP:-32}" -INGESTION_RETRIEVER_MODEL="${INGESTION_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INGESTION_RETRIEVER_MODEL="${INGESTION_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INGESTION_DOC_DIR="${INGESTION_DOC_DIR:-doc_html}" INGESTION_PASSAGES_JSON="${INGESTION_PASSAGES_JSON:-passages/doc_html_len${INGESTION_CHUNK_LEN}_ov${INGESTION_CHUNK_OVERLAP}_word.json}" INGESTION_DB="${INGESTION_DB:-vector_html_hnsw_len${INGESTION_CHUNK_LEN}_ov${INGESTION_CHUNK_OVERLAP}_word}" diff --git a/e2e-rag/scripts/run_multi_shot.sh b/e2e-rag/scripts/run_multi_shot.sh index 445bfd3bf9..105315e1db 100644 --- a/e2e-rag/scripts/run_multi_shot.sh +++ b/e2e-rag/scripts/run_multi_shot.sh @@ -14,7 +14,7 @@ # e.g.: INFERENCE_DEVICE=cpu bash scripts/run_multi_shot.sh 50 # # Prerequisites: -# - Local vLLM server running on port 8123 (default) +# - Local vLLM servers running: 20B on port 8192, 120B on port 8123 (default) # - OR set OPENROUTER_API_KEY environment variable to use OpenRouter # - scripts/run_ingestion.sh has been run (vector DB exists) # @@ -38,7 +38,7 @@ INFERENCE_DEVICE="${INFERENCE_DEVICE:-cpu}" INFERENCE_EMBEDDING_DEVICE="${INFERENCE_EMBEDDING_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_RERANKER_DEVICE="${INFERENCE_RERANKER_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INFERENCE_TOP_K_RETRIEVER="${INFERENCE_TOP_K_RETRIEVER:-15}" INFERENCE_MAX_ITERATIONS="${INFERENCE_MAX_ITERATIONS:-5}" INFERENCE_MAX_SUB_QUERIES="${INFERENCE_MAX_SUB_QUERIES:-3}" @@ -47,18 +47,18 @@ INFERENCE_REASONING="${INFERENCE_REASONING:-medium}" INFERENCE_MAX_RETRIES="${INFERENCE_MAX_RETRIES:-5}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" INFERENCE_NUM_WORKERS="${INFERENCE_NUM_WORKERS:-1}" -INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" -INFERENCE_QUERY_MODEL="${INFERENCE_QUERY_MODEL:-/model/gpt-oss-120b-mxfp4}" +INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-20b-mxfp4}" +INFERENCE_QUERY_MODEL="${INFERENCE_QUERY_MODEL:-gpt-oss-120b-mxfp4}" # Per-component endpoint splits. Empty -> inherit INFERENCE_LLM_URL / INFERENCE_MODEL. INFERENCE_GRADER_URL="${INFERENCE_GRADER_URL:-}" INFERENCE_GRADER_MODEL="${INFERENCE_GRADER_MODEL:-}" INFERENCE_QUERY_URL="${INFERENCE_QUERY_URL:-}" INFERENCE_SUFFICIENCY_URL="${INFERENCE_SUFFICIENCY_URL:-}" -INFERENCE_SUFFICIENCY_MODEL="${INFERENCE_SUFFICIENCY_MODEL:-}" -INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-gpt-oss-20b}" +INFERENCE_SUFFICIENCY_MODEL="${INFERENCE_SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4}" +INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-gpt-oss-20b-mxfp4}" INFERENCE_PERF_TEST_MODE="${INFERENCE_PERF_TEST_MODE:-}" # Positional args override config. diff --git a/e2e-rag/scripts/run_oracle.sh b/e2e-rag/scripts/run_oracle.sh index 7539a6ac15..8e57ab687a 100644 --- a/e2e-rag/scripts/run_oracle.sh +++ b/e2e-rag/scripts/run_oracle.sh @@ -26,7 +26,7 @@ else fi INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-120b-mxfp4}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" INFERENCE_ORACLE_BATCH_SIZE="${INFERENCE_ORACLE_BATCH_SIZE:-4}" INFERENCE_ORACLE_TIMEOUT="${INFERENCE_ORACLE_TIMEOUT:-2400}" diff --git a/e2e-rag/scripts/run_single_shot.sh b/e2e-rag/scripts/run_single_shot.sh index 148ce38b57..f0c4503f64 100644 --- a/e2e-rag/scripts/run_single_shot.sh +++ b/e2e-rag/scripts/run_single_shot.sh @@ -28,11 +28,11 @@ INFERENCE_DEVICE="${INFERENCE_DEVICE:-cpu}" INFERENCE_EMBEDDING_DEVICE="${INFERENCE_EMBEDDING_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_RERANKER_DEVICE="${INFERENCE_RERANKER_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INFERENCE_TOP_K_RETRIEVER="${INFERENCE_TOP_K_RETRIEVER:-15}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" -INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" +INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-20b-mxfp4}" INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-https://openrouter.ai/api/v1/chat/completions}" INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-openai/gpt-oss-20b}" diff --git a/e2e-rag/scripts/start_vllm_server.sh b/e2e-rag/scripts/start_vllm_server.sh index 6a7871a555..1227129e53 100644 --- a/e2e-rag/scripts/start_vllm_server.sh +++ b/e2e-rag/scripts/start_vllm_server.sh @@ -1,5 +1,5 @@ python3 -m vllm.entrypoints.openai.api_server \ - --model /model/gpt-oss-20b-mxfp4 \ + --model gpt-oss-20b-mxfp4 \ --dtype bfloat16 \ --enforce-eager \ --host 0.0.0.0 \ @@ -10,6 +10,6 @@ python3 -m vllm.entrypoints.openai.api_server \ --disable-log-requests \ --max-model-len=131072 \ --block-size 64 \ - --port 8123 \ + --port 8192 \ -tp 4 \ --async_scheduling \ No newline at end of file diff --git a/e2e-rag/scripts/write_db_manifest.sh b/e2e-rag/scripts/write_db_manifest.sh index 01b6580f8d..2b29767fa8 100644 --- a/e2e-rag/scripts/write_db_manifest.sh +++ b/e2e-rag/scripts/write_db_manifest.sh @@ -22,7 +22,7 @@ else fi INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" OUTPUT="${1:-db_manifest_$(hostname -s).json.gz}" From d19285607540855874c65afcbedc921959f3ee4c Mon Sep 17 00:00:00 2001 From: "Manasa(Intel)" Date: Mon, 13 Jul 2026 22:07:12 -0700 Subject: [PATCH 04/14] fixed the judge model back to meta-llama/Llama-3.1-8B-Instruct --- e2e-rag/reference_mlperf_accuracy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index f9c424e3b8..427a5ddb0c 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -58,8 +58,8 @@ export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/ export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} # Judge LLM configuration (for accuracy evaluation) -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} echo " LLM Service URL: ${LLM_SERVICE_URL}" echo " Judge Service URL: ${JUDGE_SERVICE_URL}" From 698dcff8cc6cf07da1c3f34c111102585f5d35ee Mon Sep 17 00:00:00 2001 From: Saehanseul Date: Fri, 10 Jul 2026 14:24:20 -0700 Subject: [PATCH 05/14] Fix reranker output silently ignored in multi-shot --- e2e-rag/evaluation.py | 20 ++--------------- e2e-rag/multi_shot_retrieval.py | 13 ++++------- e2e-rag/reranker_worker.py | 7 +++--- e2e-rag/retrieve/ragdb.py | 38 +++++++++------------------------ 4 files changed, 19 insertions(+), 59 deletions(-) diff --git a/e2e-rag/evaluation.py b/e2e-rag/evaluation.py index a6c59a889a..95e658efe6 100644 --- a/e2e-rag/evaluation.py +++ b/e2e-rag/evaluation.py @@ -144,27 +144,11 @@ def evaluate_retrieval_query(rag_db, query: str, expected_urls: List[str], print(f"Warning: No documents retrieved for query: {query[:50]}") else: reranking_start = time.perf_counter() - # Extract text content for reranking (rerank expects strings) - passages = [result.page_content for result in results] - scored_passages = rag_db.rerank(query, passages) - - # Reconstruct document objects with reranked order - # scored_passages is [(text, score), ...] ordered by score - reranked_results = [] - for text, score in scored_passages: - # Find the original document object for this text - for doc in results: - if doc.page_content == text: - reranked_results.append(doc) - break - - # Apply top_k_reranking limit AFTER reranking - # For adaptive strategies (top_p, relative, etc.), respect the number of documents - # selected by the strategy, only limit for fixed_k + # Rerank Documents by score; fixed_k takes top-k, adaptive keeps all. + reranked_results = rag_db.rerank_documents(query, results) if retrieval_strategy == "fixed_k": results = reranked_results[:top_k_reranking] else: - # For adaptive strategies, keep all documents selected by the strategy results = reranked_results reranking_time = time.perf_counter() - reranking_start diff --git a/e2e-rag/multi_shot_retrieval.py b/e2e-rag/multi_shot_retrieval.py index b557cda897..6a3b224937 100644 --- a/e2e-rag/multi_shot_retrieval.py +++ b/e2e-rag/multi_shot_retrieval.py @@ -1438,15 +1438,10 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], if verbose: print(f" Reranking {len(results)} docs for this subquery to top {target_docs_per_subquery}...") - # Extract contents for reranking - contents = [r.page_content for r in results] - scored_passages = rag_db.rerank(sub_query, contents) - - # Reorder results by reranking scores and take top-k - reranked_indices = [i for i, _ in sorted(enumerate(scored_passages), - key=lambda x: x[1][1], reverse=True)] - results = [results[idx] for idx in reranked_indices[:target_docs_per_subquery]] - + # Rerank Documents by score and take top-k. + results = rag_db.rerank_documents(sub_query, results, + top_k=target_docs_per_subquery) + if verbose: print(f" After reranking: keeping top {len(results)} docs") elif len(results) > target_docs_per_subquery: diff --git a/e2e-rag/reranker_worker.py b/e2e-rag/reranker_worker.py index e3acfea045..04ce552044 100644 --- a/e2e-rag/reranker_worker.py +++ b/e2e-rag/reranker_worker.py @@ -21,7 +21,7 @@ - Its own CPU affinity + memory binding (per-NUMA-node placement). - Isolation from the main process's GIL / thread pool. -Public API kept: RerankerQueue.submit(query, passages) -> [(passage, score), ...] +Public API kept: RerankerQueue.submit(query, passages) -> [(passage, score), ...] in input order. """ import ctypes @@ -126,9 +126,8 @@ def _do_rerank(model, tokenizer, device: str, query: str, passages: List[str]) - sim = sim.masked_fill(~d_mask.unsqueeze(1).bool(), float('-inf')) scores = sim.max(dim=-1).values.sum(dim=-1) - scored_passages = list(zip(passages, scores.float().tolist())) - scored_passages.sort(key=lambda x: x[1], reverse=True) - return scored_passages + # Return (passage, score) in input order; caller sorts/selects. + return list(zip(passages, scores.float().tolist())) class RerankerQueue: diff --git a/e2e-rag/retrieve/ragdb.py b/e2e-rag/retrieve/ragdb.py index 19f65c4416..33bd75900a 100644 --- a/e2e-rag/retrieve/ragdb.py +++ b/e2e-rag/retrieve/ragdb.py @@ -241,38 +241,20 @@ def shutdown_reranker(self): self._reranker_queue = None def rerank(self, query: str, passages: List[str]): - """Rerank passages via the reranker queue (ColBERT MaxSim).""" + """Score passages via the reranker; returns (passage, score) in input order.""" if self._reranker_queue: return self._reranker_queue.submit(query, passages) return [(p, 0.0) for p in passages] - def lookup_with_rerank(self, query: str, k: int, rerank_k: int = None) -> List[Any]: - """Retrieve and rerank passages.""" - if rerank_k is None: - rerank_k = k - - # Get initial results - results = self.lookup(query, k=rerank_k) - - # If no reranker or fewer results than requested, return as-is - if self._reranker_queue is None or len(results) <= k: - return results[:k] - - # Extract passages for reranking - passages = [result.page_content for result in results] - - # Rerank - reranked_passages = self.rerank(query, passages) - - # Map back to original results and return top-k - reranked_results = [] - for passage, score in reranked_passages[:k]: - for result in results: - if result.page_content == passage: - reranked_results.append(result) - break - - return reranked_results + def rerank_documents(self, query: str, documents: List[Any], top_k: int = None) -> List[Any]: + """Rerank Documents by score (desc) and return top-k; sole place sorting happens.""" + if not documents: + return [] + passages = [d.page_content for d in documents] + scored = self.rerank(query, passages) # input order: scored[i] <-> documents[i] + order = sorted(range(len(documents)), key=lambda i: scored[i][1], reverse=True) + ranked = [documents[i] for i in order] + return ranked[:top_k] if top_k is not None else ranked @property def device(self) -> str: From 2294be00f85d2670ea3724adddca1dc8c0f0b904 Mon Sep 17 00:00:00 2001 From: Saehanseul Date: Tue, 14 Jul 2026 12:25:06 -0700 Subject: [PATCH 06/14] Update the default judge to Llama3.1-8B --- e2e-rag/reference_mlperf_perf.sh | 4 ++-- e2e-rag/run_compliance_test09.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index 21e1459de3..ba5c7fdc3b 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -60,8 +60,8 @@ export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} echo "Configuration:" echo " DATASET_PATH: ${DATASET_PATH}" diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index 9eb63ae6f2..543e867c8c 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -65,8 +65,8 @@ export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/comp export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} -export JUDGE_MODEL=${JUDGE_MODEL:-gpt-oss-20b-mxfp4} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} # Performance cache file (optional - for faster testing) export PERF_CACHE_FILE=${PERF_CACHE_FILE:-""} From 1a2d6df7c921c8371bbe2007443f4686146d7712 Mon Sep 17 00:00:00 2001 From: Saehanseul Date: Tue, 14 Jul 2026 12:28:01 -0700 Subject: [PATCH 07/14] Update judge prompt for robustness --- e2e-rag/accuracy_eval.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 36f7e4138f..2ab0f72ea1 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -32,14 +32,14 @@ # OpenRouter configuration -DEFAULT_JUDGE_URL = "http://127.0.0.1:8192/v1/chat/completions" -DEFAULT_JUDGE_MODEL = "/data/gpt-oss-20b-mxfp4" +DEFAULT_JUDGE_URL = "http://127.0.0.1:8125/v1/chat/completions" +DEFAULT_JUDGE_MODEL = "meta-llama/Llama-3.1-8B-Instruct" # Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter) OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', 'sk-or-v1-****') -JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers. +JUDGE_PROMPT = """You are grading whether an LLM answer is correct against a ground truth answer. QUESTION: {question} @@ -47,8 +47,14 @@ LLM ANSWER: {llm_answer} -Evaluate if the LLM answer is factually correct compared to the ground truth. -Consider semantic equivalence, not just exact string matching. +Grade in two steps. + +STEP 1 - If the LLM answer is empty, "Unknown", "I don't know", "cannot be determined", or otherwise does not commit to an answer, then it is WRONG: output correct=false immediately and do not go to step 2. + +STEP 2 - Otherwise compare it to the ground truth by meaning, not wording. correct=true only if it supplies every fact the ground truth requires and each clearly matches; if you are unsure or the match is only partial, output correct=false. Rules: +- If the ground truth is a list or has multiple parts, an answer missing any of them is correct=false. +- Every number, date, and name must match the ground truth; a different or differently-rounded value is correct=false, a different name is correct=false. +- Do NOT penalize harmless extras or omissions when the required facts match: a missing suffix like "Inc.", an added state/country, a full middle name, missing units when the number is right, or a briefer/longer phrasing. Return your evaluation in JSON format: {{ From 4f88be9bd473558f56e2454d524941523de00c47 Mon Sep 17 00:00:00 2001 From: Saehanseul Date: Tue, 14 Jul 2026 12:37:26 -0700 Subject: [PATCH 08/14] Correct judge service port across the repo --- e2e-rag/reference_mlperf_perf.sh | 2 +- e2e-rag/run_compliance_test09.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index ba5c7fdc3b..f33dffa235 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -60,7 +60,7 @@ export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} echo "Configuration:" diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index 543e867c8c..b6f5d52019 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -65,7 +65,7 @@ export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/comp export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} # Performance cache file (optional - for faster testing) From da15dea0ce8677a6d182719f5e04e9e5c0c818fc Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 14:31:09 -0700 Subject: [PATCH 09/14] Rename e2e-rag workload components to e2e-rag-qna / e2e-rag-db Align naming end-to-end across the workload and submission tooling: - checker constants.py: e2e -> e2e-rag-qna, e2e_vectorDB -> e2e-rag-db - user.conf loadgen keys and FromConfig namespaces - reference-script output dirs (nested accuracy/performance to avoid the run_output/output collision between QnA accuracy and perf runs) - datasetup scripts: patch e2e-rag-db.Offline.* (was dead e2e-datasetup.*) - compliance/TEST09/e2e-rag -> e2e-rag-qna (+ README, run_compliance paths) - fix stale "E2E DocGrader" header comments Co-Authored-By: Claude Opus 4.8 --- .../TEST09/{e2e-rag => e2e-rag-qna}/README.md | 8 +++--- .../{e2e-rag => e2e-rag-qna}/audit.config | 0 e2e-rag/reference_mlperf.py | 2 +- e2e-rag/reference_mlperf_accuracy.sh | 6 ++-- e2e-rag/reference_mlperf_datasetup.py | 2 +- e2e-rag/reference_mlperf_datasetup.sh | 16 +++++------ .../reference_mlperf_datasetup_accuracy.sh | 16 +++++------ e2e-rag/reference_mlperf_perf.sh | 6 ++-- e2e-rag/run_compliance_test09.sh | 6 ++-- e2e-rag/user.conf | 16 +++++------ .../submission_checker/constants.py | 28 +++++++++---------- 11 files changed, 53 insertions(+), 53 deletions(-) rename compliance/TEST09/{e2e-rag => e2e-rag-qna}/README.md (94%) rename compliance/TEST09/{e2e-rag => e2e-rag-qna}/audit.config (100%) diff --git a/compliance/TEST09/e2e-rag/README.md b/compliance/TEST09/e2e-rag-qna/README.md similarity index 94% rename from compliance/TEST09/e2e-rag/README.md rename to compliance/TEST09/e2e-rag-qna/README.md index 90dcf04a39..e1d2fcb0b0 100644 --- a/compliance/TEST09/e2e-rag/README.md +++ b/compliance/TEST09/e2e-rag-qna/README.md @@ -1,4 +1,4 @@ -# TEST09 Compliance for E2E-RAG Workload +# TEST09 Compliance for E2E-RAG-QnA Workload ## Overview @@ -65,7 +65,7 @@ Copy the audit.config to your working directory: ```bash cd inference/e2e-rag -cp ../compliance/TEST09/e2e-rag/audit.config ./ +cp ../compliance/TEST09/e2e-rag-qna/audit.config ./ ``` ### Part II: Run Performance Test @@ -94,8 +94,8 @@ python3 reference_mlperf.py \ ```bash python3 inference/e2e/third_party/mlperf-inference/compliance/TEST09/run_verification.py \ -c run_output_test09 \ - -o submission/compliance/e2e-rag/Offline \ - --audit-config ../compliance/TEST09/e2e-rag/audit.config + -o submission/compliance/e2e-rag-qna/Offline \ + --audit-config ../compliance/TEST09/e2e-rag-qna/audit.config ``` ### Expected Output diff --git a/compliance/TEST09/e2e-rag/audit.config b/compliance/TEST09/e2e-rag-qna/audit.config similarity index 100% rename from compliance/TEST09/e2e-rag/audit.config rename to compliance/TEST09/e2e-rag-qna/audit.config diff --git a/e2e-rag/reference_mlperf.py b/e2e-rag/reference_mlperf.py index 83ebc0fcd0..6c98f12d29 100644 --- a/e2e-rag/reference_mlperf.py +++ b/e2e-rag/reference_mlperf.py @@ -204,7 +204,7 @@ def main(): # Load config files if os.path.exists(args.user_conf): - settings.FromConfig(args.user_conf, "rag-qna", args.scenario) + settings.FromConfig(args.user_conf, "e2e-rag-qna", args.scenario) print(f"Loaded user config from {args.user_conf}") else: print(f"Warning: User config not found: {args.user_conf}") diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index 427a5ddb0c..43abc0fb86 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# Accuracy test script for E2E DocGrader workload with MLPerf Loadgen +# Accuracy test script for E2E-RAG-QnA workload with MLPerf Loadgen echo "Time Start: $(date +%s)" @@ -23,8 +23,8 @@ export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DATA_DIR=${DATA_DIR:-"frames-benchmark-dataset"} export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output -export OUTPUT_DIR=${WORKSPACE_DIR}/output +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-qna/accuracy +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-qna/accuracy export SCENARIO="${SCENARIO:-Offline}" # Threading configuration diff --git a/e2e-rag/reference_mlperf_datasetup.py b/e2e-rag/reference_mlperf_datasetup.py index 5ec27d1232..ce2feeac4b 100644 --- a/e2e-rag/reference_mlperf_datasetup.py +++ b/e2e-rag/reference_mlperf_datasetup.py @@ -210,7 +210,7 @@ def main(): # Load config files if os.path.exists(args.user_conf): - settings.FromConfig(args.user_conf, "rag-db", args.scenario) + settings.FromConfig(args.user_conf, "e2e-rag-db", args.scenario) print(f"Loaded user config from {args.user_conf}") else: print(f"Warning: User config not found: {args.user_conf}") diff --git a/e2e-rag/reference_mlperf_datasetup.sh b/e2e-rag/reference_mlperf_datasetup.sh index fba93c0a24..df82def111 100755 --- a/e2e-rag/reference_mlperf_datasetup.sh +++ b/e2e-rag/reference_mlperf_datasetup.sh @@ -22,8 +22,8 @@ echo "Time Start: $(date +%s)" export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DOCUMENTS_DIR=${DOCUMENTS_DIR:-"doc_html"} export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output_datasetup -export OUTPUT_DIR=${WORKSPACE_DIR}/output_datasetup +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-db/performance +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-db/performance export SCENARIO="${SCENARIO:-Offline}" # Chunking configuration @@ -81,16 +81,16 @@ fi if [ -f "user.conf" ]; then # Update max_async_queries to match HTML count (send all at once) # Update min_query_count to match HTML count - if grep -q "e2e-datasetup.Offline.max_async_queries" user.conf; then - sed -i "s/^e2e-datasetup.Offline.max_async_queries = .*/e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.max_async_queries" user.conf; then + sed -i "s/^e2e-rag-db.Offline.max_async_queries = .*/e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf fi - if grep -q "e2e-datasetup.Offline.min_query_count" user.conf; then - sed -i "s/^e2e-datasetup.Offline.min_query_count = .*/e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.min_query_count" user.conf; then + sed -i "s/^e2e-rag-db.Offline.min_query_count = .*/e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}" >> user.conf fi echo " Loadgen configured to dispatch all ${HTML_COUNT} files at once" diff --git a/e2e-rag/reference_mlperf_datasetup_accuracy.sh b/e2e-rag/reference_mlperf_datasetup_accuracy.sh index 1a6faaa598..468d47bdae 100755 --- a/e2e-rag/reference_mlperf_datasetup_accuracy.sh +++ b/e2e-rag/reference_mlperf_datasetup_accuracy.sh @@ -22,8 +22,8 @@ echo "Time Start: $(date +%s)" export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DOCUMENTS_DIR=${DOCUMENTS_DIR:-"doc_html"} export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output_datasetup_accuracy -export OUTPUT_DIR=${WORKSPACE_DIR}/output_datasetup_accuracy +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-db/accuracy +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-db/accuracy export SCENARIO="${SCENARIO:-Offline}" # Chunking configuration @@ -81,16 +81,16 @@ fi if [ -f "user.conf" ]; then # Update max_async_queries to match HTML count (send all at once) # Update min_query_count to match HTML count - if grep -q "e2e-datasetup.Offline.max_async_queries" user.conf; then - sed -i "s/^e2e-datasetup.Offline.max_async_queries = .*/e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.max_async_queries" user.conf; then + sed -i "s/^e2e-rag-db.Offline.max_async_queries = .*/e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf fi - if grep -q "e2e-datasetup.Offline.min_query_count" user.conf; then - sed -i "s/^e2e-datasetup.Offline.min_query_count = .*/e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.min_query_count" user.conf; then + sed -i "s/^e2e-rag-db.Offline.min_query_count = .*/e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}" >> user.conf fi echo " Loadgen configured to dispatch all ${HTML_COUNT} files at once (accuracy mode)" diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index f33dffa235..47dbb28c0d 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# Performance test script for E2E DocGrader workload with MLPerf Loadgen +# Performance test script for E2E-RAG-QnA workload with MLPerf Loadgen echo "Time Start: $(date +%s)" @@ -23,8 +23,8 @@ export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DATA_DIR=${DATA_DIR:-"frames-benchmark-dataset"} export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output -export OUTPUT_DIR=${WORKSPACE_DIR}/output +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-qna/performance +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-qna/performance export SCENARIO="${SCENARIO:-Offline}" # Threading configuration diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index b6f5d52019..e6ba9120b8 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# TEST09 Compliance Test Runner for E2E DocGrader Workload +# TEST09 Compliance Test Runner for E2E-RAG-QnA Workload # Automates: setup -> run -> verify -> cleanup workflow set -e # Exit on error @@ -27,7 +27,7 @@ echo "" # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -COMPLIANCE_DIR="${SCRIPT_DIR}/../compliance/TEST09/e2e-rag" +COMPLIANCE_DIR="${SCRIPT_DIR}/../compliance/TEST09/e2e-rag-qna" AUDIT_CONFIG="${COMPLIANCE_DIR}/audit.config" WORKING_AUDIT_CONFIG="${SCRIPT_DIR}/audit.config" TEST09_VERIFICATION="${SCRIPT_DIR}/third_party/mlperf-inference/compliance/TEST09/run_verification.py" @@ -39,7 +39,7 @@ export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" export RUN_LOGS=${WORKSPACE_DIR}/run_output_test09 export OUTPUT_DIR=${WORKSPACE_DIR}/output_test09 -export SUBMISSION_DIR=${WORKSPACE_DIR}/submission/compliance/e2e-rag/Offline +export SUBMISSION_DIR=${WORKSPACE_DIR}/submission/compliance/e2e-rag-qna/Offline export SCENARIO="${SCENARIO:-Offline}" # Performance testing - full dataset for compliance diff --git a/e2e-rag/user.conf b/e2e-rag/user.conf index c215e19243..960c0fed7c 100644 --- a/e2e-rag/user.conf +++ b/e2e-rag/user.conf @@ -11,16 +11,16 @@ # min_query_count takes priority - loadgen will run AT LEAST this many unique queries # min_duration is secondary - only matters if fewer queries would complete faster # max_async_queries controls concurrent query processing (default: 1) -rag-qna.Offline.target_qps = 0.11 -rag-qna.Offline.min_duration = 0 -rag-qna.Offline.min_query_count = 824 -rag-qna.Offline.max_async_queries = 10 +e2e-rag-qna.Offline.target_qps = 0.11 +e2e-rag-qna.Offline.min_duration = 0 +e2e-rag-qna.Offline.min_query_count = 824 +e2e-rag-qna.Offline.max_async_queries = 10 # RAG DB workload settings # Send all documents at once for parallel processing -rag-db.Offline.target_qps = 0 -rag-db.Offline.min_duration = 0 -rag-db.Offline.min_query_count = 2503 -rag-db.Offline.max_async_queries = 2503 +e2e-rag-db.Offline.target_qps = 0 +e2e-rag-db.Offline.min_duration = 0 +e2e-rag-db.Offline.min_query_count = 2503 +e2e-rag-db.Offline.max_async_queries = 2503 e2e-datasetup.Offline.max_async_queries = 2515 e2e-datasetup.Offline.min_query_count = 2515 diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index b4000a1d86..5ecd79c589 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -20,8 +20,8 @@ "dlrm-v3", "yolo-95", "yolo-99", - "e2e", - "e2e_vectorDB" + "e2e-rag-qna", + "e2e-rag-db" ], "required-scenarios-datacenter": { "dlrm-v3": ["Server", "Offline"], @@ -36,8 +36,8 @@ "gpt-oss-120b": ["Offline"], "qwen3-vl-235b-a22b": ["Server", "Offline"], "wan-2.2-t2v-a14b": ["Offline", "SingleStream"], - "e2e": ["Offline"], - "e2e_vectorDB": ["Offline"] + "e2e-rag-qna": ["Offline"], + "e2e-rag-db": ["Offline"] }, "optional-scenarios-datacenter": { "llama2-70b-99": ["Interactive", "Server"], @@ -162,8 +162,8 @@ "wan-2.2-t2v-a14b": ("vbench_score", 70.48 * 0.99), # TODO: Set e2e accuracy threshold once reference score is # established - "e2e": ("E2E_ACCURACY", ""), - "e2e_vectorDB": ("E2E_ACCURACY", ""), + "e2e-rag-qna": ("E2E_ACCURACY", ""), + "e2e-rag-db": ("E2E_ACCURACY", ""), }, "accuracy-upper-limit": { "stable-diffusion-xl": ( @@ -202,8 +202,8 @@ "dlrm-v3": 349823, "yolo-95": 64, "yolo-99": 64, - "e2e": 824, - "e2e_vectorDB": 824 + "e2e-rag-qna": 824, + "e2e-rag-db": 824 }, "accuracy-sample-count": { "gpt-oss-120b": 4395, @@ -229,8 +229,8 @@ "dlrm-v3": 349823, "yolo-95": 1525, "yolo-99": 1525, - "e2e": 824, - "e2e_vectorDB": 824, + "e2e-rag-qna": 824, + "e2e-rag-db": 824, }, "model_mapping": { "ssd-resnet34": "retinanet", @@ -289,8 +289,8 @@ "wan-2.2-t2v-a14b": {"SingleStream": 50, "Offline": 1}, "yolo-95": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, "yolo-99": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, - "e2e": {"Offline": 824}, - "e2e_vectorDB": {"Offline": 824}, + "e2e-rag-qna": {"Offline": 824}, + "e2e-rag-db": {"Offline": 824}, }, "models_TEST01": [ "resnet", @@ -1530,8 +1530,8 @@ "dlrm-v3": 349823, "qwen3-vl-235b-a22b": 48289, "wan-2.2-t2v-a14b": 50, - "e2e": 824, - "e2e_vectorDB": 824 + "e2e-rag-qna": 824, + "e2e-rag-db": 824 } SCENARIO_MAPPING = { From de43dfc78b69781d752f30e67210a02d6f9fa3d6 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 15:23:20 -0700 Subject: [PATCH 10/14] Add DB manifest verification to RAG-DB accuracy test The RAG-DB accuracy test only did local self-consistency checks and never verified the built vector DB against the reference manifest, so a divergent corpus/embedding could still pass. - db_manifest.py: extract cmd_verify body into reusable verify_manifest() that returns {passed, failures, metrics} instead of sys.exit; add a --retriever_model override (manifest's stored path is a system-specific absolute path that breaks verify on other systems) with fallback to the manifest value. - datasetup_accuracy_eval.py: run verify_manifest() when --manifest is given; record results in accuracy_results["manifest"], gate overall pass on it, and write a manifest section into accuracy.txt. Skipped cleanly when no manifest is provided. - reference_mlperf_datasetup_accuracy.sh: pass the bundled reference manifest (scripts/db_manifest_intel_xpu.json.gz) by default. - scripts/verify_db_manifest.sh: pass local retriever model to verify. Co-Authored-By: Claude Opus 4.8 --- e2e-rag/datasetup_accuracy_eval.py | 100 +++++++++++++++++- e2e-rag/db_manifest.py | 75 ++++++++++--- .../reference_mlperf_datasetup_accuracy.sh | 15 ++- e2e-rag/scripts/verify_db_manifest.sh | 3 + 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/e2e-rag/datasetup_accuracy_eval.py b/e2e-rag/datasetup_accuracy_eval.py index 735170f563..84f3f806b9 100755 --- a/e2e-rag/datasetup_accuracy_eval.py +++ b/e2e-rag/datasetup_accuracy_eval.py @@ -268,7 +268,9 @@ def validate_database(database_path, retriever_model): return validation_results -def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): +def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None, + manifest_path=None, cosine_threshold=0.9999, + top_k_depth=3): """ Evaluate accuracy of datasetup workload. @@ -277,6 +279,11 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): output_dir: Directory containing SUT output files database_path: Path to the saved database file retriever_model: Path to retriever model (for validation) + manifest_path: Path to reference DB manifest for cross-system + verification. If None, the manifest check is skipped. + cosine_threshold: Minimum sample-embedding cosine similarity for the + manifest check. + top_k_depth: Probe-query top-K rank match depth for the manifest check. Returns: dict: Accuracy results @@ -448,6 +455,59 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): print("="*80) print() + # Cross-system manifest verification (corpus fingerprint, sample-embedding + # cosine, probe-query top-K ranks) against a reference manifest. + manifest_results = None + if manifest_path: + print("="*80) + print("DB Manifest Verification") + print("="*80) + print(f"Manifest: {manifest_path}") + print() + + if not os.path.exists(manifest_path): + manifest_results = { + "passed": False, + "error": "manifest_not_found", + "manifest_path": manifest_path, + } + print(f" ✗ Manifest not found: {manifest_path}") + elif not os.path.exists(database_path): + manifest_results = { + "passed": False, + "error": "database_not_found", + "database_path": database_path, + } + print(f" ✗ Database not found: {database_path}") + else: + try: + from db_manifest import verify_manifest + manifest_results = verify_manifest( + database_path, + manifest_path, + retriever_model=retriever_model, + cosine_threshold=cosine_threshold, + top_k_depth=top_k_depth, + ) + if manifest_results["passed"]: + print(" ✓ Manifest verification PASSED") + else: + print(" ✗ Manifest verification FAILED:") + for failure in manifest_results["failures"]: + print(f" - {failure}") + except Exception as e: + manifest_results = {"passed": False, "error": str(e)} + print(f" ✗ Manifest verification error: {e}") + + accuracy_results["manifest"] = manifest_results + # Overall pass now also requires the manifest check to pass. + accuracy_results["passed"] = accuracy_results["passed"] and manifest_results["passed"] + + print() + print(f"Manifest: {'✅ PASSED' if manifest_results['passed'] else '❌ FAILED'}") + print("="*80) + print() + # Write accuracy.txt in MLPerf format accuracy_txt_path = os.path.join(log_dir, "accuracy.txt") with open(accuracy_txt_path, 'w') as f: @@ -488,6 +548,19 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): f.write(f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n") f.write("\n") + if manifest_results: + f.write("DB Manifest Verification:\n") + f.write("-"*80 + "\n") + if manifest_results.get("error"): + f.write(f" Error: {manifest_results['error']}\n") + metrics = manifest_results.get("metrics", {}) + for key, value in metrics.items(): + f.write(f" {key}: {value}\n") + for failure in manifest_results.get("failures", []): + f.write(f" MISMATCH: {failure}\n") + f.write(f"Manifest status: {'PASS' if manifest_results['passed'] else 'FAIL'}\n") + f.write("\n") + f.write("="*80 + "\n") f.write(f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n") f.write("="*80 + "\n") @@ -523,10 +596,33 @@ def main(): default="intfloat_e5-base-v2/e5-base-v2", help="Path to retriever model (for validation)" ) + parser.add_argument( + "--manifest", + default=None, + help="Path to reference DB manifest (.json/.json.gz) for cross-system " + "verification. If omitted, the manifest check is skipped." + ) + parser.add_argument( + "--cosine_threshold", + type=float, + default=0.9999, + help="Minimum sample-embedding cosine similarity for the manifest check" + ) + parser.add_argument( + "--top_k_depth", + type=int, + default=3, + help="Probe-query top-K rank match depth for the manifest check" + ) args = parser.parse_args() - results = evaluate_accuracy(args.log_dir, args.output_dir, args.database, args.retriever_model) + results = evaluate_accuracy( + args.log_dir, args.output_dir, args.database, args.retriever_model, + manifest_path=args.manifest, + cosine_threshold=args.cosine_threshold, + top_k_depth=args.top_k_depth, + ) # Exit with appropriate code if results.get("passed", False): diff --git a/e2e-rag/db_manifest.py b/e2e-rag/db_manifest.py index 6ecf87f957..7f3545edc9 100644 --- a/e2e-rag/db_manifest.py +++ b/e2e-rag/db_manifest.py @@ -164,14 +164,42 @@ def cmd_write(args): print(f"[manifest] wrote {args.output}") -def cmd_verify(args): - with _open_manifest(args.manifest, "rt") as f: +def verify_manifest(db_path: str, manifest_path: str, + retriever_model: str = None, + cosine_threshold: float = DEFAULT_COSINE_THRESHOLD, + top_k_depth: int = DEFAULT_TOP_K_DEPTH) -> Dict: + """Verify a vector DB against a reference manifest. + + Args: + db_path: Path to the local vector DB to check. + manifest_path: Path to the reference manifest (.json or .json.gz). + retriever_model: Retriever model to load the DB with. If None, falls + back to the manifest's stored ``retriever_model``. The manifest + value is often a system-specific absolute path, so callers on other + systems should pass their own local model path here. + cosine_threshold: Minimum sample-embedding cosine similarity. + top_k_depth: Probe-query top-K rank match depth. + + Returns: + dict with keys ``passed`` (bool), ``failures`` (list[str]), and + ``metrics`` (dict of observed values). Never raises on mismatch; the + CLI wrapper is responsible for translating a failure into an exit code. + """ + with _open_manifest(manifest_path, "rt") as f: manifest = json.load(f) - db = _load_db(args.db, manifest["retriever_model"]) + # Prefer an explicit retriever model; the manifest's value may be an + # absolute path that only exists on the system that wrote it. + model = retriever_model or manifest["retriever_model"] + db = _load_db(db_path, model) total_passages = len(db._vector_store.index_to_docstore_id) failures = [] + metrics = { + "total_passages": total_passages, + "embedding_dim": db._embedding_dimension, + "retriever_model": model, + } # Exact-match fields. if total_passages != manifest["total_passages"]: @@ -186,6 +214,7 @@ def cmd_verify(args): # Corpus fingerprint (sha256 of all passage texts in index order). local_corpus_sha = _sha256_docstore(db) + metrics["corpus_sha256_match"] = (local_corpus_sha == manifest["corpus_sha256"]) if local_corpus_sha != manifest["corpus_sha256"]: failures.append( f"corpus sha256 mismatch:\n" @@ -208,12 +237,14 @@ def cmd_verify(args): if cosines: worst_idx, worst_cos = min(cosines, key=lambda x: x[1]) mean_cos = sum(c for _, c in cosines) / len(cosines) + metrics["sample_cosine_mean"] = mean_cos + metrics["sample_cosine_min"] = worst_cos print(f"[verify] sample embeddings: mean cosine={mean_cos:.6f} " - f"min={worst_cos:.6f} (idx={worst_idx}) threshold={args.cosine_threshold}") - if worst_cos < args.cosine_threshold: + f"min={worst_cos:.6f} (idx={worst_idx}) threshold={cosine_threshold}") + if worst_cos < cosine_threshold: failures.append( f"sample embedding cosine below threshold: " - f"min={worst_cos:.6f} (idx={worst_idx}) < threshold={args.cosine_threshold}\n" + f"min={worst_cos:.6f} (idx={worst_idx}) < threshold={cosine_threshold}\n" f" mean={mean_cos:.6f}" ) @@ -224,24 +255,37 @@ def cmd_verify(args): rank_failures = [] for entry in local_top: - local_urls = entry["top_k_urls"][:args.top_k_depth] - ref_urls = ref_top.get(entry["index"], [])[:args.top_k_depth] + local_urls = entry["top_k_urls"][:top_k_depth] + ref_urls = ref_top.get(entry["index"], [])[:top_k_depth] if local_urls != ref_urls: rank_failures.append( - f" query idx {entry['index']}: top-{args.top_k_depth} differs\n" + f" query idx {entry['index']}: top-{top_k_depth} differs\n" f" local : {local_urls}\n" f" ref : {ref_urls}" ) + metrics["probe_queries_total"] = len(probe_queries) + metrics["probe_queries_matched"] = len(probe_queries) - len(rank_failures) print(f"[verify] probe queries: {len(probe_queries)} queries, " - f"top-{args.top_k_depth} {len(probe_queries) - len(rank_failures)}/" + f"top-{top_k_depth} {len(probe_queries) - len(rank_failures)}/" f"{len(probe_queries)} match") if rank_failures: failures.append("probe-query top-K rank mismatch:\n" + "\n".join(rank_failures)) - if failures: + return {"passed": not failures, "failures": failures, "metrics": metrics} + + +def cmd_verify(args): + result = verify_manifest( + args.db, + args.manifest, + retriever_model=args.retriever_model, + cosine_threshold=args.cosine_threshold, + top_k_depth=args.top_k_depth, + ) + if not result["passed"]: print("\n[verify] FAILED:") - for f in failures: + for f in result["failures"]: print(f" - {f}") sys.exit(1) print("\n[verify] OK") @@ -262,6 +306,13 @@ def main(): pv = sub.add_parser("verify", help="Verify a DB against a reference manifest.") pv.add_argument("--db", required=True) pv.add_argument("--manifest", required=True) + pv.add_argument( + "--retriever_model", + default=None, + help="Retriever model to load the DB with. Defaults to the manifest's " + "stored value, which may be a system-specific absolute path; pass " + "your local model path to verify on a different system.", + ) pv.add_argument("--cosine-threshold", type=float, default=DEFAULT_COSINE_THRESHOLD) pv.add_argument("--top-k-depth", type=int, default=DEFAULT_TOP_K_DEPTH) pv.set_defaults(func=cmd_verify) diff --git a/e2e-rag/reference_mlperf_datasetup_accuracy.sh b/e2e-rag/reference_mlperf_datasetup_accuracy.sh index 468d47bdae..e68e7c2038 100755 --- a/e2e-rag/reference_mlperf_datasetup_accuracy.sh +++ b/e2e-rag/reference_mlperf_datasetup_accuracy.sh @@ -42,6 +42,12 @@ export NUM_EMBEDDING_DEVICES=${NUM_EMBEDDING_DEVICES:-1} # Vector database configuration export VECTOR_INDEX_METHOD=${VECTOR_INDEX_METHOD:-"hnsw"} +# Reference DB manifest for cross-system verification (corpus fingerprint, +# sample-embedding cosine, probe-query top-K ranks). Set to "" to skip. +export MANIFEST=${MANIFEST:-scripts/db_manifest_intel_xpu.json.gz} +export COSINE_THRESHOLD=${COSINE_THRESHOLD:-0.9999} +export TOP_K_DEPTH=${TOP_K_DEPTH:-3} + # Performance options export BENCHMARK=${BENCHMARK:-false} export MAX_WORKERS=${MAX_WORKERS:-4} @@ -133,11 +139,18 @@ if [ ${EXIT_CODE} -eq 0 ]; then echo "Running Accuracy Evaluation" echo "============================================================" + # Build optional manifest argument (skip check if MANIFEST is empty) + MANIFEST_ARG="" + if [ -n "${MANIFEST}" ]; then + MANIFEST_ARG="--manifest ${MANIFEST} --cosine_threshold ${COSINE_THRESHOLD} --top_k_depth ${TOP_K_DEPTH}" + fi + python3 datasetup_accuracy_eval.py \ --log_dir ${RUN_LOGS} \ --output_dir ${OUTPUT_DIR} \ --database ${DATABASE}.db \ - --retriever_model ${RETRIEVER_MODEL} + --retriever_model ${RETRIEVER_MODEL} \ + ${MANIFEST_ARG} EVAL_EXIT_CODE=$? diff --git a/e2e-rag/scripts/verify_db_manifest.sh b/e2e-rag/scripts/verify_db_manifest.sh index 495f61478a..1a825fc386 100644 --- a/e2e-rag/scripts/verify_db_manifest.sh +++ b/e2e-rag/scripts/verify_db_manifest.sh @@ -29,6 +29,7 @@ else fi INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" MANIFEST="$1" COSINE_THRESHOLD="${2:-0.9999}" @@ -36,6 +37,7 @@ TOP_K_DEPTH="${3:-3}" echo "=== Verifying DB against manifest ===" echo " DB: ${INFERENCE_DB}" +echo " Retriever: ${INFERENCE_RETRIEVER_MODEL}" echo " Manifest: ${MANIFEST}" echo " Cosine threshold: ${COSINE_THRESHOLD}" echo " Top-K depth: ${TOP_K_DEPTH}" @@ -44,5 +46,6 @@ echo "" python3 -u db_manifest.py verify \ --db "${INFERENCE_DB}" \ --manifest "${MANIFEST}" \ + --retriever_model "${INFERENCE_RETRIEVER_MODEL}" \ --cosine-threshold "${COSINE_THRESHOLD}" \ --top-k-depth "${TOP_K_DEPTH}" From 665d16503a4691ebc741bd823eed476203fdc3aa Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 15:31:26 -0700 Subject: [PATCH 11/14] RAG-QnA: write LLM judge score to accuracy.txt The RAG-QnA accuracy run produced only accuracy_results.json; the submission checker needs an accuracy.txt with an "Accuracy:" line. - accuracy_eval.py: write accuracy.txt into the loadgen log dir with the LLM judge answer accuracy as a percentage (answer_accuracy * 100), matching the checker's E2E_ACCURACY pattern. hash= line and truncation are added later by truncate_accuracy_log.py during submission prep. - reference_mlperf_{accuracy,perf}.sh: fix stale user.conf sed patch that still targeted the old e2e.Offline key (now e2e-rag-qna.Offline). Co-Authored-By: Claude Opus 4.8 --- e2e-rag/accuracy_eval.py | 9 +++++++++ e2e-rag/reference_mlperf_accuracy.sh | 2 +- e2e-rag/reference_mlperf_perf.sh | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 2ab0f72ea1..cec921683c 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -311,6 +311,15 @@ def main(): json.dump(metrics, f, indent=2) print(f"Detailed results saved to {args.output}") + # Write accuracy.txt into the loadgen log dir in MLPerf format. The + # submission checker parses the LLM judge answer accuracy (as a percentage) + # from the "Accuracy:" line. The hash= line and log truncation are added + # later by tools/submission/truncate_accuracy_log.py during submission prep. + accuracy_txt_path = os.path.join(args.log_dir, "accuracy.txt") + with open(accuracy_txt_path, 'w') as f: + f.write(f"Accuracy: {metrics['answer_accuracy'] * 100:.4f}\n") + print(f"Accuracy report saved to {accuracy_txt_path}") + if __name__ == "__main__": main() diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index 43abc0fb86..dc1e8d6fac 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -84,7 +84,7 @@ if [ -n "${PERF_COUNT}" ]; then fi # Update user.conf with threading configuration -sed -i "s/^e2e.Offline.max_async_queries = .*/e2e.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf +sed -i "s/^e2e-rag-qna.Offline.max_async_queries = .*/e2e-rag-qna.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf # Run loadgen accuracy test python3 reference_mlperf.py \ diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index 47dbb28c0d..96255226c1 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -83,7 +83,7 @@ echo " JUDGE_SERVICE_URL: ${JUDGE_SERVICE_URL}" echo " JUDGE_MODEL: ${JUDGE_MODEL}" # Update user.conf with threading configuration -sed -i "s/^e2e.Offline.max_async_queries = .*/e2e.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf +sed -i "s/^e2e-rag-qna.Offline.max_async_queries = .*/e2e-rag-qna.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf # Build perf cache argument if file exists PERF_CACHE_ARG="" From 4a4ded83446224a5e10be3378f27a08e0d9e1583 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 15:46:02 -0700 Subject: [PATCH 12/14] Enable TEST09 compliance check for e2e-rag-qna The submission checker gates compliance validation on the models_TESTxx lists; e2e-rag-qna was in none, so TEST09 results were silently unchecked despite the workload shipping run_compliance_test09.sh and producing verify_output_len.txt. Add e2e-rag-qna to models_TEST09 (v6.1) so the checker requires TEST09/verify_output_len.txt and validates it contains "TEST PASS". e2e-rag-db has no compliance test and is left out. Co-Authored-By: Claude Opus 4.8 --- tools/submission/submission_checker/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 5ecd79c589..aefe635a4d 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -323,6 +323,7 @@ ], "models_TEST09": [ "gpt-oss-120b", + "e2e-rag-qna", ], "models_TEST08": [ "dlrm-v3", From a93957243b6e30e66ac8a10aa749d176a1df92b0 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 22:07:06 -0700 Subject: [PATCH 13/14] e2e-rag: robustness fixes for submission-checker compatibility Workload-side fixes validated end-to-end via a 10-doc/10-query smoke run through OpenRouter in a container: - run_compliance_test09.sh: pass --audit_conf to reference_mlperf.py so loadgen actually applies the TEST09 audit.config (previously the default audit.conf was used, leaving accuracy_log_sampling_target=0 and an empty mlperf_log_accuracy.json -> verification found no entries). Make PERF_COUNT overridable and sync the audit.config min_query_count to it. Source the TEST09 config from the main repo's compliance tree (copied in / overridable via COMPLIANCE_DIR). - reference_mlperf_{perf,accuracy}.sh: capture and propagate the python exit code (was masked by a trailing echo, so failures looked like success). - datasetup_accuracy_eval.py: write a checker-compliant "Accuracy:" line (indexing success rate) and always emit the DB Manifest Verification section (PASS/FAIL/SKIPPED) in accuracy.txt. - reference_mlperf_datasetup_accuracy.sh: honor an explicit empty/"none" MANIFEST to skip the manifest check (use ${VAR-default} so empty is respected). - user.conf: e2e-rag-db counts 2503->2515; drop dead e2e-datasetup.* keys. - .gitignore: ignore run_output_*/output/submission smoke artifacts. Co-Authored-By: Claude Opus 4.8 --- e2e-rag/.gitignore | 10 ++++++++ e2e-rag/datasetup_accuracy_eval.py | 19 +++++++++++---- e2e-rag/reference_mlperf_accuracy.sh | 3 +++ .../reference_mlperf_datasetup_accuracy.sh | 11 +++++---- e2e-rag/reference_mlperf_perf.sh | 3 +++ e2e-rag/run_compliance_test09.sh | 23 +++++++++++++++---- e2e-rag/user.conf | 6 ++--- 7 files changed, 57 insertions(+), 18 deletions(-) diff --git a/e2e-rag/.gitignore b/e2e-rag/.gitignore index 817a93c1f3..b52c4e4a5e 100644 --- a/e2e-rag/.gitignore +++ b/e2e-rag/.gitignore @@ -37,6 +37,16 @@ result_*.json temp_complete_kpi_*.json run_output_datasetup_accuracy/ run_output_datasetup/ +run_output_*/ +run_output_test09/ +accuracy_results.json +/smoke_manifest.json +/db_manifest_*.json +/db_manifest_*.json.gz +doc_html_smoke/ +/submission/ +/audit.config +/verify_output_len.txt colbert-ir_colbertv2.0/ frames-benchmark-dataset/ intfloat_e5-base-v2/ diff --git a/e2e-rag/datasetup_accuracy_eval.py b/e2e-rag/datasetup_accuracy_eval.py index 84f3f806b9..d92dfa7972 100755 --- a/e2e-rag/datasetup_accuracy_eval.py +++ b/e2e-rag/datasetup_accuracy_eval.py @@ -511,6 +511,11 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None, # Write accuracy.txt in MLPerf format accuracy_txt_path = os.path.join(log_dir, "accuracy.txt") with open(accuracy_txt_path, 'w') as f: + # Checker-compliant metric line first: the submission checker parses the + # RAG-DB accuracy (document indexing success rate as a percentage) from + # this "Accuracy:" line via the E2E_ACCURACY pattern. The hash= line and + # log truncation are added later by truncate_accuracy_log.py. + f.write(f"Accuracy: {actual_success_rate * 100:.4f}\n") f.write("="*80 + "\n") f.write("RAG-DB Accuracy Report\n") f.write("="*80 + "\n") @@ -548,9 +553,13 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None, f.write(f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n") f.write("\n") - if manifest_results: - f.write("DB Manifest Verification:\n") - f.write("-"*80 + "\n") + # Always emit the manifest section so its status (or that it was + # skipped) is visible in every accuracy report. + f.write("DB Manifest Verification:\n") + f.write("-"*80 + "\n") + if manifest_results is None: + f.write(" Manifest status: SKIPPED (no manifest provided)\n") + else: if manifest_results.get("error"): f.write(f" Error: {manifest_results['error']}\n") metrics = manifest_results.get("metrics", {}) @@ -558,8 +567,8 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None, f.write(f" {key}: {value}\n") for failure in manifest_results.get("failures", []): f.write(f" MISMATCH: {failure}\n") - f.write(f"Manifest status: {'PASS' if manifest_results['passed'] else 'FAIL'}\n") - f.write("\n") + f.write(f" Manifest status: {'PASS' if manifest_results['passed'] else 'FAIL'}\n") + f.write("\n") f.write("="*80 + "\n") f.write(f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n") diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index dc1e8d6fac..542a222cfb 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -110,4 +110,7 @@ python3 reference_mlperf.py \ --judge_model ${JUDGE_MODEL} \ --accuracy +EXIT_CODE=$? + echo "Time Stop: $(date +%s)" +exit ${EXIT_CODE} diff --git a/e2e-rag/reference_mlperf_datasetup_accuracy.sh b/e2e-rag/reference_mlperf_datasetup_accuracy.sh index e68e7c2038..85e262c88b 100755 --- a/e2e-rag/reference_mlperf_datasetup_accuracy.sh +++ b/e2e-rag/reference_mlperf_datasetup_accuracy.sh @@ -43,8 +43,11 @@ export NUM_EMBEDDING_DEVICES=${NUM_EMBEDDING_DEVICES:-1} export VECTOR_INDEX_METHOD=${VECTOR_INDEX_METHOD:-"hnsw"} # Reference DB manifest for cross-system verification (corpus fingerprint, -# sample-embedding cosine, probe-query top-K ranks). Set to "" to skip. -export MANIFEST=${MANIFEST:-scripts/db_manifest_intel_xpu.json.gz} +# sample-embedding cosine, probe-query top-K ranks). +# Set to "" or "none" to skip the manifest check. Note: ${VAR:-default} treats +# an empty value the same as unset, so an explicit "none" sentinel is the +# reliable way to skip from a parent script that exports MANIFEST="". +export MANIFEST=${MANIFEST-scripts/db_manifest_intel_xpu.json.gz} export COSINE_THRESHOLD=${COSINE_THRESHOLD:-0.9999} export TOP_K_DEPTH=${TOP_K_DEPTH:-3} @@ -139,9 +142,9 @@ if [ ${EXIT_CODE} -eq 0 ]; then echo "Running Accuracy Evaluation" echo "============================================================" - # Build optional manifest argument (skip check if MANIFEST is empty) + # Build optional manifest argument (skip check if MANIFEST is empty or "none") MANIFEST_ARG="" - if [ -n "${MANIFEST}" ]; then + if [ -n "${MANIFEST}" ] && [ "${MANIFEST,,}" != "none" ]; then MANIFEST_ARG="--manifest ${MANIFEST} --cosine_threshold ${COSINE_THRESHOLD} --top_k_depth ${TOP_K_DEPTH}" fi diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index 96255226c1..7e784f502d 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -118,4 +118,7 @@ python3 reference_mlperf.py \ --judge_model ${JUDGE_MODEL} \ ${PERF_CACHE_ARG} +EXIT_CODE=$? + echo "Time Stop: $(date +%s)" +exit ${EXIT_CODE} diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index e6ba9120b8..4ee05e7209 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -27,7 +27,11 @@ echo "" # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -COMPLIANCE_DIR="${SCRIPT_DIR}/../compliance/TEST09/e2e-rag-qna" +# The TEST09 config comes from the main inference repo's compliance tree. +# Copy compliance/TEST09/e2e-rag-qna/ into this directory before running +# (e.g. when e2e-rag is mounted standalone into a container), or override +# COMPLIANCE_DIR to point at it. +COMPLIANCE_DIR="${COMPLIANCE_DIR:-${SCRIPT_DIR}/../compliance/TEST09/e2e-rag-qna}" AUDIT_CONFIG="${COMPLIANCE_DIR}/audit.config" WORKING_AUDIT_CONFIG="${SCRIPT_DIR}/audit.config" TEST09_VERIFICATION="${SCRIPT_DIR}/third_party/mlperf-inference/compliance/TEST09/run_verification.py" @@ -42,8 +46,9 @@ export OUTPUT_DIR=${WORKSPACE_DIR}/output_test09 export SUBMISSION_DIR=${WORKSPACE_DIR}/submission/compliance/e2e-rag-qna/Offline export SCENARIO="${SCENARIO:-Offline}" -# Performance testing - full dataset for compliance -export PERF_COUNT=824 +# Performance testing - full dataset for compliance. +# Overridable (e.g. for a smoke run); a valid TEST09 submission needs 824. +export PERF_COUNT=${PERF_COUNT:-824} # Threading configuration export MAX_ASYNC_QUERIES=${MAX_ASYNC_QUERIES:-10} @@ -111,7 +116,11 @@ mkdir -p "${SUBMISSION_DIR}" # Copy audit.config to working directory echo "Copying audit.config to working directory..." cp "${AUDIT_CONFIG}" "${WORKING_AUDIT_CONFIG}" -echo "✓ audit.config copied to ${WORKING_AUDIT_CONFIG}" +# Keep the audit.config's min_query_count in sync with PERF_COUNT. Otherwise +# loadgen honors the config's min_query_count (824) and loops back up to it even +# when PERF_COUNT is smaller (e.g. a smoke run). A real submission uses 824. +sed -i "s/^\*\.\*\.min_query_count = .*/*.*.min_query_count = ${PERF_COUNT}/" "${WORKING_AUDIT_CONFIG}" +echo "✓ audit.config copied to ${WORKING_AUDIT_CONFIG} (min_query_count=${PERF_COUNT})" echo "" # ============================================================================ @@ -131,11 +140,15 @@ if [ -n "${PERF_CACHE_FILE}" ] && [ -f "${PERF_CACHE_FILE}" ]; then fi # Run loadgen performance test -# Note: LoadGen automatically detects audit.config in the current directory +# reference_mlperf.py passes --audit_conf explicitly to StartTestWithLogSettings, +# so we must point it at the copied audit.config (named audit.config, whereas +# the default arg is audit.conf). Without this, loadgen never applies the TEST09 +# accuracy_log_sampling_target and mlperf_log_accuracy.json comes out empty. python3 reference_mlperf.py \ --dataset_path ${DATASET_PATH} \ --database ${DATABASE} \ --scenario ${SCENARIO} \ + --audit_conf ${WORKING_AUDIT_CONFIG} \ --log_dir ${RUN_LOGS} \ --output_dir ${OUTPUT_DIR} \ --perf_count ${PERF_COUNT} \ diff --git a/e2e-rag/user.conf b/e2e-rag/user.conf index 960c0fed7c..c023795e57 100644 --- a/e2e-rag/user.conf +++ b/e2e-rag/user.conf @@ -20,7 +20,5 @@ e2e-rag-qna.Offline.max_async_queries = 10 # Send all documents at once for parallel processing e2e-rag-db.Offline.target_qps = 0 e2e-rag-db.Offline.min_duration = 0 -e2e-rag-db.Offline.min_query_count = 2503 -e2e-rag-db.Offline.max_async_queries = 2503 -e2e-datasetup.Offline.max_async_queries = 2515 -e2e-datasetup.Offline.min_query_count = 2515 +e2e-rag-db.Offline.min_query_count = 2515 +e2e-rag-db.Offline.max_async_queries = 2515 From 177fc2ab61b09b8cc806e92e40f8fe1a860f64cd Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 21 Jul 2026 23:20:15 -0700 Subject: [PATCH 14/14] Remove submission_checker/constants.py change from workload branch The e2e-rag rename originally also edited the submission checker's constants.py, but all checker changes now live in the dedicated e2e-rag-submission-checker-fixes branch/PR. Revert constants.py to upstream here so the workload PR touches no checker files and does not conflict with the checker PR. Co-Authored-By: Claude Opus 4.8 --- .../submission_checker/constants.py | 126 +++++++++++------- 1 file changed, 78 insertions(+), 48 deletions(-) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index aefe635a4d..4b3de4a55b 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -17,11 +17,12 @@ "gpt-oss-120b", "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "dlrm-v3", "yolo-95", "yolo-99", - "e2e-rag-qna", - "e2e-rag-db" + "e2e", + "e2e_vectorDB" ], "required-scenarios-datacenter": { "dlrm-v3": ["Server", "Offline"], @@ -36,8 +37,8 @@ "gpt-oss-120b": ["Offline"], "qwen3-vl-235b-a22b": ["Server", "Offline"], "wan-2.2-t2v-a14b": ["Offline", "SingleStream"], - "e2e-rag-qna": ["Offline"], - "e2e-rag-db": ["Offline"] + "e2e": ["Offline"], + "e2e_vectorDB": ["Offline"] }, "optional-scenarios-datacenter": { "llama2-70b-99": ["Interactive", "Server"], @@ -58,6 +59,7 @@ "whisper": ["Offline"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-edge": {}, "required-scenarios-datacenter-edge": { @@ -79,6 +81,7 @@ "dlrm-v3": ["Offline", "Server"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-datacenter-edge": { "llama2-70b-99": ["Interactive", "Server"], @@ -162,8 +165,9 @@ "wan-2.2-t2v-a14b": ("vbench_score", 70.48 * 0.99), # TODO: Set e2e accuracy threshold once reference score is # established - "e2e-rag-qna": ("E2E_ACCURACY", ""), - "e2e-rag-db": ("E2E_ACCURACY", ""), + "e2e": ("E2E_ACCURACY", ""), + "e2e_vectorDB": ("E2E_ACCURACY", ""), + "qwen3.6-27b": ("mAP", 86.23 * 0.99), }, "accuracy-upper-limit": { "stable-diffusion-xl": ( @@ -202,12 +206,19 @@ "dlrm-v3": 349823, "yolo-95": 64, "yolo-99": 64, - "e2e-rag-qna": 824, - "e2e-rag-db": 824 + "e2e": 824, + "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "accuracy-sample-count": { "gpt-oss-120b": 4395, "wan-2.2-t2v-a14b": 248, + "qwen3.6-27b": 995, + "qwen3-vl-235b-a22b": { + "Offline": 48289, + "Interactive": 8000, + "Server": 48289, + }, }, "dataset-size": { "resnet": 50000, @@ -229,8 +240,9 @@ "dlrm-v3": 349823, "yolo-95": 1525, "yolo-99": 1525, - "e2e-rag-qna": 824, - "e2e-rag-db": 824, + "e2e": 824, + "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "model_mapping": { "ssd-resnet34": "retinanet", @@ -289,8 +301,9 @@ "wan-2.2-t2v-a14b": {"SingleStream": 50, "Offline": 1}, "yolo-95": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, "yolo-99": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, - "e2e-rag-qna": {"Offline": 824}, - "e2e-rag-db": {"Offline": 824}, + "e2e": {"Offline": 824}, + "e2e_vectorDB": {"Offline": 824}, + "qwen3.6-27b": {"SingleStream": 995}, }, "models_TEST01": [ "resnet", @@ -323,7 +336,6 @@ ], "models_TEST09": [ "gpt-oss-120b", - "e2e-rag-qna", ], "models_TEST08": [ "dlrm-v3", @@ -1502,6 +1514,9 @@ "compliance_accuracy.txt", ] +TEST09_LOW = 1150.38 +TEST09_HIGH = 1406.02 + OFFLINE_MIN_SPQ_SINCE_V4 = { "resnet": 24576, "retinanet": 24576, @@ -1531,8 +1546,8 @@ "dlrm-v3": 349823, "qwen3-vl-235b-a22b": 48289, "wan-2.2-t2v-a14b": 50, - "e2e-rag-qna": 824, - "e2e-rag-db": 824 + "e2e": 824, + "e2e_vectorDB": 824 } SCENARIO_MAPPING = { @@ -1574,6 +1589,17 @@ "SingleStream": "early_stopping_latency_ss", "MultiStream": "early_stopping_latency_ms", "Server": "result_completed_samples_per_sec", + "Interactive": "result_completed_samples_per_sec", + }, +} + +RESULT_FIELD_ENDPOINTS = { + "v6.1": { + "offline": "result_samples_per_second", + "singlestream": "result_mean_latency_ns", + "multistream": "result_mean_latency_ns", + "server": "result_completed_samples_per_sec", + "interactive": "result_completed_samples_per_sec", }, } @@ -2030,16 +2056,19 @@ ENDPOINTS_ALLOWED_MODELS = [ "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "llama3.1-8b", "llama3.1-8b-edge", "gpt-oss-120b", + "deepseek-r1" ] -PERFORMANCE_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", +ENDPOINTS_SCENARIO_DIR = { + "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", } ACCURACY_LOG_PATH = { @@ -2066,14 +2095,6 @@ "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json", } -ACCURACY_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", -} - - POWER_DIR_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", @@ -2206,7 +2227,7 @@ "effective_sample_concatenate_permutation": "effective_sample_concatenate_permutation", "effective_samples_per_query": "effective_samples_per_query", "generated_query_count": "generated_query_count", - "generated_query_duration": "generated_query_duration", + "duration_ns": "generated_query_duration", "target_qps": "effective_target_qps", "result_scheduled_samples_per_sec": "result_scheduled_samples_per_sec", "qps": "result_completed_samples_per_sec", @@ -2215,30 +2236,31 @@ "latency.min": "result_min_latency_ns", "latency.max": "result_max_latency_ns", "latency.avg": "result_mean_latency_ns", - "latency.percentiles.50.0": "result_50.00_percentile_latency_ns", - "latency.percentiles.90.0": "result_90.00_percentile_latency_ns", - "latency.percentiles.95.0": "result_95.00_percentile_latency_ns", - "latency.percentiles.99.0": "result_99.00_percentile_latency_ns", - "latency.percentiles.99.9": "result_99.90_percentile_latency_ns", + "latency.early_stopping_percentiles.50.0": "result_50.00_percentile_latency_ns", + "latency.early_stopping_percentiles.90.0": "result_90.00_percentile_latency_ns", + "latency.early_stopping_percentiles.95.0": "result_95.00_percentile_latency_ns", + "latency.early_stopping_percentiles.99.0": "result_99.00_percentile_latency_ns", + "latency.early_stopping_percentiles.99.9": "result_99.90_percentile_latency_ns", "ttft.min": "result_first_token_min_latency_ns", "ttft.max": "result_first_token_max_latency_ns", "ttft.avg": "result_first_token_mean_latency_ns", - "ttft.percentiles.50.0": "result_first_token_50.00_percentile_latency_ns", - "ttft.percentiles.90.0": "result_first_token_90.00_percentile_latency_ns", - "ttft.percentiles.95.0": "result_first_token_95.00_percentile_latency_ns", - "ttft.percentiles.99.0": "result_first_token_99.00_percentile_latency_ns", - "ttft.percentiles.99.9": "result_first_token_99.90_percentile_latency_ns", - "tpot.percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns", - "tpot.percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns", - "tpot.percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns", - "tpot.percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns", - "tpot.percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns", + "ttft.early_stopping_percentiles.50.0": "result_first_token_50.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.90.0": "result_first_token_90.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.95.0": "result_first_token_95.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.99.0": "result_first_token_99.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.99.9": "result_first_token_99.90_percentile_latency_ns", + "tpot.early_stopping_percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns", + "tpot.early_stopping_percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns", + "tpot.early_stopping_percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns", + "tpot.early_stopping_percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns", + "tpot.early_stopping_percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns", "tpot.min": "result_time_per_output_token_min", "tpot.max": "result_time_per_output_token_max", "tpot.avg": "result_time_per_output_token_mean", "tps": "result_completed_tokens_per_second", "result.total": "result_query_count", "result.failed": "num_errors", + "output_sequence_lengths.avg": "mean_output_tokens", } @@ -2260,9 +2282,9 @@ # Alternative JSON paths for endpoints keys that don't directly match the # JSON structure ENDPOINTS_JSON_ALT_PATHS = { - "result.total": "results.total", - "result.failed": "results.failed", - "qps": "results.qps", + "result.total": "n_samples_completed", + "result.failed": "n_samples_failed", + "results.qps": "qps", "generated_query_count": "n_samples_issued", "generated_query_duration": "duration_ns", "test_datetime": "test_started_at", @@ -2271,5 +2293,13 @@ } ENDPOINTS_INFERRED_FIELDS = { - "effective_accuracy_sample_count": "result_query_count" + "generated_query_count": "qsl_reported_total_count", +} + +ENDPOINTS_COMPLIANCE_MAPPING = { + "TEST01": "accuracy_in_performance_test", + "TEST04": "output_caching_test", + "TEST06": "consistency_output_test", + "TEST07": "accuracy_in_performance_full_test", + "TEST08": "accuracy_in_performance_dlrmv3_test", }