diff --git a/app/models/document.rb b/app/models/document.rb index 2fdd6a6c..3c0755e8 100644 --- a/app/models/document.rb +++ b/app/models/document.rb @@ -290,7 +290,7 @@ def primary_source end def get_crawl_status_display - if document_status == DOCUMENT_STATUS_NEW && last_crawl_date.present? && last_crawl_date.after?(1.week.ago) + if document_status == DOCUMENT_STATUS_NEW && last_crawl_date.present? && last_crawl_date.after?(1.month.ago) DOCUMENT_STATUS_NEW elsif document_status == DOCUMENT_STATUS_REMOVED DOCUMENT_STATUS_REMOVED diff --git a/bin/crawl b/bin/crawl index 5c03c063..bc871962 100755 --- a/bin/crawl +++ b/bin/crawl @@ -3,6 +3,7 @@ # Script to crawl a website and extract PDF information # Usage: bin/crawl [previous_crawl_directory] [previous_link_json] # Site site_url must exist in python_components/crawler/config.json. +# output_dir, where to put the new crawl information. # previous_crawl_directory is the path to a previous version of the crawl to compare against. # previous_link_json is a list of links and sources produced by the first phase of the crawler. # Useful if metadata fetching failed or to resume an otherwise halted process. @@ -46,12 +47,13 @@ OUTPUT_FILE=$(jq -r --arg site_url "$SITE_URL" ' end ' "$CONFIG_FILE" 2>/dev/null || echo "output.csv") +CRAWLER_TMP_DIR="$PROJECT_ROOT/crawler_tmp" + if [ -n "$PREVIOUS_CRAWL" ]; then if [ ! -e "$PREVIOUS_CRAWL" ]; then echo "Error: Previous crawl path '$PREVIOUS_CRAWL' does not exist" exit 1 fi - CRAWLER_TMP_DIR="$PROJECT_ROOT/crawler_tmp" PREVIOUS_CRAWL_DIR="$CRAWLER_TMP_DIR/previous_crawl" mkdir -p $PREVIOUS_CRAWL_DIR if [ -f "$PREVIOUS_CRAWL" ] && [[ "$PREVIOUS_CRAWL" == *.zip ]]; then @@ -87,9 +89,16 @@ if [ -n "$PREVIOUS_LINK_JSON" ]; then fi TMP_OUTPUT="$CRAWLER_TMP_DIR/output" -mkdir -p $TMP_OUTPUT -mkdir -p $OUTPUT_DIR -echo "$OUTPUT_DIR" +mkdir -p "$TMP_OUTPUT" +mkdir -p "$OUTPUT_DIR" + +# Verify directories were created +if [ ! -d "$TMP_OUTPUT" ]; then + echo "Error: Failed to create temp output directory: $TMP_OUTPUT" + exit 1 +fi +echo "Output will be saved to: $OUTPUT_DIR" +echo "Temp directory: $TMP_OUTPUT" set -x diff --git a/db/seeds/site_documents_2026_01_22.zip b/db/seeds/site_documents_2026_01_22.zip new file mode 100644 index 00000000..6dad912c Binary files /dev/null and b/db/seeds/site_documents_2026_01_22.zip differ diff --git a/python_components/crawler/crawler.py b/python_components/crawler/crawler.py index 7cf07e44..069f8d92 100644 --- a/python_components/crawler/crawler.py +++ b/python_components/crawler/crawler.py @@ -29,6 +29,65 @@ # To enforce SSL verification remove this line. urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +minimal_headers = { + "Content-Type": "application/pdf", + "Content-Disposition": "inline", +} + +browser_headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", +} + +strategies = [ + {"headers": minimal_headers, "verify": True}, + {"headers": browser_headers, "verify": True}, + {"headers": minimal_headers, "verify": False}, + {"headers": browser_headers, "verify": False}, +] + +preferred_strategy_index = 0 + + +def fetch_with_retry(url, timeout=90, wait_between_retries=2): + """ + Fetch a URL trying multiple strategies (header combinations, SSL verification). + Remembers which strategy worked and tries it first next time. + Returns the response object on success, None on failure. + """ + global preferred_strategy_index + + order = [preferred_strategy_index] + [ + i for i in range(len(strategies)) if i != preferred_strategy_index + ] + + for strategy_index in order: + strategy = strategies[strategy_index] + try: + response = requests.get( + url, + timeout=timeout, + headers=strategy["headers"], + verify=strategy["verify"], + allow_redirects=True, + ) + if response.status_code < 400: + if strategy_index != preferred_strategy_index: + preferred_strategy_index = strategy_index + return response + tqdm.write( + f"Strategy {strategy_index}: {url} returned status {response.status_code}" + ) + except requests.exceptions.RequestException as e: + tqdm.write(f"Strategy {strategy_index}: {url} failed: {e}") + time.sleep(wait_between_retries) + return None + def get_url(url, timeout=90, use_webdriver=False): if use_webdriver: @@ -71,8 +130,8 @@ def get_url(url, timeout=90, use_webdriver=False): return atags else: - response = requests.get(url, timeout=timeout) - if response.status_code >= 400: + response = fetch_with_retry(url, timeout=timeout) + if response is None: return None soup = BeautifulSoup(response.content, "html.parser") @@ -106,17 +165,25 @@ def parse_robots_txt(url, manual_crawl_delay): return sitemap, manual_crawl_delay -def parse_sitemap(sitemap): - r = requests.get(sitemap) +def parse_sitemap(sitemap, delay=0): + r = fetch_with_retry(sitemap) + if r is None: + tqdm.write(f"Failed to fetch sitemap: {sitemap}") + return set() + soup = BeautifulSoup(r.text, "xml") more_site_maps = [site.text for site in soup.find_all("loc")] all_pages = set() for site in more_site_maps: - if manual_crawl_delay: - time.sleep(manual_crawl_delay) + if delay: + time.sleep(delay) + + r = fetch_with_retry(site) + if r is None: + tqdm.write(f"Failed to fetch sitemap: {site}") + continue - r = requests.get(site) soup = BeautifulSoup(r.text, "xml") all_pages.update([x.find("loc").text for x in soup.find_all("url")]) @@ -180,6 +247,7 @@ def bfs_search_pdfs( max_depth=7, timeout=90, use_webdriver=False, + max_pages=None, ): # Restricts search to links sharing the same domain, capture all PDFs # along the way @@ -188,7 +256,14 @@ def bfs_search_pdfs( pdfs = defaultdict(list) pbar = tqdm(unit=" pages") + if max_pages: + tqdm.write(f"Will stop after {max_pages} pages.") while queue: + if max_pages and len(visited) >= max_pages: + tqdm.write( + f"Reached max_pages limit ({max_pages}), visited {len(visited)} pages." + ) + break node, depth = queue.popleft() # Get the next node from the queue pbar.update(1) if node not in visited: @@ -262,7 +337,10 @@ def parse_pdf_date(date_string): def add_pdf_metadata(pdfs: dict) -> pd.DataFrame: output = [] - for pdf_url in tqdm(pdfs.keys(), ncols=100): + pdf_urls = list(pdfs.keys()) + total = len(pdf_urls) + tqdm.write(f"Starting metadata fetch for {total} PDFs...") + for idx, pdf_url in enumerate(tqdm(pdf_urls, ncols=100)): source = list(set([dat["source"] for dat in pdfs[pdf_url]])) texts = list(set([dat["text"] for dat in pdfs[pdf_url]])) try: @@ -270,21 +348,10 @@ def add_pdf_metadata(pdfs: dict) -> pd.DataFrame: default_file_name = url_parsed.path.split("/")[-1] if len(default_file_name) == 0: default_file_name = url_parsed.netloc.split("\\")[-1] - headers = { - "Content-Type": "application/pdf", - "Content-Disposition": "inline", - } - # Since we control domain lists, it feels safe enough to disable SSL verification. - # To enforce SSL cert verification, set verify to True. - response = requests.get( - url=pdf_url, - timeout=90, - headers=headers, - allow_redirects=True, - verify=False, - ) - tqdm.write(f"Attempting metadata fetch for: {pdf_url}") - if response.status_code < 400: + tqdm.write(f"[{idx + 1}/{total}] Fetching: {pdf_url}") + response = fetch_with_retry(pdf_url) + if response is not None: + tqdm.write(" Downloaded, processing...") with io.BytesIO(response.content) as mem_obj: try: pdf_file = pymupdf.Document(stream=mem_obj) @@ -292,12 +359,16 @@ def add_pdf_metadata(pdfs: dict) -> pd.DataFrame: raise RuntimeError( "Could not read document metadata to get page count." ) + tqdm.write( + f" {pdf_file.page_count} pages, scanning for images/tables..." + ) file_name = default_file_name pdf_title = pdf_file.metadata.get("title") if pdf_title and (len(pdf_title.strip()) > 0): file_name = pdf_title file_bytes = mem_obj.getbuffer().nbytes n_images, n_tables = get_images_and_tables(pdf_file.pages()) + tqdm.write(f" Done: {n_images} images, {n_tables} tables") modified = parse_pdf_date(pdf_file.metadata.get("modDate")) created = parse_pdf_date(pdf_file.metadata.get("creationDate")) row = { @@ -325,6 +396,7 @@ def add_pdf_metadata(pdfs: dict) -> pd.DataFrame: tqdm.write(f"Document isn't a PDF: {pdf_url}") except Exception as e: tqdm.write(f"Error reading: {pdf_url} Error: {str(e)}") + tqdm.write(f"Finished processing {len(output)} PDFs, creating DataFrame...") return pd.DataFrame(output) @@ -374,8 +446,11 @@ def add_crawl_date(pdf_df: pd.DataFrame) -> pd.DataFrame: def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> None: if os.path.isdir(output_path): - output_path = f"{output_path}/{site_config.get("output_file")}" + output_file = site_config.get("output_file", "crawl_results.csv") + output_path = f"{output_path}/{output_file}" + print(f"Writing {len(pdf_df)} rows to {output_path}", flush=True) pdf_df.to_csv(output_path, index=False) + print(f"Done. Output saved to {output_path}", flush=True) if __name__ == "__main__": @@ -392,6 +467,12 @@ def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> No default=None, help="Skip the link gathering process and use a previously created JSON file.", ) + parser.add_argument( + "--max_pages", + type=int, + default=None, + help="Maximum number of pages to crawl (for testing).", + ) parser.add_argument( "output_path", help="Path where a CSV with PDF information will be saved" ) @@ -409,7 +490,7 @@ def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> No sitemap, manual_crawl_delay = parse_robots_txt(args.url, args.delay) if use_sitemap: - all_pages = parse_sitemap(sitemap) + all_pages = parse_sitemap(sitemap, delay=manual_crawl_delay) tqdm.write(f"Pages found from sitemap: {len(all_pages)}") crawled_pdfs = get_all_pages(all_pages, delay=manual_crawl_delay) @@ -423,14 +504,23 @@ def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> No delay=manual_crawl_delay, max_depth=depth, use_webdriver=use_webdriver, + max_pages=args.max_pages, ) tqdm.write(f"PDFs found: {len(crawled_pdfs)}") + output_dir = os.path.dirname(args.output_path) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + tqdm.write(f"Created output directory: {output_dir}") with open(args.output_path.replace(".csv", ".json"), "w") as f: json.dump(dict(crawled_pdfs), f, indent=4) else: with open(args.crawled_links_json) as f: crawled_pdfs = json.load(f) crawled_pdfs = add_pdf_metadata(crawled_pdfs) + print( + f"Metadata collection complete. {len(crawled_pdfs)} documents processed.", + flush=True, + ) if args.comparison_crawl is not None: comparison_df = pd.read_csv(args.comparison_crawl) crawled_pdfs = compare_crawled_documents(crawled_pdfs, comparison_df) diff --git a/python_components/document_inference/document_inference/helpers.py b/python_components/document_inference/document_inference/helpers.py index 07f204d3..4f8f531f 100644 --- a/python_components/document_inference/document_inference/helpers.py +++ b/python_components/document_inference/document_inference/helpers.py @@ -62,7 +62,7 @@ def get_secret(secret_name: str, local_mode: bool, aws_env: str) -> str: return response["SecretString"] -def get_file(url: str, output_path: str, wait_to_retry: int = 1000) -> str: +def get_file(url: str, output_path: str, wait_to_retry: int = 2) -> str: file_name = os.path.basename(url) local_path = f"{output_path}/{file_name}" strategies = [ diff --git a/spec/features/document_spec.rb b/spec/features/document_spec.rb index 8f83a0ec..b339e1a0 100644 --- a/spec/features/document_spec.rb +++ b/spec/features/document_spec.rb @@ -736,7 +736,7 @@ expect(page).to have_content "Removed" expect(page).to have_no_content "New" end - document.last_crawl_date = 9.days.ago + document.last_crawl_date = 31.days.ago document.document_status = Document::DOCUMENT_STATUS_NEW document.save! visit "sites/#{site.id}/documents" @@ -746,7 +746,7 @@ expect(page).to have_no_content "Removed" expect(page).to have_no_content "New" end - document.last_crawl_date = 3.days.ago + document.last_crawl_date = 19.days.ago document.save! visit "sites/#{site.id}/documents" expect(page).to have_content "Colorado: City of Denver"