diff --git a/db/seeds/site_documents_2025_09_25.zip b/db/seeds/site_documents_2025_09_25.zip deleted file mode 100644 index 652def94..00000000 Binary files a/db/seeds/site_documents_2025_09_25.zip and /dev/null differ diff --git a/db/seeds/site_documents_2025_10_07.zip b/db/seeds/site_documents_2025_10_07.zip new file mode 100644 index 00000000..0eb6bddf Binary files /dev/null and b/db/seeds/site_documents_2025_10_07.zip differ diff --git a/python_components/ci/scripts/local_run_tests.sh b/python_components/ci/scripts/local_run_tests.sh index 1358fc25..11479611 100755 --- a/python_components/ci/scripts/local_run_tests.sh +++ b/python_components/ci/scripts/local_run_tests.sh @@ -15,4 +15,4 @@ BASE_COMMAND="docker run --rm -v $SCRIPT_DIR/../../../:/workspace $LOCAL_CI_IMAG docker build -t $LOCAL_CI_IMAGE $SCRIPT_DIR/../. # Run our pytests. -$BASE_COMMAND pytest -W ignore::DeprecationWarning python_components \ No newline at end of file +$BASE_COMMAND pytest -s -W ignore::DeprecationWarning python_components \ No newline at end of file diff --git a/python_components/crawler/crawler.py b/python_components/crawler/crawler.py index 0fe5cb94..7cf07e44 100644 --- a/python_components/crawler/crawler.py +++ b/python_components/crawler/crawler.py @@ -328,19 +328,33 @@ def add_pdf_metadata(pdfs: dict) -> pd.DataFrame: return pd.DataFrame(output) +def _normalize_url(url: str) -> str: + url = re.sub(r"^https?://", "", str(url).lower()) + url = urllib.parse.unquote(url) + return url + + def compare_crawled_documents(pdf_df: pd.DataFrame, comparison_crawl_df: pd.DataFrame): overlapping_columns = [ column for column in pdf_df.columns if column in comparison_crawl_df.columns ] - overlapping_columns.remove("url") - pdf_df = pdf_df.merge(comparison_crawl_df, how="outer", on="url", indicator=True) + pdf_df["_normalized_url"] = pdf_df["url"].apply(_normalize_url) + comparison_crawl_df["_normalized_url"] = comparison_crawl_df["url"].apply( + _normalize_url + ) + pdf_df = pdf_df.merge( + comparison_crawl_df, how="outer", on="_normalized_url", indicator=True + ) + pdf_df = pdf_df.drop_duplicates(subset=["_normalized_url"]) + + pdf_df = pdf_df.drop("_normalized_url", axis=1) for column in overlapping_columns: pdf_df[column] = np.where( pdf_df["_merge"] == "right_only", pdf_df[f"{column}_y"], pdf_df[f"{column}_x"], ) - # We don't care about the right side, suffixed columns. + # We don't care about the suffixed columns anymore. pdf_df = pdf_df.filter(regex="^(?!.*_[yx]$)") if "crawl_status" in pdf_df.columns: pdf_df = pdf_df.drop("crawl_status", axis=1) @@ -382,14 +396,13 @@ def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> No "output_path", help="Path where a CSV with PDF information will be saved" ) args = parser.parse_args() + config = get_config(args.url) if args.crawled_links_json is None: - config = get_config(args.url) allow_list = config["allow_list"] allowable_subdomains = config.get("allow_subdomains") use_sitemap = config["use_sitemap"] depth = config["depth"] use_webdriver = config.get("use_webdriver", False) - allowable_domains = [ tldextract.extract(link).registered_domain for link in allow_list ] diff --git a/python_components/crawler/tests/fixtures/first_crawl.csv b/python_components/crawler/tests/fixtures/first_crawl.csv index e37da04f..1c6655fb 100644 --- a/python_components/crawler/tests/fixtures/first_crawl.csv +++ b/python_components/crawler/tests/fixtures/first_crawl.csv @@ -1,3 +1,4 @@ file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence download,https://oca.georgia.gov/document/document/guardian-ad-litem-dependency-proceedings-guide-best-interest-advocacy/download,895.4KB,895.443359375,2022-12-16 09:44:53,,,,2022-11-16 17:02:00,Adobe PDF Library 15.0,84,30,3,PDF 1.6,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol#main-content', 'https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/advisory-board', 'https://oca.georgia.gov/frequently-asked-questions#main-content', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/frequently-asked-questions', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings#main-content', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/mandated-reporting', 'https://oca.georgia.gov/national-resources-0#main-content', 'https://oca.georgia.gov/protocol-training#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources/national-resources#main-content', 'https://oca.georgia.gov/training/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting#main-content', 'https://oca.georgia.gov/about-us/advisory-board#main-content', 'https://oca.georgia.gov/about-us/staff#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/protocols-resources/georgia-resources#main-content', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources/national-resources', 'https://oca.georgia.gov/georgia-resources-0#main-content', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/about-us#main-content', 'https://oca.georgia.gov', 'https://oca.georgia.gov/protocols-resources#main-content', 'https://oca.georgia.gov/georgia-resources-0', 'https://oca.georgia.gov/national-resources-0']",['Best Interest Advocacy Guide for Guardians ad Litem'],Brochure,0.6673063635826111 -download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 +download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 +download,http://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources%20p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 diff --git a/python_components/crawler/tests/fixtures/second_crawl.csv b/python_components/crawler/tests/fixtures/second_crawl.csv index d39b9f58..cb65d252 100644 --- a/python_components/crawler/tests/fixtures/second_crawl.csv +++ b/python_components/crawler/tests/fixtures/second_crawl.csv @@ -1,5 +1,5 @@ file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence download,https://oca.georgia.gov/document/document/guardian-ad-litem-dependency-proceedings-guide-best-interest-advocacy/download,895.4KB,895.443359375,2022-12-16 09:44:53,,,,2022-11-16 17:02:00,Adobe PDF Library 15.0,84,30,3,PDF 1.6,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol#main-content', 'https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/advisory-board', 'https://oca.georgia.gov/frequently-asked-questions#main-content', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/frequently-asked-questions', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings#main-content', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/mandated-reporting', 'https://oca.georgia.gov/national-resources-0#main-content', 'https://oca.georgia.gov/protocol-training#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources/national-resources#main-content', 'https://oca.georgia.gov/training/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting#main-content', 'https://oca.georgia.gov/about-us/advisory-board#main-content', 'https://oca.georgia.gov/about-us/staff#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/protocols-resources/georgia-resources#main-content', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources/national-resources', 'https://oca.georgia.gov/georgia-resources-0#main-content', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/about-us#main-content', 'https://oca.georgia.gov', 'https://oca.georgia.gov/protocols-resources#main-content', 'https://oca.georgia.gov/georgia-resources-0', 'https://oca.georgia.gov/national-resources-0']",['Best Interest Advocacy Guide for Guardians ad Litem'],Brochure,0.6673063635826111 -download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 +download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 download,https://oca.georgia.gov/document/document/casa-volunteer-training-manual/download,642.8KB,642.8125,2023-08-22 11:15:30,,,,2023-07-18 14:30:22,Adobe PDF Library 16.0,56,12,8,PDF 1.7,"['https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov#main-content']",['CASA Volunteer Training Manual - Court Appointed Special Advocate'],Manual,0.7234891442298157 download,https://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847 \ No newline at end of file diff --git a/python_components/crawler/tests/fixtures/third_crawl.csv b/python_components/crawler/tests/fixtures/third_crawl.csv index 8a074ad7..6aa575df 100644 --- a/python_components/crawler/tests/fixtures/third_crawl.csv +++ b/python_components/crawler/tests/fixtures/third_crawl.csv @@ -1,5 +1,5 @@ file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence -download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 -download,https://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847 +download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172 +download,http://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847 download,https://oca.georgia.gov/document/document/statewide-child-abuse-protocol-2024/download,1.2MB,1228.8,2024-03-11 13:07:42,Georgia Child Abuse Protocol Committee,,,2024-02-28 09:15:33,Adobe PDF Library 17.0,127,18,6,PDF 1.7,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/protocols-resources#main-content']",['2024 Statewide Model Child Abuse Protocol'],Protocol,0.8923456789012345 download,https://oca.georgia.gov/document/document/family-engagement-strategies-handbook/download,423.1KB,423.1328125,2023-05-17 10:35:28,Dr. Sarah Mitchell,,,2023-05-10 16:45:12,LibreOffice,38,7,15,PDF 1.5,"['https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training#main-content']", \ No newline at end of file diff --git a/python_components/crawler/tests/test_crawler.py b/python_components/crawler/tests/test_crawler.py index ff8f0a28..be21045b 100644 --- a/python_components/crawler/tests/test_crawler.py +++ b/python_components/crawler/tests/test_crawler.py @@ -38,7 +38,7 @@ def test_comparison(): first_crawl = pd.read_csv(f"{test_dir}/fixtures/first_crawl.csv") second_crawl = pd.read_csv(f"{test_dir}/fixtures/second_crawl.csv") third_crawl = pd.read_csv(f"{test_dir}/fixtures/third_crawl.csv") - comparison = compare_crawled_documents(second_crawl, first_crawl) + comparison = compare_crawled_documents(second_crawl, first_crawl.copy()) assert len(comparison[comparison["crawl_status"] == "new"]) == 2 assert len(comparison[comparison["crawl_status"] == "active"]) == 2 assert len(comparison[comparison["crawl_status"] == "removed"]) == 0