Asap 234 new tagged mixed protocol bug

python_components/ci/scripts/local_run_tests.sh

@@ -15,4 +15,4 @@ BASE_COMMAND="docker run --rm -v $SCRIPT_DIR/../../../:/workspace $LOCAL_CI_IMAG
 docker build -t $LOCAL_CI_IMAGE $SCRIPT_DIR/../.
 
 # Run our pytests.
-$BASE_COMMAND pytest -W ignore::DeprecationWarning python_components
+$BASE_COMMAND pytest -s -W ignore::DeprecationWarning python_components

python_components/crawler/crawler.py

@@ -328,19 +328,33 @@ def add_pdf_metadata(pdfs: dict) -> pd.DataFrame:
     return pd.DataFrame(output)
 
 
+def _normalize_url(url: str) -> str:
+    url = re.sub(r"^https?://", "", str(url).lower())
+    url = urllib.parse.unquote(url)
+    return url
+
+
 def compare_crawled_documents(pdf_df: pd.DataFrame, comparison_crawl_df: pd.DataFrame):
     overlapping_columns = [
         column for column in pdf_df.columns if column in comparison_crawl_df.columns
     ]
-    overlapping_columns.remove("url")
-    pdf_df = pdf_df.merge(comparison_crawl_df, how="outer", on="url", indicator=True)
+    pdf_df["_normalized_url"] = pdf_df["url"].apply(_normalize_url)
+    comparison_crawl_df["_normalized_url"] = comparison_crawl_df["url"].apply(
+        _normalize_url
+    )
+    pdf_df = pdf_df.merge(
+        comparison_crawl_df, how="outer", on="_normalized_url", indicator=True
+    )
+    pdf_df = pdf_df.drop_duplicates(subset=["_normalized_url"])
+
+    pdf_df = pdf_df.drop("_normalized_url", axis=1)
     for column in overlapping_columns:
         pdf_df[column] = np.where(
             pdf_df["_merge"] == "right_only",
             pdf_df[f"{column}_y"],
             pdf_df[f"{column}_x"],
         )
-    # We don't care about the right side, suffixed columns.
+    # We don't care about the suffixed columns anymore.
     pdf_df = pdf_df.filter(regex="^(?!.*_[yx]$)")
     if "crawl_status" in pdf_df.columns:
         pdf_df = pdf_df.drop("crawl_status", axis=1)
@@ -382,14 +396,13 @@ def output_pdfs(pdf_df: pd.DataFrame, output_path: str, site_config: dict) -> No
         "output_path", help="Path where a CSV with PDF information will be saved"
     )
     args = parser.parse_args()
+    config = get_config(args.url)
     if args.crawled_links_json is None:
-        config = get_config(args.url)
         allow_list = config["allow_list"]
         allowable_subdomains = config.get("allow_subdomains")
         use_sitemap = config["use_sitemap"]
         depth = config["depth"]
         use_webdriver = config.get("use_webdriver", False)
-
         allowable_domains = [
             tldextract.extract(link).registered_domain for link in allow_list
         ]

python_components/crawler/tests/fixtures/first_crawl.csv

@@ -1,3 +1,4 @@
 file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence
 download,https://oca.georgia.gov/document/document/guardian-ad-litem-dependency-proceedings-guide-best-interest-advocacy/download,895.4KB,895.443359375,2022-12-16 09:44:53,,,,2022-11-16 17:02:00,Adobe PDF Library 15.0,84,30,3,PDF 1.6,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol#main-content', 'https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/advisory-board', 'https://oca.georgia.gov/frequently-asked-questions#main-content', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/frequently-asked-questions', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings#main-content', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/mandated-reporting', 'https://oca.georgia.gov/national-resources-0#main-content', 'https://oca.georgia.gov/protocol-training#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources/national-resources#main-content', 'https://oca.georgia.gov/training/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting#main-content', 'https://oca.georgia.gov/about-us/advisory-board#main-content', 'https://oca.georgia.gov/about-us/staff#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/protocols-resources/georgia-resources#main-content', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources/national-resources', 'https://oca.georgia.gov/georgia-resources-0#main-content', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/about-us#main-content', 'https://oca.georgia.gov', 'https://oca.georgia.gov/protocols-resources#main-content', 'https://oca.georgia.gov/georgia-resources-0', 'https://oca.georgia.gov/national-resources-0']",['Best Interest Advocacy Guide for Guardians ad Litem'],Brochure,0.6673063635826111
-download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
+download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
+download,http://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources%20p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172

python_components/crawler/tests/fixtures/second_crawl.csv

@@ -1,5 +1,5 @@
 file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence
 download,https://oca.georgia.gov/document/document/guardian-ad-litem-dependency-proceedings-guide-best-interest-advocacy/download,895.4KB,895.443359375,2022-12-16 09:44:53,,,,2022-11-16 17:02:00,Adobe PDF Library 15.0,84,30,3,PDF 1.6,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol#main-content', 'https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/advisory-board', 'https://oca.georgia.gov/frequently-asked-questions#main-content', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/frequently-asked-questions', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings#main-content', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/mandated-reporting', 'https://oca.georgia.gov/national-resources-0#main-content', 'https://oca.georgia.gov/protocol-training#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources/national-resources#main-content', 'https://oca.georgia.gov/training/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting#main-content', 'https://oca.georgia.gov/about-us/advisory-board#main-content', 'https://oca.georgia.gov/about-us/staff#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/protocols-resources/georgia-resources#main-content', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources/national-resources', 'https://oca.georgia.gov/georgia-resources-0#main-content', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/training/guardian-ad-litem-juvenile-court-dependency-proceedings', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/about-us#main-content', 'https://oca.georgia.gov', 'https://oca.georgia.gov/protocols-resources#main-content', 'https://oca.georgia.gov/georgia-resources-0', 'https://oca.georgia.gov/national-resources-0']",['Best Interest Advocacy Guide for Guardians ad Litem'],Brochure,0.6673063635826111
-download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
+download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
 download,https://oca.georgia.gov/document/document/casa-volunteer-training-manual/download,642.8KB,642.8125,2023-08-22 11:15:30,,,,2023-07-18 14:30:22,Adobe PDF Library 16.0,56,12,8,PDF 1.7,"['https://oca.georgia.gov/training/court-appointed-special-advocate-casa', 'https://oca.georgia.gov/court-appointed-special-advocate-casa#main-content', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training#main-content', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov#main-content']",['CASA Volunteer Training Manual - Court Appointed Special Advocate'],Manual,0.7234891442298157
 download,https://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847

python_components/crawler/tests/fixtures/third_crawl.csv

@@ -1,5 +1,5 @@
 file_name,url,file_size,file_size_kilobytes,last_modified_date,author,subject,keywords,creation_date,producer,number_of_pages,number_of_tables,number_of_images,version,source,text_around_link,predicted_category,predicted_category_confidence
-download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
-download,https://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847
+download,https://oca.georgia.gov/document/document/get-ga-reading-free-learning-resources p/download,275.2KB,275.2490234375,2020-03-16 14:43:46,Arianne Weldon,,,2020-03-16 14:43:46,,4,1,9,PDF 1.7,"['https://oca.georgia.gov#main-content', 'https://oca.georgia.gov']",['Get GA Reading - Free Learning Resources'],Brochure,0.5853822827339172
+download,http://oca.georgia.gov/document/document/mandated-reporter-quick-reference-card/download,156.7KB,156.6845703125,2021-09-14 16:22:18,Georgia OCA,,,2021-09-14 16:22:18,Microsoft Word,2,0,2,PDF 1.4,"['https://oca.georgia.gov/training/mandated-reporting', 'https://oca.georgia.gov/mandated-reporting#main-content', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov#main-content']",['Mandated Reporter Quick Reference Card'],Reference,0.8156734521902847
 download,https://oca.georgia.gov/document/document/statewide-child-abuse-protocol-2024/download,1.2MB,1228.8,2024-03-11 13:07:42,Georgia Child Abuse Protocol Committee,,,2024-02-28 09:15:33,Adobe PDF Library 17.0,127,18,6,PDF 1.7,"['https://oca.georgia.gov/protocols-resources/statewide-model-child-abuse-protocol', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov/child-abuse-and-neglect-reporting', 'https://oca.georgia.gov/protocol-training', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/about-us', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/protocols-resources#main-content']",['2024 Statewide Model Child Abuse Protocol'],Protocol,0.8923456789012345
 download,https://oca.georgia.gov/document/document/family-engagement-strategies-handbook/download,423.1KB,423.1328125,2023-05-17 10:35:28,Dr. Sarah Mitchell,,,2023-05-10 16:45:12,LibreOffice,38,7,15,PDF 1.5,"['https://oca.georgia.gov/protocols-resources/georgia-resources', 'https://oca.georgia.gov/training', 'https://oca.georgia.gov/about-us/staff', 'https://oca.georgia.gov/protocols-resources', 'https://oca.georgia.gov#main-content', 'https://oca.georgia.gov/training#main-content']",

python_components/crawler/tests/test_crawler.py

@@ -38,7 +38,7 @@ def test_comparison():
     first_crawl = pd.read_csv(f"{test_dir}/fixtures/first_crawl.csv")
     second_crawl = pd.read_csv(f"{test_dir}/fixtures/second_crawl.csv")
     third_crawl = pd.read_csv(f"{test_dir}/fixtures/third_crawl.csv")
-    comparison = compare_crawled_documents(second_crawl, first_crawl)
+    comparison = compare_crawled_documents(second_crawl, first_crawl.copy())
     assert len(comparison[comparison["crawl_status"] == "new"]) == 2
     assert len(comparison[comparison["crawl_status"] == "active"]) == 2
     assert len(comparison[comparison["crawl_status"] == "removed"]) == 0