Merge pull request #138 from jaredkhan/iucn-wikidata

IUCN wikidata

Author: jaredkhan

dvc.lock

@@ -7,26 +7,26 @@ stages:
     deps:
     - path: data/Wiki/wd_JSON/latest-all-json-bz2-url.txt
       hash: md5
-      md5: e094b0f57c0c14e1016842c2dac5482e
+      md5: aa1692624401f4a1c759bc9526787fd7
       size: 90
     outs:
     - path: data/filtered/OneZoom_latest-all.json
       hash: md5
-      md5: 65b22a5c761c78d79b30faf871d1e404
-      size: 1542990225
+      md5: e6f69def9d6fa2bb4b90060509c079bb
+      size: 1567956473
   extract_wikidata_titles:
     cmd: extract_wikidata_titles data/filtered/OneZoom_latest-all.json -o 
       data/filtered/wikidata_titles.txt
     deps:
     - path: data/filtered/OneZoom_latest-all.json
       hash: md5
-      md5: 65b22a5c761c78d79b30faf871d1e404
-      size: 1542990225
+      md5: e6f69def9d6fa2bb4b90060509c079bb
+      size: 1567956473
     outs:
     - path: data/filtered/wikidata_titles.txt
       hash: md5
-      md5: e498b85311c8a84a0d5157a0bbbcb23f
-      size: 9382189
+      md5: 8a8d1fc7e30c8c90890a761a4ba22f2d
+      size: 9540289
   download_wikipedia_sql:
     cmd: wget --progress=bar:force -O data/Wiki/wp_SQL/enwiki-page.sql.gz "$(cat
       data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt)"
@@ -46,13 +46,13 @@ stages:
     deps:
     - path: data/filtered/wikidata_titles.txt
       hash: md5
-      md5: e498b85311c8a84a0d5157a0bbbcb23f
-      size: 9382189
+      md5: 8a8d1fc7e30c8c90890a761a4ba22f2d
+      size: 9540289
     outs:
     - path: data/filtered/pageviews/
       hash: md5
-      md5: f021afa12f9d7c893412a0b2980ab187.dir
-      size: 104535557
+      md5: 701c03fc874aa8a4d5578a6a74267bee.dir
+      size: 128216689
       nfiles: 13
   filter_wikipedia_sql:
     cmd: filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz 
@@ -65,13 +65,13 @@ stages:
       size: 2397370114
     - path: data/filtered/wikidata_titles.txt
       hash: md5
-      md5: e498b85311c8a84a0d5157a0bbbcb23f
-      size: 9382189
+      md5: 8a8d1fc7e30c8c90890a761a4ba22f2d
+      size: 9540289
     outs:
     - path: data/filtered/OneZoom_enwiki-latest-page.sql
       hash: md5
-      md5: af577077774060f78a752f22e2f1c6d3
-      size: 21445833
+      md5: f7437ba9e2a89ff658749e434a38b39a
+      size: 21507159
   download_opentree:
     cmd: download_opentree --version v16.1 --output-dir data/OpenTree
     params:
@@ -225,20 +225,20 @@ stages:
       size: 417054016
     - path: data/filtered/OneZoom_enwiki-latest-page.sql
       hash: md5
-      md5: af577077774060f78a752f22e2f1c6d3
-      size: 21445833
+      md5: f7437ba9e2a89ff658749e434a38b39a
+      size: 21507159
     - path: data/filtered/OneZoom_latest-all.json
       hash: md5
-      md5: 65b22a5c761c78d79b30faf871d1e404
-      size: 1542990225
+      md5: e6f69def9d6fa2bb4b90060509c079bb
+      size: 1567956473
     - path: data/filtered/OneZoom_provider_ids.csv
       hash: md5
       md5: f7c9bb8374957c07168bec36d6591347
       size: 221682224
     - path: data/filtered/pageviews/
       hash: md5
-      md5: f021afa12f9d7c893412a0b2980ab187.dir
-      size: 104535557
+      md5: 701c03fc874aa8a4d5578a6a74267bee.dir
+      size: 128216689
       nfiles: 13
     params:
       params.yaml:
@@ -249,8 +249,8 @@ stages:
     outs:
     - path: data/output_files/
       hash: md5
-      md5: 81dfa9c3747bbe94c57161cc8e22d5c9.dir
-      size: 1158499655
+      md5: 3a18d1302a90a924d080c6a84b2429ad.dir
+      size: 1158320679
       nfiles: 7
   make_js_treefiles:
     cmd:
@@ -260,16 +260,16 @@ stages:
     deps:
     - path: data/output_files/
       hash: md5
-      md5: 81dfa9c3747bbe94c57161cc8e22d5c9.dir
-      size: 1158499655
+      md5: 3a18d1302a90a924d080c6a84b2429ad.dir
+      size: 1158320679
       nfiles: 7
   discover_latest_wikidata_dump_url:
     cmd: discover_latest_wikidata_dump_url > 
       data/Wiki/wd_JSON/latest-all-json-bz2-url.txt
     outs:
     - path: data/Wiki/wd_JSON/latest-all-json-bz2-url.txt
       hash: md5
-      md5: e094b0f57c0c14e1016842c2dac5482e
+      md5: aa1692624401f4a1c759bc9526787fd7
       size: 90
   discover_latest_enwiki_sql_url:
     cmd: discover_latest_enwiki_sql_url > 

oz_tree_build/images_and_vernaculars/get_wiki_images.py

@@ -44,6 +44,7 @@
 from PIL import Image
 
 from .._OZglobals import src_flags
+from ..user_agent import USER_AGENT_HEADERS
 from ..utilities.db_helper import (
     connect_to_database,
     default_appconfig,
@@ -70,9 +71,6 @@
     "img",
 )
 
-# See https://meta.wikimedia.org/wiki/User-Agent_policy
-wiki_http_headers = {"User-Agent": "OneZoomBot/0.1 (https://www.onezoom.org/; mail@onezoom.org) get-wiki-images/0.1"}
-
 
 # Copied from OZTree/OZprivate/ServerScripts/Utilities/getEOL_crops.py
 def subdir_name(doID):
@@ -95,7 +93,7 @@ def make_http_request_with_retries(url):
     retries = 6
     delay = 1
     for i in range(retries):
-        r = requests.get(url, headers=wiki_http_headers)
+        r = requests.get(url, headers=USER_AGENT_HEADERS)
         if r.status_code == 200:
             return r
 
@@ -371,7 +369,7 @@ def save_wiki_image(db, leaf_data, image_name, src, src_id, rating, output_dir,
 
     # Download the uncropped image
     uncropped_image_path = f"{image_dir}/{src_id}_uncropped.jpg"
-    response = requests.get(image_url, headers=wiki_http_headers)
+    response = requests.get(image_url, headers=USER_AGENT_HEADERS)
     response.raise_for_status()
     with open(uncropped_image_path, "wb") as f:
         for chunk in response.iter_content(1024):

oz_tree_build/user_agent.py

@@ -0,0 +1,5 @@
+"""Shared HTTP User-Agent for outbound requests from tree-build."""
+
+# See https://meta.wikimedia.org/wiki/User-Agent_policy
+USER_AGENT = "OneZoom-tree-build/1.0 (https://www.onezoom.org/; mail@onezoom.org)"
+USER_AGENT_HEADERS = {"User-Agent": USER_AGENT}

oz_tree_build/utilities/download_and_filter_pageviews.py

@@ -16,6 +16,8 @@
 import tempfile
 import urllib.request
 
+from oz_tree_build.user_agent import USER_AGENT_HEADERS
+
 from .file_utils import stream_bz2_lines_from_url
 from .filter_pageviews import filter_pageview_lines, write_filtered_pageviews
 from .filter_wikidata import load_titles_file
@@ -28,7 +30,7 @@
 def _fetch_index(url):
     """Fetch an Apache directory index page and return its HTML."""
     print(f"Fetching index from {url}")
-    req = urllib.request.Request(url, headers={"User-Agent": "OneZoom-tree-build/1.0"})
+    req = urllib.request.Request(url, headers=USER_AGENT_HEADERS)
     with urllib.request.urlopen(req, timeout=30) as resp:
         return resp.read().decode("utf-8")
 

oz_tree_build/utilities/download_and_filter_wikidata.py

@@ -13,6 +13,8 @@
 import tempfile
 import urllib.request
 
+from oz_tree_build.user_agent import USER_AGENT_HEADERS
+
 from .file_utils import stream_bz2_lines_from_url
 from .filter_wikidata import filter_wikidata
 
@@ -28,7 +30,14 @@ def discover_latest_wikidata_dump_url(base_url=WIKIDATA_ENTITIES_URL, timeout=30
     folder_re = re.compile(r'href="(\d{8})/"')
     file_re_template = r'href="(wikidata-{date}-all\.json\.bz2)"'
 
-    index_html = urllib.request.urlopen(base_url, timeout=timeout).read().decode()
+    index_html = (
+        urllib.request.urlopen(
+            urllib.request.Request(base_url, headers=USER_AGENT_HEADERS),
+            timeout=timeout,
+        )
+        .read()
+        .decode()
+    )
 
     dates = sorted(folder_re.findall(index_html), reverse=True)
     if not dates:
@@ -38,7 +47,14 @@ def discover_latest_wikidata_dump_url(base_url=WIKIDATA_ENTITIES_URL, timeout=30
         folder_url = f"{base_url}{date}/"
         logger.info("Checking %s", folder_url)
         try:
-            folder_html = urllib.request.urlopen(folder_url, timeout=timeout).read().decode()
+            folder_html = (
+                urllib.request.urlopen(
+                    urllib.request.Request(folder_url, headers=USER_AGENT_HEADERS),
+                    timeout=timeout,
+                )
+                .read()
+                .decode()
+            )
         except urllib.error.URLError as exc:
             logger.warning("Could not fetch %s: %s", folder_url, exc)
             continue

oz_tree_build/utilities/file_utils.py

@@ -9,6 +9,8 @@
 import shutil
 import subprocess
 
+from oz_tree_build.user_agent import USER_AGENT
+
 __author__ = "David Ebbo"
 
 
@@ -40,7 +42,7 @@ def stream_bz2_lines_from_url(url, read_timeout=120):
             "-",
             "--connect-timeout=30",
             f"--read-timeout={read_timeout}",
-            "--header=User-Agent: OneZoom-tree-build/1.0",
+            f"--header=User-Agent: {USER_AGENT}",
             url,
         ],
         stdout=subprocess.PIPE,

oz_tree_build/utilities/filter_wikidata.py

@@ -40,7 +40,15 @@
         "P961": [{"mainsnak": {"datavalue": {"value": KEEP}}}],
         "P9157": [{"mainsnak": {"datavalue": {"value": KEEP}}}],
         "P3151": [{"mainsnak": {"datavalue": {"value": KEEP}}}],
-        "P141": [{"references": [{"snaks": {"P627": [{"datavalue": {"value": KEEP}}]}}]}],
+        "P627": [{"mainsnak": {"datavalue": {"value": KEEP}}}],
+        "P141": [
+            {
+                "mainsnak": {"datavalue": {"value": {"numeric-id": KEEP}}},
+                "rank": KEEP,
+                "qualifiers": {"P585": [{"datavalue": {"value": {"time": KEEP}}}]},
+                "references": [{"snaks": {"P627": [{"datavalue": {"value": KEEP}}]}}],
+            }
+        ],
         "P1420": [{"mainsnak": {"datavalue": {"value": {"numeric-id": KEEP}}}}],
         "P18": [
             {
@@ -136,9 +144,7 @@ def trim_and_write_json_item(json_item, filtered_wiki_f):
             else:
                 potential_extra_json_items.append(("vernacular", vernaculars_matches, json_item))
 
-        logging.info(
-            "Writing extra lines at the end of the file " f"(subset of {len(potential_extra_json_items)} lines)"
-        )
+        logging.info(f"Writing extra lines at the end of the file (subset of {len(potential_extra_json_items)} lines)")
 
         for desc, linked_qids, json_item in potential_extra_json_items:
             for qid in linked_qids:

oz_tree_build/wiki_extraction/mwparserfromhell_helpers.py

@@ -9,13 +9,15 @@
 import mwparserfromhell
 import requests_cache
 
+from oz_tree_build.user_agent import USER_AGENT_HEADERS
+
 session = requests_cache.CachedSession("http_cache")
 
 API_URL = "https://en.wikipedia.org/w/api.php"
 
 
 def wiki_api_request(params):
-    headers = {"User-Agent": "My-Bot-Name/1.0"}
+    headers = USER_AGENT_HEADERS
     # Use this line to avoid caching when testing
     # headers["Cache-Control"] = "no-cache"
     max_retries = 30