From 4b9777255aa2e7e20fee90259c015082947f7a21 Mon Sep 17 00:00:00 2001 From: "Nathaniel V. KELSO" Date: Wed, 25 May 2022 23:24:38 -0700 Subject: [PATCH 1/4] Add new Indic languages --- tools/wikidata/fetch_wikidata.py | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/tools/wikidata/fetch_wikidata.py b/tools/wikidata/fetch_wikidata.py index e4760a1f..c7121988 100755 --- a/tools/wikidata/fetch_wikidata.py +++ b/tools/wikidata/fetch_wikidata.py @@ -184,6 +184,14 @@ def fetchwikidata(a_wid): ?name_zh ?name_zh_hans ?name_zh_hant + ?name_gu + ?name_kn + ?name_ml + ?name_mr + ?name_pa + ?name_ta + ?name_te + ?name_th WHERE { { SELECT DISTINCT ?e ?i ?r @@ -222,6 +230,15 @@ def fetchwikidata(a_wid): OPTIONAL{?e rdfs:label ?name_zh FILTER((LANG(?name_zh))="zh").} OPTIONAL{?e rdfs:label ?name_zh_hans FILTER((LANG(?name_zh_hans))="zh-hans").} OPTIONAL{?e rdfs:label ?name_zh_hant FILTER((LANG(?name_zh_hant))="zh-hant").} + OPTIONAL{?e rdfs:label ?name_gu FILTER((LANG(?name_gu))="gu").} + OPTIONAL{?e rdfs:label ?name_kn FILTER((LANG(?name_kn))="kn").} + OPTIONAL{?e rdfs:label ?name_ml FILTER((LANG(?name_ml))="ml").} + OPTIONAL{?e rdfs:label ?name_mr FILTER((LANG(?name_mr))="mr").} + OPTIONAL{?e rdfs:label ?name_pa FILTER((LANG(?name_pa))="pa").} + OPTIONAL{?e rdfs:label ?name_ta FILTER((LANG(?name_ta))="ta").} + OPTIONAL{?e rdfs:label ?name_te FILTER((LANG(?name_te))="te").} + OPTIONAL{?e rdfs:label ?name_th FILTER((LANG(?name_th))="th").} + } """ @@ -322,7 +339,16 @@ def fetchwikidata(a_wid): "name_ur", "name_vi", "name_zh", - "name_zht" + "name_zht", + "name_gu", + "name_kn", + "name_ml", + "name_mr", + "name_pa", + "name_ta", + "name_te", + "name_th" + )) @@ -389,6 +415,14 @@ def fetchwikidata(a_wid): name_uk = get_sparql_label(result, 'name_uk') name_ur = get_sparql_label(result, 'name_ur') name_vi = get_sparql_label(result, 'name_vi') + name_gu = get_sparql_label(result, 'name_gu') + name_kn = get_sparql_label(result, 'name_kn') + name_ml = get_sparql_label(result, 'name_ml') + name_mr = get_sparql_label(result, 'name_mr') + name_pa = get_sparql_label(result, 'name_pa') + name_ta = get_sparql_label(result, 'name_ta') + name_te = get_sparql_label(result, 'name_te') + name_th = get_sparql_label(result, 'name_th') # not all Wikidata places have all name (label) translations try: @@ -458,7 +492,16 @@ def fetchwikidata(a_wid): name_ur, name_vi, name_zh, - name_zht + name_zht, + name_gu, + name_kn, + name_ml, + name_mr, + name_pa, + name_ta, + name_te, + name_th + )) -print(' - JOB end -') \ No newline at end of file +print(' - JOB end -') From 4d744b88e3f0cc578f18e9b714ced62505a41686 Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 31 May 2022 11:27:41 -0700 Subject: [PATCH 2/4] alphabetized languages in fetch_wikidata.py added additional language support to write_wikidata.py Added fix for arabic and Chinese/Japanese commas --- tools/wikidata/fetch_wikidata.py | 89 +++++++++++++++----------------- tools/wikidata/write_wikidata.py | 42 ++++++++++++++- 2 files changed, 84 insertions(+), 47 deletions(-) diff --git a/tools/wikidata/fetch_wikidata.py b/tools/wikidata/fetch_wikidata.py index c7121988..ec455b89 100755 --- a/tools/wikidata/fetch_wikidata.py +++ b/tools/wikidata/fetch_wikidata.py @@ -34,7 +34,7 @@ help='input natural-earth shape file - with wikidataid columns') parser.add_argument('-input_lettercase', default='uppercase', - help='variables in thes hape file - lowercase or uppercase') + help='variables in the shape file - lowercase or uppercase') parser.add_argument('-output_csv_name', default='ne_10m_populated_places.csv', help='output csv file with wikidata labels') @@ -165,18 +165,26 @@ def fetchwikidata(a_wid): ?name_es ?name_fa ?name_fr + ?name_gu ?name_he ?name_hi ?name_hu ?name_id ?name_it ?name_ja + ?name_kn ?name_ko + ?name_ml + ?name_mr ?name_nl + ?name_pa ?name_pl ?name_pt ?name_ru ?name_sv + ?name_ta + ?name_te + ?name_th ?name_tr ?name_uk ?name_ur @@ -184,14 +192,6 @@ def fetchwikidata(a_wid): ?name_zh ?name_zh_hans ?name_zh_hant - ?name_gu - ?name_kn - ?name_ml - ?name_mr - ?name_pa - ?name_ta - ?name_te - ?name_th WHERE { { SELECT DISTINCT ?e ?i ?r @@ -211,18 +211,26 @@ def fetchwikidata(a_wid): OPTIONAL{?e rdfs:label ?name_es FILTER((LANG(?name_es))="es").} OPTIONAL{?e rdfs:label ?name_fa FILTER((LANG(?name_fa))="fa").} OPTIONAL{?e rdfs:label ?name_fr FILTER((LANG(?name_fr))="fr").} + OPTIONAL{?e rdfs:label ?name_gu FILTER((LANG(?name_gu))="gu").} OPTIONAL{?e rdfs:label ?name_he FILTER((LANG(?name_he))="he").} OPTIONAL{?e rdfs:label ?name_hi FILTER((LANG(?name_hi))="hi").} OPTIONAL{?e rdfs:label ?name_hu FILTER((LANG(?name_hu))="hu").} OPTIONAL{?e rdfs:label ?name_id FILTER((LANG(?name_id))="id").} OPTIONAL{?e rdfs:label ?name_it FILTER((LANG(?name_it))="it").} OPTIONAL{?e rdfs:label ?name_ja FILTER((LANG(?name_ja))="ja").} + OPTIONAL{?e rdfs:label ?name_kn FILTER((LANG(?name_kn))="kn").} OPTIONAL{?e rdfs:label ?name_ko FILTER((LANG(?name_ko))="ko").} + OPTIONAL{?e rdfs:label ?name_ml FILTER((LANG(?name_ml))="ml").} + OPTIONAL{?e rdfs:label ?name_mr FILTER((LANG(?name_mr))="mr").} OPTIONAL{?e rdfs:label ?name_nl FILTER((LANG(?name_nl))="nl").} + OPTIONAL{?e rdfs:label ?name_pa FILTER((LANG(?name_pa))="pa").} OPTIONAL{?e rdfs:label ?name_pl FILTER((LANG(?name_pl))="pl").} OPTIONAL{?e rdfs:label ?name_pt FILTER((LANG(?name_pt))="pt").} OPTIONAL{?e rdfs:label ?name_ru FILTER((LANG(?name_ru))="ru").} OPTIONAL{?e rdfs:label ?name_sv FILTER((LANG(?name_sv))="sv").} + OPTIONAL{?e rdfs:label ?name_ta FILTER((LANG(?name_ta))="ta").} + OPTIONAL{?e rdfs:label ?name_te FILTER((LANG(?name_te))="te").} + OPTIONAL{?e rdfs:label ?name_th FILTER((LANG(?name_th))="th").} OPTIONAL{?e rdfs:label ?name_tr FILTER((LANG(?name_tr))="tr").} OPTIONAL{?e rdfs:label ?name_uk FILTER((LANG(?name_uk))="uk").} OPTIONAL{?e rdfs:label ?name_ur FILTER((LANG(?name_ur))="ur").} @@ -230,15 +238,6 @@ def fetchwikidata(a_wid): OPTIONAL{?e rdfs:label ?name_zh FILTER((LANG(?name_zh))="zh").} OPTIONAL{?e rdfs:label ?name_zh_hans FILTER((LANG(?name_zh_hans))="zh-hans").} OPTIONAL{?e rdfs:label ?name_zh_hant FILTER((LANG(?name_zh_hant))="zh-hant").} - OPTIONAL{?e rdfs:label ?name_gu FILTER((LANG(?name_gu))="gu").} - OPTIONAL{?e rdfs:label ?name_kn FILTER((LANG(?name_kn))="kn").} - OPTIONAL{?e rdfs:label ?name_ml FILTER((LANG(?name_ml))="ml").} - OPTIONAL{?e rdfs:label ?name_mr FILTER((LANG(?name_mr))="mr").} - OPTIONAL{?e rdfs:label ?name_pa FILTER((LANG(?name_pa))="pa").} - OPTIONAL{?e rdfs:label ?name_ta FILTER((LANG(?name_ta))="ta").} - OPTIONAL{?e rdfs:label ?name_te FILTER((LANG(?name_te))="te").} - OPTIONAL{?e rdfs:label ?name_th FILTER((LANG(?name_th))="th").} - } """ @@ -322,33 +321,32 @@ def fetchwikidata(a_wid): "name_es", "name_fa", "name_fr", + "name_gu", "name_he", "name_hi", "name_hu", "name_id", "name_it", "name_ja", + "name_kn", "name_ko", + "name_ml", + "name_mr", "name_nl", + "name_pa", "name_pl", "name_pt", "name_ru", "name_sv", + "name_ta", + "name_te", + "name_th", "name_tr", "name_uk", "name_ur", "name_vi", "name_zh", - "name_zht", - "name_gu", - "name_kn", - "name_ml", - "name_mr", - "name_pa", - "name_ta", - "name_te", - "name_th" - + "name_zht" )) @@ -398,31 +396,31 @@ def fetchwikidata(a_wid): name_es = get_sparql_label(result, 'name_es') name_fa = get_sparql_label(result, 'name_fa') name_fr = get_sparql_label(result, 'name_fr') + name_gu = get_sparql_label(result, 'name_gu') name_he = get_sparql_label(result, 'name_he') name_hi = get_sparql_label(result, 'name_hi') name_hu = get_sparql_label(result, 'name_hu') name_id = get_sparql_label(result, 'name_id') name_it = get_sparql_label(result, 'name_it') name_ja = get_sparql_label(result, 'name_ja') + name_kn = get_sparql_label(result, 'name_kn') name_ko = get_sparql_label(result, 'name_ko') name_lt = get_sparql_label(result, 'name_lt') + name_ml = get_sparql_label(result, 'name_ml') + name_mr = get_sparql_label(result, 'name_mr') name_nl = get_sparql_label(result, 'name_nl') + name_pa = get_sparql_label(result, 'name_pa') name_pl = get_sparql_label(result, 'name_pl') name_pt = get_sparql_label(result, 'name_pt') name_ru = get_sparql_label(result, 'name_ru') name_sv = get_sparql_label(result, 'name_sv') + name_ta = get_sparql_label(result, 'name_ta') + name_te = get_sparql_label(result, 'name_te') + name_th = get_sparql_label(result, 'name_th') name_tr = get_sparql_label(result, 'name_tr') name_uk = get_sparql_label(result, 'name_uk') name_ur = get_sparql_label(result, 'name_ur') name_vi = get_sparql_label(result, 'name_vi') - name_gu = get_sparql_label(result, 'name_gu') - name_kn = get_sparql_label(result, 'name_kn') - name_ml = get_sparql_label(result, 'name_ml') - name_mr = get_sparql_label(result, 'name_mr') - name_pa = get_sparql_label(result, 'name_pa') - name_ta = get_sparql_label(result, 'name_ta') - name_te = get_sparql_label(result, 'name_te') - name_th = get_sparql_label(result, 'name_th') # not all Wikidata places have all name (label) translations try: @@ -475,33 +473,32 @@ def fetchwikidata(a_wid): name_es, name_fa, name_fr, + name_gu, name_he, name_hi, name_hu, name_id, name_it, name_ja, + name_kn, name_ko, + name_ml, + name_mr, name_nl, + name_pa, name_pl, name_pt, name_ru, name_sv, + name_ta, + name_te, + name_th, name_tr, name_uk, name_ur, name_vi, name_zh, - name_zht, - name_gu, - name_kn, - name_ml, - name_mr, - name_pa, - name_ta, - name_te, - name_th - + name_zht )) print(' - JOB end -') diff --git a/tools/wikidata/write_wikidata.py b/tools/wikidata/write_wikidata.py index 62e5a378..d9cf7bd8 100755 --- a/tools/wikidata/write_wikidata.py +++ b/tools/wikidata/write_wikidata.py @@ -77,7 +77,7 @@ def isNotEmpty(s): wdredirects = defaultdict(dict) name_field_prefix = 'name_' -languages = ['ar','bn','de','el','en','es','fa','fr','he','hi','hu','id','it','ja','ko','nl','pl','pt','ru','sv','tr','uk','ur','vi','zh','zht'] +languages = ['ar','bn','de','el','en','es','fa','fr','gu','he','hi','hu','id','it','ja','kn','ko','ml','mr','nl','pa','pl','pt','ru','sv','ta','te','th','tr','uk','ur','vi','zh','zht'] new_properties = [] with open(args.input_csv, newline='') as csvfile: @@ -104,6 +104,16 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Chinese/Japanese Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -119,6 +129,16 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Chinese/Japanese Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -133,6 +153,16 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Chinese/Japanese Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -150,6 +180,16 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Chinese/Japanese Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 From 5a5f73ef58f2b54463dea28765f13da61be6956c Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 1 Jun 2022 14:00:03 -0700 Subject: [PATCH 3/4] Added ideographic comma --- tools/wikidata/write_wikidata.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tools/wikidata/write_wikidata.py b/tools/wikidata/write_wikidata.py index d9cf7bd8..0cd447f8 100755 --- a/tools/wikidata/write_wikidata.py +++ b/tools/wikidata/write_wikidata.py @@ -109,7 +109,12 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] - # Chinese/Japanese Comma ... + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... if wddic[qid][d].find('，') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] @@ -134,7 +139,12 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] - # Chinese/Japanese Comma ... + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... if wddic[qid][d].find('，') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] @@ -158,7 +168,12 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] - # Chinese/Japanese Comma ... + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... if wddic[qid][d].find('，') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] @@ -185,7 +200,12 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] - # Chinese/Japanese Comma ... + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... if wddic[qid][d].find('，') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] From 5623328af990c34858bba0456d36e9b73bf029ea Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 2 Jun 2022 10:28:55 -0700 Subject: [PATCH 4/4] add arg to allow/disallow overwriting existing rows --- tools/wikidata/write_wikidata.py | 50 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tools/wikidata/write_wikidata.py b/tools/wikidata/write_wikidata.py index 0cd447f8..d47135e4 100755 --- a/tools/wikidata/write_wikidata.py +++ b/tools/wikidata/write_wikidata.py @@ -41,6 +41,10 @@ parser.add_argument('-output_csvsumlog', default='ne_10m_populated_places_latest.sumlog.csv', help='audit sum log about the changes') +parser.add_argument('-overwrite_values', + default=False, + action='store_true', + help='allow overwriting existing data in shapefile') args = parser.parse_args() @@ -312,6 +316,7 @@ def isNotEmpty(s): stat_new_value_new_lang = 0 stat_del = 0 stat_mod = 0 + stat_exists = 0 stat_wikidataid_redirect = 0 stat_wikidataid_notfound = 0 @@ -370,22 +375,38 @@ def isNotEmpty(s): if f['properties'][updatefield] != wddic[qid][updatefield]: - if f['properties'][updatefield] == '': - if updatefield in new_properties: - status = 'NEWvalue_NEW_LANG' - stat_new_value_new_lang += 1 + if args.overwrite_values: + if f['properties'][updatefield] == '': + if updatefield in new_properties: + status = 'NEWvalue_NEW_LANG' + stat_new_value_new_lang += 1 + else: + status = 'NEWvalue' + stat_new += 1 + elif wddic[qid][updatefield] == '': + status = 'DELvalue' + stat_del += 1 else: - status = 'NEWvalue' - stat_new += 1 - elif wddic[qid][updatefield] == '': - status = 'DELvalue' - stat_del += 1 - else: - status = 'MODvalue' - stat_mod += 1 + status = 'MODvalue' + stat_mod += 1 + + writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) + f['properties'][updatefield] = wddic[qid][updatefield] - writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) - f['properties'][updatefield] = wddic[qid][updatefield] + else: + if f['properties'][updatefield] == '': + if updatefield in new_properties: + status = 'NEWvalue_NEW_LANG' + stat_new_value_new_lang += 1 + else: + status = 'NEWvalue' + stat_new += 1 + + writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) + f['properties'][updatefield] = wddic[qid][updatefield] + else: + status = 'EXIvalue' + stat_exists += 1 else: if isNotEmpty(f['properties'][updatefield]): @@ -437,6 +458,7 @@ def isNotEmpty(s): sumWriter.writerow((args.input_shape, 'Modified_name', stat_mod)) sumWriter.writerow((args.input_shape, 'Empty_name', stat_empty)) sumWriter.writerow((args.input_shape, 'Same_name', stat_equal)) + sumWriter.writerow((args.input_shape, 'Name_exists_row_unmodified', stat_exists)) sumWriter.writerow((args.input_shape, 'Wikidataid_redirected', stat_wikidataid_redirect)) sumWriter.writerow((args.input_shape, 'Wikidataid_notfound', stat_wikidataid_notfound)) sumWriter.writerow((args.input_shape, 'Wikidataid_null', stat_wikidataid_null))