diff --git a/tools/wikidata/fetch_wikidata.py b/tools/wikidata/fetch_wikidata.py index e4760a1f..ec455b89 100755 --- a/tools/wikidata/fetch_wikidata.py +++ b/tools/wikidata/fetch_wikidata.py @@ -34,7 +34,7 @@ help='input natural-earth shape file - with wikidataid columns') parser.add_argument('-input_lettercase', default='uppercase', - help='variables in thes hape file - lowercase or uppercase') + help='variables in the shape file - lowercase or uppercase') parser.add_argument('-output_csv_name', default='ne_10m_populated_places.csv', help='output csv file with wikidata labels') @@ -165,18 +165,26 @@ def fetchwikidata(a_wid): ?name_es ?name_fa ?name_fr + ?name_gu ?name_he ?name_hi ?name_hu ?name_id ?name_it ?name_ja + ?name_kn ?name_ko + ?name_ml + ?name_mr ?name_nl + ?name_pa ?name_pl ?name_pt ?name_ru ?name_sv + ?name_ta + ?name_te + ?name_th ?name_tr ?name_uk ?name_ur @@ -203,18 +211,26 @@ def fetchwikidata(a_wid): OPTIONAL{?e rdfs:label ?name_es FILTER((LANG(?name_es))="es").} OPTIONAL{?e rdfs:label ?name_fa FILTER((LANG(?name_fa))="fa").} OPTIONAL{?e rdfs:label ?name_fr FILTER((LANG(?name_fr))="fr").} + OPTIONAL{?e rdfs:label ?name_gu FILTER((LANG(?name_gu))="gu").} OPTIONAL{?e rdfs:label ?name_he FILTER((LANG(?name_he))="he").} OPTIONAL{?e rdfs:label ?name_hi FILTER((LANG(?name_hi))="hi").} OPTIONAL{?e rdfs:label ?name_hu FILTER((LANG(?name_hu))="hu").} OPTIONAL{?e rdfs:label ?name_id FILTER((LANG(?name_id))="id").} OPTIONAL{?e rdfs:label ?name_it FILTER((LANG(?name_it))="it").} OPTIONAL{?e rdfs:label ?name_ja FILTER((LANG(?name_ja))="ja").} + OPTIONAL{?e rdfs:label ?name_kn FILTER((LANG(?name_kn))="kn").} OPTIONAL{?e rdfs:label ?name_ko FILTER((LANG(?name_ko))="ko").} + OPTIONAL{?e rdfs:label ?name_ml FILTER((LANG(?name_ml))="ml").} + OPTIONAL{?e rdfs:label ?name_mr FILTER((LANG(?name_mr))="mr").} OPTIONAL{?e rdfs:label ?name_nl FILTER((LANG(?name_nl))="nl").} + OPTIONAL{?e rdfs:label ?name_pa FILTER((LANG(?name_pa))="pa").} OPTIONAL{?e rdfs:label ?name_pl FILTER((LANG(?name_pl))="pl").} OPTIONAL{?e rdfs:label ?name_pt FILTER((LANG(?name_pt))="pt").} OPTIONAL{?e rdfs:label ?name_ru FILTER((LANG(?name_ru))="ru").} OPTIONAL{?e rdfs:label ?name_sv FILTER((LANG(?name_sv))="sv").} + OPTIONAL{?e rdfs:label ?name_ta FILTER((LANG(?name_ta))="ta").} + OPTIONAL{?e rdfs:label ?name_te FILTER((LANG(?name_te))="te").} + OPTIONAL{?e rdfs:label ?name_th FILTER((LANG(?name_th))="th").} OPTIONAL{?e rdfs:label ?name_tr FILTER((LANG(?name_tr))="tr").} OPTIONAL{?e rdfs:label ?name_uk FILTER((LANG(?name_uk))="uk").} OPTIONAL{?e rdfs:label ?name_ur FILTER((LANG(?name_ur))="ur").} @@ -305,18 +321,26 @@ def fetchwikidata(a_wid): "name_es", "name_fa", "name_fr", + "name_gu", "name_he", "name_hi", "name_hu", "name_id", "name_it", "name_ja", + "name_kn", "name_ko", + "name_ml", + "name_mr", "name_nl", + "name_pa", "name_pl", "name_pt", "name_ru", "name_sv", + "name_ta", + "name_te", + "name_th", "name_tr", "name_uk", "name_ur", @@ -372,19 +396,27 @@ def fetchwikidata(a_wid): name_es = get_sparql_label(result, 'name_es') name_fa = get_sparql_label(result, 'name_fa') name_fr = get_sparql_label(result, 'name_fr') + name_gu = get_sparql_label(result, 'name_gu') name_he = get_sparql_label(result, 'name_he') name_hi = get_sparql_label(result, 'name_hi') name_hu = get_sparql_label(result, 'name_hu') name_id = get_sparql_label(result, 'name_id') name_it = get_sparql_label(result, 'name_it') name_ja = get_sparql_label(result, 'name_ja') + name_kn = get_sparql_label(result, 'name_kn') name_ko = get_sparql_label(result, 'name_ko') name_lt = get_sparql_label(result, 'name_lt') + name_ml = get_sparql_label(result, 'name_ml') + name_mr = get_sparql_label(result, 'name_mr') name_nl = get_sparql_label(result, 'name_nl') + name_pa = get_sparql_label(result, 'name_pa') name_pl = get_sparql_label(result, 'name_pl') name_pt = get_sparql_label(result, 'name_pt') name_ru = get_sparql_label(result, 'name_ru') name_sv = get_sparql_label(result, 'name_sv') + name_ta = get_sparql_label(result, 'name_ta') + name_te = get_sparql_label(result, 'name_te') + name_th = get_sparql_label(result, 'name_th') name_tr = get_sparql_label(result, 'name_tr') name_uk = get_sparql_label(result, 'name_uk') name_ur = get_sparql_label(result, 'name_ur') @@ -441,18 +473,26 @@ def fetchwikidata(a_wid): name_es, name_fa, name_fr, + name_gu, name_he, name_hi, name_hu, name_id, name_it, name_ja, + name_kn, name_ko, + name_ml, + name_mr, name_nl, + name_pa, name_pl, name_pt, name_ru, name_sv, + name_ta, + name_te, + name_th, name_tr, name_uk, name_ur, @@ -461,4 +501,4 @@ def fetchwikidata(a_wid): name_zht )) -print(' - JOB end -') \ No newline at end of file +print(' - JOB end -') diff --git a/tools/wikidata/write_wikidata.py b/tools/wikidata/write_wikidata.py index 62e5a378..d47135e4 100755 --- a/tools/wikidata/write_wikidata.py +++ b/tools/wikidata/write_wikidata.py @@ -41,6 +41,10 @@ parser.add_argument('-output_csvsumlog', default='ne_10m_populated_places_latest.sumlog.csv', help='audit sum log about the changes') +parser.add_argument('-overwrite_values', + default=False, + action='store_true', + help='allow overwriting existing data in shapefile') args = parser.parse_args() @@ -77,7 +81,7 @@ def isNotEmpty(s): wdredirects = defaultdict(dict) name_field_prefix = 'name_' -languages = ['ar','bn','de','el','en','es','fa','fr','he','hi','hu','id','it','ja','ko','nl','pl','pt','ru','sv','tr','uk','ur','vi','zh','zht'] +languages = ['ar','bn','de','el','en','es','fa','fr','gu','he','hi','hu','id','it','ja','kn','ko','ml','mr','nl','pa','pl','pt','ru','sv','ta','te','th','tr','uk','ur','vi','zh','zht'] new_properties = [] with open(args.input_csv, newline='') as csvfile: @@ -104,6 +108,21 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -119,6 +138,21 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -133,6 +167,21 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -150,6 +199,21 @@ def isNotEmpty(s): # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] + # Arabic Comma ... + if wddic[qid][d].find('،') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')] + + # Ideographic Comma ... + if wddic[qid][d].find('、') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')] + + # Full Width Comma ... + if wddic[qid][d].find('，') > 0: + # RTL languages + wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')] + # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 @@ -252,6 +316,7 @@ def isNotEmpty(s): stat_new_value_new_lang = 0 stat_del = 0 stat_mod = 0 + stat_exists = 0 stat_wikidataid_redirect = 0 stat_wikidataid_notfound = 0 @@ -310,22 +375,38 @@ def isNotEmpty(s): if f['properties'][updatefield] != wddic[qid][updatefield]: - if f['properties'][updatefield] == '': - if updatefield in new_properties: - status = 'NEWvalue_NEW_LANG' - stat_new_value_new_lang += 1 + if args.overwrite_values: + if f['properties'][updatefield] == '': + if updatefield in new_properties: + status = 'NEWvalue_NEW_LANG' + stat_new_value_new_lang += 1 + else: + status = 'NEWvalue' + stat_new += 1 + elif wddic[qid][updatefield] == '': + status = 'DELvalue' + stat_del += 1 else: - status = 'NEWvalue' - stat_new += 1 - elif wddic[qid][updatefield] == '': - status = 'DELvalue' - stat_del += 1 - else: - status = 'MODvalue' - stat_mod += 1 + status = 'MODvalue' + stat_mod += 1 - writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) - f['properties'][updatefield] = wddic[qid][updatefield] + writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) + f['properties'][updatefield] = wddic[qid][updatefield] + + else: + if f['properties'][updatefield] == '': + if updatefield in new_properties: + status = 'NEWvalue_NEW_LANG' + stat_new_value_new_lang += 1 + else: + status = 'NEWvalue' + stat_new += 1 + + writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) + f['properties'][updatefield] = wddic[qid][updatefield] + else: + status = 'EXIvalue' + stat_exists += 1 else: if isNotEmpty(f['properties'][updatefield]): @@ -377,6 +458,7 @@ def isNotEmpty(s): sumWriter.writerow((args.input_shape, 'Modified_name', stat_mod)) sumWriter.writerow((args.input_shape, 'Empty_name', stat_empty)) sumWriter.writerow((args.input_shape, 'Same_name', stat_equal)) + sumWriter.writerow((args.input_shape, 'Name_exists_row_unmodified', stat_exists)) sumWriter.writerow((args.input_shape, 'Wikidataid_redirected', stat_wikidataid_redirect)) sumWriter.writerow((args.input_shape, 'Wikidataid_notfound', stat_wikidataid_notfound)) sumWriter.writerow((args.input_shape, 'Wikidataid_null', stat_wikidataid_null))