Skip to content

WIP: Add new Indic languages #740

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nvkelso wants to merge 4 commits into
base: master
Choose a base branch
from
Open

WIP: Add new Indic languages #740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4b97772
Add new Indic languages
nvkelso May 26, 2022
4d744b8
alphabetized languages in fetch_wikidata.py
May 31, 2022
5a5f73e
Added ideographic comma
Jun 1, 2022
5623328
add arg to allow/disallow overwriting existing rows
Jun 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions tools/wikidata/fetch_wikidata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
help='input natural-earth shape file - with wikidataid columns')
parser.add_argument('-input_lettercase',
default='uppercase',
help='variables in thes hape file - lowercase or uppercase')
help='variables in the shape file - lowercase or uppercase')
parser.add_argument('-output_csv_name',
default='ne_10m_populated_places.csv',
help='output csv file with wikidata labels')
Expand Down Expand Up @@ -165,18 +165,26 @@ def fetchwikidata(a_wid):
?name_es
?name_fa
?name_fr
?name_gu
?name_he
?name_hi
?name_hu
?name_id
?name_it
?name_ja
?name_kn
?name_ko
?name_ml
?name_mr
?name_nl
?name_pa
?name_pl
?name_pt
?name_ru
?name_sv
?name_ta
?name_te
?name_th
?name_tr
?name_uk
?name_ur
Expand All @@ -203,18 +211,26 @@ def fetchwikidata(a_wid):
OPTIONAL{?e rdfs:label ?name_es FILTER((LANG(?name_es))="es").}
OPTIONAL{?e rdfs:label ?name_fa FILTER((LANG(?name_fa))="fa").}
OPTIONAL{?e rdfs:label ?name_fr FILTER((LANG(?name_fr))="fr").}
OPTIONAL{?e rdfs:label ?name_gu FILTER((LANG(?name_gu))="gu").}
OPTIONAL{?e rdfs:label ?name_he FILTER((LANG(?name_he))="he").}
OPTIONAL{?e rdfs:label ?name_hi FILTER((LANG(?name_hi))="hi").}
OPTIONAL{?e rdfs:label ?name_hu FILTER((LANG(?name_hu))="hu").}
OPTIONAL{?e rdfs:label ?name_id FILTER((LANG(?name_id))="id").}
OPTIONAL{?e rdfs:label ?name_it FILTER((LANG(?name_it))="it").}
OPTIONAL{?e rdfs:label ?name_ja FILTER((LANG(?name_ja))="ja").}
OPTIONAL{?e rdfs:label ?name_kn FILTER((LANG(?name_kn))="kn").}
OPTIONAL{?e rdfs:label ?name_ko FILTER((LANG(?name_ko))="ko").}
OPTIONAL{?e rdfs:label ?name_ml FILTER((LANG(?name_ml))="ml").}
OPTIONAL{?e rdfs:label ?name_mr FILTER((LANG(?name_mr))="mr").}
OPTIONAL{?e rdfs:label ?name_nl FILTER((LANG(?name_nl))="nl").}
OPTIONAL{?e rdfs:label ?name_pa FILTER((LANG(?name_pa))="pa").}
OPTIONAL{?e rdfs:label ?name_pl FILTER((LANG(?name_pl))="pl").}
OPTIONAL{?e rdfs:label ?name_pt FILTER((LANG(?name_pt))="pt").}
OPTIONAL{?e rdfs:label ?name_ru FILTER((LANG(?name_ru))="ru").}
OPTIONAL{?e rdfs:label ?name_sv FILTER((LANG(?name_sv))="sv").}
OPTIONAL{?e rdfs:label ?name_ta FILTER((LANG(?name_ta))="ta").}
OPTIONAL{?e rdfs:label ?name_te FILTER((LANG(?name_te))="te").}
OPTIONAL{?e rdfs:label ?name_th FILTER((LANG(?name_th))="th").}
OPTIONAL{?e rdfs:label ?name_tr FILTER((LANG(?name_tr))="tr").}
OPTIONAL{?e rdfs:label ?name_uk FILTER((LANG(?name_uk))="uk").}
OPTIONAL{?e rdfs:label ?name_ur FILTER((LANG(?name_ur))="ur").}
Expand Down Expand Up @@ -305,18 +321,26 @@ def fetchwikidata(a_wid):
"name_es",
"name_fa",
"name_fr",
"name_gu",
"name_he",
"name_hi",
"name_hu",
"name_id",
"name_it",
"name_ja",
"name_kn",
"name_ko",
"name_ml",
"name_mr",
"name_nl",
"name_pa",
"name_pl",
"name_pt",
"name_ru",
"name_sv",
"name_ta",
"name_te",
"name_th",
"name_tr",
"name_uk",
"name_ur",
Expand Down Expand Up @@ -372,19 +396,27 @@ def fetchwikidata(a_wid):
name_es = get_sparql_label(result, 'name_es')
name_fa = get_sparql_label(result, 'name_fa')
name_fr = get_sparql_label(result, 'name_fr')
name_gu = get_sparql_label(result, 'name_gu')
name_he = get_sparql_label(result, 'name_he')
name_hi = get_sparql_label(result, 'name_hi')
name_hu = get_sparql_label(result, 'name_hu')
name_id = get_sparql_label(result, 'name_id')
name_it = get_sparql_label(result, 'name_it')
name_ja = get_sparql_label(result, 'name_ja')
name_kn = get_sparql_label(result, 'name_kn')
name_ko = get_sparql_label(result, 'name_ko')
name_lt = get_sparql_label(result, 'name_lt')
name_ml = get_sparql_label(result, 'name_ml')
name_mr = get_sparql_label(result, 'name_mr')
name_nl = get_sparql_label(result, 'name_nl')
name_pa = get_sparql_label(result, 'name_pa')
name_pl = get_sparql_label(result, 'name_pl')
name_pt = get_sparql_label(result, 'name_pt')
name_ru = get_sparql_label(result, 'name_ru')
name_sv = get_sparql_label(result, 'name_sv')
name_ta = get_sparql_label(result, 'name_ta')
name_te = get_sparql_label(result, 'name_te')
name_th = get_sparql_label(result, 'name_th')
name_tr = get_sparql_label(result, 'name_tr')
name_uk = get_sparql_label(result, 'name_uk')
name_ur = get_sparql_label(result, 'name_ur')
Expand Down Expand Up @@ -441,18 +473,26 @@ def fetchwikidata(a_wid):
name_es,
name_fa,
name_fr,
name_gu,
name_he,
name_hi,
name_hu,
name_id,
name_it,
name_ja,
name_kn,
name_ko,
name_ml,
name_mr,
name_nl,
name_pa,
name_pl,
name_pt,
name_ru,
name_sv,
name_ta,
name_te,
name_th,
name_tr,
name_uk,
name_ur,
Expand All @@ -461,4 +501,4 @@ def fetchwikidata(a_wid):
name_zht
))

print(' - JOB end -')
print(' - JOB end -')
112 changes: 97 additions & 15 deletions tools/wikidata/write_wikidata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@
parser.add_argument('-output_csvsumlog',
default='ne_10m_populated_places_latest.sumlog.csv',
help='audit sum log about the changes')
parser.add_argument('-overwrite_values',
default=False,
action='store_true',
help='allow overwriting existing data in shapefile')

args = parser.parse_args()

Expand Down Expand Up @@ -77,7 +81,7 @@ def isNotEmpty(s):
wdredirects = defaultdict(dict)

name_field_prefix = 'name_'
languages = ['ar','bn','de','el','en','es','fa','fr','he','hi','hu','id','it','ja','ko','nl','pl','pt','ru','sv','tr','uk','ur','vi','zh','zht']
languages = ['ar','bn','de','el','en','es','fa','fr','gu','he','hi','hu','id','it','ja','kn','ko','ml','mr','nl','pa','pl','pt','ru','sv','ta','te','th','tr','uk','ur','vi','zh','zht']
new_properties = []

with open(args.input_csv, newline='') as csvfile:
Expand All @@ -104,6 +108,21 @@ def isNotEmpty(s):
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')]

# Arabic Comma ...
if wddic[qid][d].find('،') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')]

# Ideographic Comma ...
if wddic[qid][d].find('、') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')]

# Full Width Comma ...
if wddic[qid][d].find('，') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')]

# Parenthetical ...
if wddic[qid][d].find('(') > 0:
# RTL languages and LTR figure each other out in python 3
Expand All @@ -119,6 +138,21 @@ def isNotEmpty(s):
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')]

# Arabic Comma ...
if wddic[qid][d].find('،') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')]

# Ideographic Comma ...
if wddic[qid][d].find('、') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')]

# Full Width Comma ...
if wddic[qid][d].find('，') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')]

# Parenthetical ...
if wddic[qid][d].find('(') > 0:
# RTL languages and LTR figure each other out in python 3
Expand All @@ -133,6 +167,21 @@ def isNotEmpty(s):
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')]

# Arabic Comma ...
if wddic[qid][d].find('،') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')]

# Ideographic Comma ...
if wddic[qid][d].find('、') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')]

# Full Width Comma ...
if wddic[qid][d].find('，') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')]

# Parenthetical ...
if wddic[qid][d].find('(') > 0:
# RTL languages and LTR figure each other out in python 3
Expand All @@ -150,6 +199,21 @@ def isNotEmpty(s):
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')]

# Arabic Comma ...
if wddic[qid][d].find('،') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('،')]

# Ideographic Comma ...
if wddic[qid][d].find('、') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('、')]

# Full Width Comma ...
if wddic[qid][d].find('，') > 0:
# RTL languages
wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('，')]

# Parenthetical ...
if wddic[qid][d].find('(') > 0:
# RTL languages and LTR figure each other out in python 3
Expand Down Expand Up @@ -252,6 +316,7 @@ def isNotEmpty(s):
stat_new_value_new_lang = 0
stat_del = 0
stat_mod = 0
stat_exists = 0

stat_wikidataid_redirect = 0
stat_wikidataid_notfound = 0
Expand Down Expand Up @@ -310,22 +375,38 @@ def isNotEmpty(s):

if f['properties'][updatefield] != wddic[qid][updatefield]:

if f['properties'][updatefield] == '':
if updatefield in new_properties:
status = 'NEWvalue_NEW_LANG'
stat_new_value_new_lang += 1
if args.overwrite_values:
if f['properties'][updatefield] == '':
if updatefield in new_properties:
status = 'NEWvalue_NEW_LANG'
stat_new_value_new_lang += 1
else:
status = 'NEWvalue'
stat_new += 1
elif wddic[qid][updatefield] == '':
status = 'DELvalue'
stat_del += 1
else:
status = 'NEWvalue'
stat_new += 1
elif wddic[qid][updatefield] == '':
status = 'DELvalue'
stat_del += 1
else:
status = 'MODvalue'
stat_mod += 1
status = 'MODvalue'
stat_mod += 1

writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield]))
f['properties'][updatefield] = wddic[qid][updatefield]
writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield]))
f['properties'][updatefield] = wddic[qid][updatefield]

else:
if f['properties'][updatefield] == '':
if updatefield in new_properties:
status = 'NEWvalue_NEW_LANG'
stat_new_value_new_lang += 1
else:
status = 'NEWvalue'
stat_new += 1

writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield]))
f['properties'][updatefield] = wddic[qid][updatefield]
else:
status = 'EXIvalue'
stat_exists += 1

else:
if isNotEmpty(f['properties'][updatefield]):
Expand Down Expand Up @@ -377,6 +458,7 @@ def isNotEmpty(s):
sumWriter.writerow((args.input_shape, 'Modified_name', stat_mod))
sumWriter.writerow((args.input_shape, 'Empty_name', stat_empty))
sumWriter.writerow((args.input_shape, 'Same_name', stat_equal))
sumWriter.writerow((args.input_shape, 'Name_exists_row_unmodified', stat_exists))
sumWriter.writerow((args.input_shape, 'Wikidataid_redirected', stat_wikidataid_redirect))
sumWriter.writerow((args.input_shape, 'Wikidataid_notfound', stat_wikidataid_notfound))
sumWriter.writerow((args.input_shape, 'Wikidataid_null', stat_wikidataid_null))
Expand Down