From 1624240987e0df0630aa22ed5d77db924fbc379f Mon Sep 17 00:00:00 2001 From: Anna Shvets Date: Thu, 25 Nov 2021 16:04:38 +0100 Subject: [PATCH 1/3] Updating Universal bias filter --- filters/universal_bias/README.md | 5 ++- filters/universal_bias/filter.py | 23 ++++++----- filters/universal_bias/lexicals.json | 24 +++++------ filters/universal_bias/test.json | 62 ++++++++++++++++------------ 4 files changed, 63 insertions(+), 51 deletions(-) diff --git a/filters/universal_bias/README.md b/filters/universal_bias/README.md index d7e46be7b..cdfeee8ea 100644 --- a/filters/universal_bias/README.md +++ b/filters/universal_bias/README.md @@ -2,8 +2,9 @@ ## What type of a filter is this? -This filter is currently contains lexical seeds for 10 categories () in English, however it can be extended to any language or topic by simple addition of desired entries to `lexicals.json` -file in current directoryalong with the text corpus in corresponding language. +This filter is currently contains lexical seeds for 10 categories (religion, race, ethnicity, gender, sexual orientation, age, appearance, disability, experience, education, economic status). +The lexical seeds representing these categories are currently available in English only, however the pool of languages can be extended by a simple addition of the lexical seeds in a desired language to the lexicals.json file. + The minority parameter is a potentially underrepresented group, defined with its own set of keywords; the majority parameter is a set of keywords, representing the dominating group. The filter returns "True" if the minority group is indeed underrepresented, "False" otherwise. diff --git a/filters/universal_bias/filter.py b/filters/universal_bias/filter.py index b7b1836a6..2ab3de3f5 100644 --- a/filters/universal_bias/filter.py +++ b/filters/universal_bias/filter.py @@ -1,17 +1,16 @@ from interfaces.SentenceOperation import SentenceOperation from tasks.TaskTypes import TaskType import re +import json class UniversalBiasFilter(SentenceOperation): tasks = [TaskType.TEXT_TO_TEXT_GENERATION] keywords = ["rule-based", "social-reasoning"] - def __init__(self, language=None, category=None, minority_group=None, majority_group=None, minority=None, majority=None): + def __init__(self, language=None, category=None, minority=None, majority=None): super().__init__() self.language = language - self.category = category - self.minority_group = minority_group - self.majority_group = majority_group + self.category = category self.minority = minority self.majority = majority @@ -31,8 +30,9 @@ def flag_sentences(self, sentences): # Retrieve relevant data extracts try: - minority_group = data[self.language][self.category][self.minority_group] - majority_group = data[self.language][self.category][self.majority_group] + minority_group = data[self.language][self.category][self.minority] + majority_group = data[self.language][self.category][self.majority] + except NameError as error: print('The specified language, category of group is not supported or misformatted. Please provide valid arguments to the filter() method.') @@ -49,7 +49,7 @@ def flag_sentences(self, sentences): sentence_cleaned = sentence.lower() sentence_cleaned = re.sub('^',' ', sentence_cleaned) sentence_cleaned = re.sub('$',' ', sentence_cleaned) - + # Take care of urls words = [] for word in sentence_cleaned.split(): @@ -59,7 +59,7 @@ def flag_sentences(self, sentences): words.append(word.strip()) sentence_cleaned = ' '.join(words) sentence_cleaned = re.sub(r'\[([^\]]*)\] \( *__url__ *\)', r'\1', sentence_cleaned) - + # Remove illegal chars and extra space sentence_cleaned = re.sub('__url__','URL', sentence_cleaned) sentence_cleaned = re.sub(r"[^A-Za-z0-9():,.!?\"\']", " ", sentence_cleaned) @@ -70,13 +70,13 @@ def flag_sentences(self, sentences): # Split the words in the sentence to find the intersection with the minority array of keywords intersection_minority = set(sentence_cleaned.split()).intersection( - set(minority_group + minority) + set(minority_group) ) # Split the words in the sentence to find the intersection with the majority array of keywords intersection_majority = set(sentence_cleaned.split()).intersection( - set(majority_group + majority) + set(majority_group) ) - + # If the intersection occurred, the intersection_minority and intersection_majority will contain at least one common keyword # use this intersection information to get the value for the corresponding flags minority_flag = len(intersection_minority) > 0 @@ -105,6 +105,7 @@ def flag_sentences(self, sentences): "majority_flag": majority_flag, "neutral_flag": neutral_flag, } + print("sentence_object", sentence_object) # Append the object to the array we return flagged_sentences.append(sentence_object) diff --git a/filters/universal_bias/lexicals.json b/filters/universal_bias/lexicals.json index 1f255f49f..31a5ed302 100644 --- a/filters/universal_bias/lexicals.json +++ b/filters/universal_bias/lexicals.json @@ -47,7 +47,7 @@ }, "disability": { - "healthy": ["healthy"], + "healthy": ["healthy", "fit", "sound", "healthful", "buxom", "bonny", "sanitary", "salubrious", "healthful", "wholesome", "salutary"], "disabled": ["disabled", "impaired"] }, @@ -57,12 +57,12 @@ }, "education": { - "educated": ["educated"], - "uneducated": ["uneducated", "unlearned"] + "educated": ["educated", "cultivated", "cultured", "literate", "refined", "scholarly", "taught", "instructed", "trained"], + "uneducated": ["uneducated", "unlearned", "untrained", "uncultivated", "untilled", "uncivil"] }, "economic_status": { - "rich": ["rich", "wealth", "wealthy", "affluent", "abundant", "copious"], + "rich": ["rich", "wealth", "wealthy", "affluent", "abundant", "abundance", "copious"], "poor": ["poor", "pauvre", "poverty", "broke", "homeless", "unhoused", "roofless", "unsheltered", "vagabond", "hipster", "hobo"] } }, @@ -115,23 +115,23 @@ }, "disability": { - "healthy": [], - "disabled": [] + "healthy": ["sain", "saine", "sains", "saines", "salubre", "salubres", "sanitaire", "sanitaires", "salutaire", "salutaires"], + "disabled": ["handicapé", "handicapée", "handicapés", "handicapées", "infirme", "infirmes", "mutilé", "mutilée", "mutilés", "mutilées", "impotent", "impotente", "impotents", "impotentes", "invalide", "invalides"] }, "experience": { - "experienced": [], - "inexperienced" : [] + "experienced": ["expérimenté", "expert", "avancé", "compétent", "familier", "senior"], + "inexperienced" : ["inexpérimenté", "débutant", "junior"] }, "education": { - "educated": [], - "uneducated": [] + "educated": ["éduqué", "éduquée", "éduqués", "éduquées", "cultivé", "cultivée", "cultivés", "cultivées", "enseigné", "enseignée", "enseignés", "enseignées", "alphabétisé", "alphabétisée", "alphabétisés", "alphabétisées", "raffiné", "raffinée", "raffinés", "raffinées", "érudit", "érudite", "érudits", "érudites", "instruit", "instruite", "instruits", "instruites", "formé", "formée", "formés", "formées"], + "uneducated": ["inculte"] }, "economic_status": { - "rich": [], - "poor": [] + "rich": ["riche", "riches", "abondant", "abondante", "abondantes"], + "poor": ["pauvre", "pauvreté", "cassé", "sans-abri", "sans logement", "sans toit", "vagabond", "hipster", "clochard"] } } } \ No newline at end of file diff --git a/filters/universal_bias/test.json b/filters/universal_bias/test.json index ede3be152..021aa94d1 100644 --- a/filters/universal_bias/test.json +++ b/filters/universal_bias/test.json @@ -4,12 +4,14 @@ { "class": "UniversalBiasFilter", "args": { - "minority": ["she", "her", "hers"], - "majority": ["he", "him", "his"] + "language": "en", + "category": "gender", + "minority": "female", + "majority": "majority" }, "inputs": { - "sentences": ["He is going to make a cake.", - "She is going to program", + "sentences": ["He went home", + "She is going to work", "Nobody likes washing dishes", "He agreed to help him"] }, @@ -18,12 +20,14 @@ { "class": "UniversalBiasFilter", "args": { - "minority": ["she", "her", "hers"], - "majority": ["he", "him", "his"] + "language": "en", + "category": "gender", + "minority": "female", + "majority": "majority" }, "inputs": { - "sentences": ["He is going to make a cake.", - "She is going to program", + "sentences": ["He went home", + "She is going to work", "Nobody likes washing dishes", "She agreed to help him"] }, @@ -32,42 +36,48 @@ { "class": "UniversalBiasFilter", "args": { - "minority": ["elle", "madame", "maman"], - "majority": ["il", "monsieur", "papa"] + "language": "fr", + "category": "gender", + "minority": "female", + "majority": "majority" }, "inputs": { - "sentences": ["Il va preparer un gateau", - "Elle va créer un logiciel", + "sentences": ["Il va à la maison", + "Elle va au travail", "Personne n'aime pas laver la vaiselle", - "Maman va à la conférence"] + "Il participe à la conférence"] }, - "outputs": false + "outputs": true }, { "class": "UniversalBiasFilter", "args": { - "minority": ["она", "мадам", "мама"], - "majority": ["он", "сэр", "папа"] + "language": "fr", + "category": "gender", + "minority": "female", + "majority": "majority" }, "inputs": { - "sentences": ["Он собирается испечь торт", - "Она создаст софт", - "Никто не любит мыть посуду", - "Мама идет на конференцию"] + "sentences": ["Il va à la maison", + "Elle va au travail", + "Personne n'aime pas laver la vaiselle", + "Maman participe à la conférence"] }, "outputs": false }, { "class": "UniversalBiasFilter", "args": { - "minority": ["ona", "pani", "mama"], - "majority": ["on", "pan", "ojciec"] + "language": "fr", + "category": "gender", + "minority": "female", + "majority": "majority" }, "inputs": { - "sentences": ["Pan upiecze ciasto", - "Pani stworzy oprogramowanie", - "Nikt nie lubi zmywać naczyń", - "Mama jedzie na konferencję"] + "sentences": ["Il va à la maison", + "Elle va au travail", + "Personne n'aime pas laver la vaiselle", + "Marie participe à la conférence"] }, "outputs": false } From 7fa1b0e7e6b09506cde701fe8ee37fe1684421bc Mon Sep 17 00:00:00 2001 From: Anna Shvets Date: Fri, 3 Dec 2021 16:08:45 +0100 Subject: [PATCH 2/3] Adding complete leist of French lexical seeds for all categories --- filters/universal_bias/lexicals.json | 70 ++++++++++++++-------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/filters/universal_bias/lexicals.json b/filters/universal_bias/lexicals.json index 31a5ed302..08f7a0a3b 100644 --- a/filters/universal_bias/lexicals.json +++ b/filters/universal_bias/lexicals.json @@ -1,11 +1,11 @@ { "en": { "religion": { - "christianism": ["jesus", "god", "godly", "messiah", "apostle", "apostles", "apostolic", "disciple", "disciples", "preach", "preaching", "preached", "preacher", "preachers", "prophet", "prophets", "prophecy", "prophecies", "almighty", "eternal", "resurrection", "blessing", "sanctity", "salvation", "damnation", "excommunication", "excommunicated", "celibate", "celibacy", "divinity", "ascension", "redemption", "crucifiction", "crucified", "apocalypse", "holy", "holiness", "sacred", "spirit", "trinity", "saint", "saints", "angel", "satan", "antichrist", "satanic", "devil", "devils", "demon", "demons", "heaven", "hell", "hellfire", "bible", "biblical", "evangel", "evangelical", "church", "churches", "cathedral", "cathedrals", "chapel", "sanctuary", "monastery", "monastic", "pastor", "pastors", "priest", "priests", "bishop", "bishops", "pope", "cardinal", "cardinals", "clergy", "missionaries", "pilgrimage", "priesthood", "ordination", "ordained", "franciscan", "franciscans", "carthusian", "carthusians", "jesuit", "jesuits", "benedictine", "benedictines", "salesian", "salesians", "missionary", "missionaries", "dominican", "dominicans", "augustinian", "augustinians", "carmelite", "carmelites", "theology", "theologian", "parish", "worship", "prayer", "adoration", "sermon", "sacrament", "communion", "christ", "christianity", "christian", "catholicism", "catholic", "catholics", "orthodox", "protestant", "protestants", "baptist", "baptists", "evangelist", "evangelists", "mormon", "mormons", "gospel", "funeral", "cemetery"], - "buddhism_hinduism_jainism": ["dharmic", "om", "nirvana", "buddha", "buddhist", "buddhism", "jainism", "jain", "hinduism", "hindu", "transtheistic", "enlightenment", "jiva", "atman", "moksha", "arihant", "pramana", "dravya", "tattva", "karma", "reincarnation", "saṃsāra", "ahiṃsā", "anekāntavāda", "aparigraha", "vada", "dharma", "sangha", "svadharma", "sutra", "triratna", "ti-ratana", "dravya", "ratnatraya", "samyagdarshana", "monk", "nun", "guru", "samayika", "meditation", "sallekhana", "deva", "avatar", "abhisheka", "mantra", "temple", "sangha", "loka", "lokas", "diyas", "sky-clad", "sarees"], + "christianism": ["jesus", "godly", "messiah", "apostle", "apostles", "apostolic", "disciple", "disciples", "preach", "preaching", "preached", "preacher", "preachers", "prophet", "prophets", "prophecy", "prophecies", "almighty", "eternal", "resurrection", "blessing", "sanctity", "salvation", "damnation", "excommunication", "excommunicated", "celibate", "celibacy", "divinity", "ascension", "redemption", "crucifiction", "crucified", "apocalypse", "holy", "holiness", "sacred", "trinity", "saint", "saints", "angel", "satan", "antichrist", "satanic", "devil", "devils", "demon", "demons", "heaven", "hell", "hellfire", "bible", "biblical", "evangel", "evangelical", "church", "churches", "cathedral", "cathedrals", "chapel", "chapels", "sanctuary", "sanctuaries", "pastor", "pastors", "priest", "priests", "bishop", "bishops", "pope", "cardinal", "cardinals", "clergy", "missionaries", "pilgrimage", "priesthood", "ordination", "ordained", "franciscan", "franciscans", "carthusian", "carthusians", "jesuit", "jesuits", "benedictine", "benedictines", "salesian", "salesians", "missionary", "missionaries", "dominican", "dominicans", "augustinian", "augustinians", "carmelite", "carmelites", "theology", "theologian", "parish", "worship", "prayer", "adoration", "sermon", "sacrament", "communion", "christ", "christianity", "christian", "catholicism", "catholic", "catholics", "orthodox", "orthodoxy", "protestant", "protestants", "baptist", "baptists", "evangelist", "evangelists", "mormon", "mormons", "gospel", "lithany", "litanies", "funeral", "cemetery"], + "buddhism_hinduism_jainism": ["dharmic", "om", "nirvana", "buddha", "buddhist", "buddhism", "jainism", "jain", "jains", "hinduism", "hindu", "hinduist", "transtheistic", "enlightenment", "jiva", "atman", "moksha", "arihant", "pramana", "dravya", "tattva", "karma", "reincarnation", "saṃsāra", "ahiṃsā", "anekāntavāda", "aparigraha", "vada", "dharma", "sangha", "svadharma", "sutra", "triratna", "ti-ratana", "dravya", "ratnatraya", "samyagdarshana", "guru", "samayika", "meditation", "sallekhana", "deva", "avatar", "abhisheka", "mantra", "temple", "sangha", "loka", "lokas", "diyas", "sky-clad", "sarees"], "confucianism": ["confucianism", "ruism", "confucius", "tiān", "brahman", "lǐ", "dào", "pinyin", "ritual", "rite", "rites", "self-cultivation", "self-creation", "self-identification", "benevolence", "humaneness", "rén", "yì", "lǐ", "zhì", "ru", "rú", "rújiā", "rújiào", "rúxué", "kǒngjiào", "kǒngjiādiàn", "hundun", "qi", "yin", "yang", "wu", "wei"], - "islam": ["islam", "islamic", "allah", "muslim", "muslims", "arab", "arabs", "arabic", "hamas", "gaza", "jihad", "jihadists", "infidels", "sinners", "coran", "quran", "hadith", "imam", "hijab", "burqa", "burka", "turban"], - "judaism": ["judaism", "jewish", "jew", "jews", "hebrew", "torah", "tanakh", "pentateuch", "mishnah", "talmud", "gemara", "jerusalem", "synagogue", "sabbath", "rabbi", "rabbis", "six-pointed", "david", "abraham", "isaac", "moses", "hasidic", "passover", "rosh", "hashanah", "yom", "kippur", "hanukkah", "purim"], + "islam": ["islam", "islamic", "allah", "muslim", "muslims", "arab", "arabs", "arabic", "hamas", "gaza", "jihad", "jihadist", "jihadists", "infidels", "sinners", "coran", "quran", "hadith", "imam", "imams", "mosque", "hijab", "burqa", "burka", "turban", "niqab"], + "judaism": ["judaism", "jewish", "jew", "jews", "hebrew", "torah", "tanakh", "pentateuch", "mishnah", "talmud", "gemara", "jerusalem", "synagogue", "sabbath", "rabbi", "rabbis", "six-pointed", "david", "abraham", "isaac", "moses", "hasidic", "hasidism", "passover", "rosh", "hashanah", "yom", "kippur", "hanukkah", "purim"], "atheism": ["atheism", "atheist"] }, @@ -18,12 +18,12 @@ }, "ethnicity" : { - "european": ["european", "europeans", "american", "americans", "canadian", "canadians", "english", "englishman", "englishmen", "irish", "irishman", "irishmen", "french", "frenchman", "frenchmen", "german", "germans", "italian", "italians", "spanish", "spanishman", "spanishmen", "belgian", "belgians", "swiss", "nordic", "scandinavian", "scandinavians", "jewish", "slavs", "russian", "russians", "belarusian", "belarusians", "ukrainian", "ukrainians", "polish", "czech", "czechs", "slovak", "slovaks", "finn", "finns", "hungarian", "hungarians", "lithuanian", "lithuanians", "bulgarian", "bulgarians", "romanian", "romanians", "serb", "serbs", "croat", "croats", "bosniak", "bosniaks", "herzegovinian", "herzegovinians", "macedonian", "macedonians", "albanian", "albanians", "montenegrin", "montenegrins"], + "european": ["european", "europeans", "american", "americans", "canadian", "canadians", "english", "englishman", "englishmen", "irish", "irishman", "irishmen", "french", "frenchman", "frenchmen", "german", "germans", "italian", "italians", "spanish", "spanishman", "spanishmen", "belgian", "belgians", "swiss", "nordic", "scandinavian", "scandinavians", "jewish", "slavs", "russian", "russians", "belarusian", "belarusians", "ukrainian", "ukrainians", "polish", "czech", "czechs", "slovak", "slovaks", "finn", "finns", "hungarian", "hungarians", "lithuanian", "lithuanians", "bulgarian", "bulgarians", "romanian", "romanians", "serb", "serbs", "slovenian", "slovenians", "croat", "croats", "bosniak", "bosniaks", "herzegovinian", "herzegovinians", "macedonian", "macedonians", "albanian", "albanians", "montenegrin", "montenegrins"], "african": ["african", "africans", "jamaican", "jamaicans", "haitian", "haitians", "nigerian", "nigerians", "ethiopian", "ethiopians", "somali", "ghanaian", "ghanaians", "barbadian", "barbadians", "kenyan", "kenyans", "liberian", "liberians", "bahamian", "bahamians"], - "eurasian": ["eurasian", "eurasians", "turk", "turks", "turkmen", "turkmens", "kazakh", "kazakhs", "mongolian", "mongolians", "caucassian", "afgan", "afgans", "georgian", "georgians"], - "asian": ["asian", "asians", "chinese", "japanese", "korean", "koreans", "vietnamese", "arab", "arabs", "arabic", "filipino", "filipinos", "pakistani", "cambodian", "hmong"], + "eurasian": ["eurasian", "eurasians", "turk", "turks", "turkmen", "turkmens", "kazakh", "kazakhs", "mongolian", "mongolians", "caucassian", "afgan", "afgans", "uzbek", "uzbeks", "karluk", "uyghur", "tadjik", "tadjiks", "kyrgyz", "kyrghyz", "kirgiz", "kirghiz", "georgian", "georgians"], + "asian": ["asian", "asians", "chinese", "japanese", "korean", "koreans", "vietnamese", "arab", "arabs", "arabic", "filipino", "filipinos", "pakistani", "pakistanis", "pakistanian", "cambodian", "hmong"], "hispanic": ["mexican", "mexicans", "puertorican", "puertoricans", "cuban", "cubans", "salvadoran", "salvadorans", "dominican", "dominicans", "spaniard", "colombian", "colombians", "guatemalan", "guatemalans", "ecuadorian", "ecuadorians", "peruvian", "peruvias", "venezuelan", "venezuelans", "honduran", "hondurans"], - "american_indian": ["alaska", "navajo", "blackfeet", "tribe", "mayan", "mayans", "aztec", "aztecs", "barrow", "inupiat", "eskimo", "navajo"] + "american_indian": ["alaska", "navajo", "navaho", "navajos", "navahos", "blackfeet", "tribe", "mayan", "mayans", "mayank", "aztec", "aztecs", "inupiat", "iñupiaq", "inupiaq", "iñupiat", "iñupiatun", "inuit", "eskimo"] }, "gender" : { @@ -42,8 +42,8 @@ }, "appearencence": { - "attractive": ["pretty", "cute", "thin"], - "unattractive": ["ugly", "repellent", "fat"] + "attractive": ["beautiful", "pretty", "cute", "attractive", "thin", "slim"], + "unattractive": ["ugly", "naughty", "nasty", "wicked", "horrid", "repugnant", "disgusting", "repellent", "fat"] }, "disability": { @@ -69,49 +69,49 @@ "fr": { "religion": { - "christianity": [], - "buddhism_hinduism_jainism": [], - "confucianism": [], - "islam": [], - "judaism": [], - "atheism": [] + "christianity": ["jésus", "pieux", "messie", "apôtre", "apôtres", "apostolique", "disciple", "disciples", "prêche", "prêcher", "prédicateur", "prédicateurs", "prophète", "prophètes", "prophétie", "prophéties", "éternel", "résurrection", "bénédiction", "sainteté", "damnation", "excommunication", "excommunié", "célibat", "divinité", "ascension", "rédemption", "crucifiction", "crucifié", "apocalypse", "saint", "sainte", "saints", "saintes", "sainteté", "sacré", "trinité", "ange", "satan", "antéchrist", "satanique", "diable", "diables", "démon", "démons", "paradis", "enfer", "bible", "biblique", "évangile", "évangélique", "église", "églises", "cathédrale", "cathédrales", "chapelle", "chapelles", "sanctuaire", "sanctuaires", "pasteur", "pasteurs", "prêtre", "prêtres", "évêque", "évêques", "pape", "cardinal", "cardinaux", "clergé", "missionnaires", "pèlerinage", "sacerdoce", "ordination", "franciscain", "franciscains", "charteux", "jésuite", "jésuites", "bénédictin", "bénédictins", "salésien", "salésiens", "missionnaire", "missionnaires", "dominicain", "dominicains", "augustinien", "augustiniens", "carmélite", "carmélites", "théologie", "théologien ", "paroisse", "prière", "adoration", "sermon", "sacrement", "communion", "christ", "christianisme", "chrétien", "catholicisme", "catholique", "catholiques", "orthodoxe", "orthodoxes", "orthodoxie", "protestant", "protestante", "protestants", "protestantes", "baptiste", "baptistes", "évangéliste", "évangélistes", "mormon", "mormone", "mormons", "mormones", "évangile", "litanie", "litanies", "enterrement", "cimetière"], + "buddhism_hinduism_jainism": ["dharmique", "om", "nirvana", "bouddha", "bouddhiste", "bouddhistes", "bouddhique", "bouddhisme", "jaïnisme", "jinisme", "jaïn", "jaïns", "hindouisme", "sikhisme", "hindou", "hindoue", "hindous", "hindouiste", "hindouistes", "transthéiste", "illumination", "jiva", "atman", "moksha", "arihant", "pramana", "dravya", "tattva", "karma", "réincarnation", "saṃsāra", "ahiṃsā", "anekāntavāda", "aparigraha", "vada", "dharma", "sangha", "svadharma", "sutra", "triratna", "ti-ratana", "dravya", "ratnatraya", "samyagdarshana", "gourou", "samayika", "méditation", "sallekhana", "deva", "avatar", "abhisheka", "mantra", "temple", "sangha", "loka", "lokas", "diyas", "sari"], + "confucianism": ["confucianisme", "ruisme", "confucius", "tiān", "brahman", "lǐ", "dào", "pinyin", "rituel", "rituels", "rite", "rites", "auto-culture", "auto-création", "auto-identification", "bienveillance", "humanité", "rén", "yì", "lǐ", "zhì", "ru", "rú", "rújiā", "rújiào", "rúxué", "kǒngjiào", "kǒngjiādiàn", "hundun", "qi", "yin", "yang", "wu", "wei"], + "islam": ["islam", "islamique", "allah", "musulman", "musulmane", "musulmans", "musulmanes", "arabe", "arabes", "hamas", "gaza", "jihad", "jihadiste", "jihadistes", "djihad", "djihadiste", "djihadistes", "infidèles", "pécheurs", "coran", "hadith", "hadiths", "imam", "imams", "mosquée", "burqa", "burka", "turban", "niqab"], + "judaism": ["judaïsme", "juif", "juive", "juifs", "juives", "hébreu", "hébreue", "hébreux", "torah", "tanakh", "pentateuque", "mishnah", "talmud", "gemara", "jérusalem", "synagogue", "sabbat", "rabbin", "rabbins", "david", "abraham", "isaac", "mose", "hassidisme", "hassidique", "roch", "rosh", "pesah", "hachana", "lachanim", "yom", "kippour", "hanoucca", "hanoukka", "pourim"], + "atheism": ["athéisme", "athée", "athées"] }, "race" : { - "white": [], - "black": [], - "asian": [], - "latino": [], - "american_indian": [] + "white": ["blanc", "blanche", "blancs", "blanches"], + "black": ["noir", "noire", "noirs", "noires"], + "asian": ["asiatique", "asiatiques"], + "latino": ["latino", "latine"], + "american_indian": ["indien", "indienne", "indiens", "indiennes"] }, "ethnicity" : { - "european": [], - "african": [], - "eurasian": [], - "asian": [], - "hispanic": [], - "american_indian": [] + "european": ["européen", "européenne", "européens", "européennes", "américain", "américaine", "américains", "américaines", "canadien", "canadienne", "canadiens", "canadiennes", "anglais", "anglaise", "anglaises", "irlandais", "irlandaise", "irlandaises", "français", "française", "françaises", "allemand", "allemande", "allemands", "allemandes", "italien", "italienne", "italiens", "italiennes", "espagnol", "espagnole", "espagnols", "espagnoles", "belge", "belges", "suisse", "suisses", "nordique", "scandinave", "scandinaves", "juif", "juive", "juifs", "juives", "slave", "slaves", "balkanique", "russe", "russes", "biélorusse", "biélorusses", "ukrainien", "ukrainienne", "ukrainiens", "ukrainiennes", "polonais", "polonaise", "polonaises", "tchèque", "tchèques", "slovaque", "slovaques", "finlandais", "hongrois", "hongroise", "hongroises", "lituanien", "lituanienne", "lituaniens", "lituaniennes", "bulgare", "bulgares", "roumain", "roumainne", "roumains", "roumainnes", "serbe", "serbes", "croate", "croates", "slovène", "slovènes", "bosniaque", "bosniaques", "herzégovinien", "herzégoviniens", "macédonien", "macédonienne", "macédoniens", "macédoniennes", "albanais", "albanaise", "albanaises", "montenegrin", "montenegrin"], + "african": ["africain", "africaine", "africains", "africaines", "jamaïquain", "jamaïquaine", "jamaïquains", "jamaïquaines", "haïtien", "haïtienne", "haïtiens", "haïtiennes", "nigérian", "nigériane", "nigérians", "nigérianes", "éthiopien", "éthiopienne", "éthiopiens", "éthiopiennes", "somali", "ghanéen", "ghanéenne", "ghanéens", "ghanéennes", "barbade", "barbades", "kenyan", "kenyane", "kenyans", "kenyanes", "libérien", "libérienne", "libériens", "libériennes", "bahaméen", "bahaméenne", "bahaméens", "bahaméennes"], + "eurasian": ["eurasien", "eurasienne", "eurasiens", "eurasiennes", "turc", "turcs", "turque", "turques", "turciques", "turkmène", "turkmènes", "turcomans", "kazakh", "kazakhe", "kazakhs", "kazakhes", "mongol", "mongole", "mongols", "mongoles", "caucasien", "caucasienne", "caucasiens", "caucasiennes", "afghan", "afghane", "afghans", "afghanes", "afghani", "ouzbek", "ouzbèke", "ouzbeks", "ouzbèkes", "karlouk", "kiptchak", "oghouz", "tadjik", "tadjiks", "kirghiz", "kirghize", "kirghizes", "géorgien", "géorgienne", "géorgiens", "géorgiennes"], + "asian": ["asiatique", "asiatiques", "chinois", "chinoise", "chinoises", "japonais", "japonaise", "japonaises", "coréen", "coréenne", "coréens", "coréennes", "vietnamien", "vietnamienne", "vietnamiens", "vietnamiennes", "arabe", "arabes", "philippin", "philippines", "pakistanais", "pakistanaise", "pakistanaises", "cambodgien", "cambodgienne", "cambodgiens", "cambodgiennes", "hmong"], + "hispanic": ["mexicain", "mexicaine", "mexicains", "mexicaines", "portoricain", "portoricaine", "portoricains", "portoricaines", "cubain", "cubaine", "cubains", "cubaines", "salvadorien", "salvadorienne", "salvadoriens", "salvadoriennes", "dominicain", "dominicaine", "dominicains", "dominicaines", "colombien", "colombienne", "colombiens", "colombiennes", "guatémaltèque", "guatémaltèques", "équatorien", "équatorienne", "équatoriens", "équatoriennes", "péruvien", "péruvienne", "péruviens", "péruviennes", "vénézuélien", "vénézuélienne", "vénézuéliens", "vénézuéliennes", "hondurien", "hondurienne", "honduriens", "honduriennes"], + "american_indian": ["alaska", "navajo", "navaho", "navajos", "navahos", "amérindien", "amérindiens", "tribu", "maya", "mayas", "aztèque", "aztèques", "inupiaq", "inupiak", "inupiat", "iñupiaq", "yupik", "unangax", "eskaléoutes", "inuites", "aléoutes", "esquimau", "esquimaude", "esquimaux", "esquimaudes"] }, "gender" : { - "male": [], - "female": [] + "male": ["mr", "monsieur", "monseigneur", "baron", "compte", "marquis", "duc", "prince", "dauphin", "empereur", "roi", "homme", "mec", "oncle", "neveu", "garçon", "fils", "gars", "gamin", "ami", "pot", "maître", "frère", "amant", "époux", "mari", "marié", "père", "papa", "daron", "grand-père", "papie", "hommes", "mecs", "oncles", "neveux", "garçons", "fils", "gars", "gamins", "amis", "pots", "maîtres", "frères", "amants", "époux", "maris", "mariés", "pères", "papas", "darons", "grand-pères", "papies", "acteur", "comédien", "héros", "amateur", "réalisateur", "producteur", "opérateur", "danseur", "chanteur", "musicien", "animateur", "étudiant", "enseignant", "chercheur", "innovateur", "propriétaire", "directeur", "dirigeant", "conducteur", "programmeur", "informaticien", "ingénieur", "techicien", "aiguilleur", "moine", "prêtre", "steward", "serveur", "servant", "cuisinier", "boulanger", "policier", "gardien", "infermier", "pharmacien", "acheteur", "vendeur", "éleveur", "agriculteur", "acteurs", "comédiens", "héros", "amateurs", "réalisateurs", "producteurs", "opérateurs", "danseurs", "chanteurs", "musiciens", "animateurs", "étudiants", "enseignants", "chercheurs", "innovateurs", "propriétaires", "directeurs", "dirigeants", "conducteurs", "programmeurs", "informaticiens", "ingénieurs", "techiciens", "aiguilleurs", "moines", "prêtres", "stewards", "serveurs", "servants", "cuisiniers", "boulangers", "policiers", "gardiens", "infirmiers", "pharmaciens", "acheteurs", "vendeurs", "éleveurs", "agriculteurs", "Gabriel", "Léo", "Théo", "Raphaël", "Arthur", "Louis", "Lucas", "Adam", "Jules", "Hugo", "Paul", "Gabin", "Nathan", "Noé", "Léon", "Martin", "Mathis", "Victor", "Marius", "Axel", "Antoine", "Valentin", "Augustin", "Clément", "Samuel", "Marceau", "Baptiste", "Gaspard", "Maxence", "Malo", "Maxime", "Simon", "Alexandre", "Camille", "Thomas", "Côme", "Léandre", "Oscar", "Mathys", "Joseph", "William", "Basile", "Charles", "Antonin", "Adrien", "Marin", "Jean", "Timothée", "Auguste", "Alexis", "Roméo", "Justin", "Aloïs", "Morgan", "Joachim", "Georges", "Louison", "Marc", "Alix", "Edgar", "Eloi", "Gaétan", "Zacharie", "Loïc", "Thibaut", "Aurèle", "Etienne", "Aurélien", "Matthieu", "Anthony", "Ambroise", "Loup", "Siméon", "Lubin", "Rémi", "Angel", "Hadrien", "Théophile", "Emilien", "Clovis", "Albin", "Paulin", "Jacques", "Florian", "Hippolyte", "Michaël", "Stanislas", "Colin", "Adonis", "Olivier", "Matthias", "Damien", "Guillaume", "Grégoire", "Balthazar", "François", "Aymeric", "Pacôme", "Maximilien", "Vincent", "Luc", "Eric", "Eugène", "Christian", "André", "Anaël", "Merlin", "Sébastien", "Armel", "Constant", "Abraham", "Aimé", "Michel", "Mayeul", "Constantin", "Lazare", "Arnaud", "Rémy", "Clarence", "Aristide", "Moïse", "Benoit", "Albert", "Yoann", "Denis", "Ferdinand", "Gatien", "Xavier", "Ariel", "Archibald", "Jude", "Calixte", "Joël", "Jérémie", "Ronan", "Florent", "Gautier", "Philippe", "Stéphane", "Vianney", "Alphonse", "Lionel", "Barthélémy", "Cédric", "Melchior", "Auxence", "Cyprien", "Zéphyr", "Fabien", "Bruno", "Cyril", "Gaspar", "Laurent", "Faustin", "Alain", "Anselme", "Flavien", "Ludovic", "Firmin", "Richard", "Germain", "Edmond", "Robert", "Patrick", "Brieuc", "Mathurin", "Honoré", "Marcelin", "Noël", "Martial", "Roch", "Alfred", "Sylvain", "Franck", "Maurice", "Victorien", "Frédéric", "Dominique", "Raoul", "Christophe", "Florentin", "Raymond", "Hugues", "Baudouin", "Anthime", "Judicaël", "Samson", "Salomon", "Théotime", "Emeric", "Séraphin", "Blaise", "Brice", "Audric", "Vivien", "Rodrigue", "Francis", "Fernand", "Roland", "Landry", "Enguerrand", "Jocelyn", "Isidore", "Barnabé", "Wandrille", "Théophane", "Yves", "Florin", "Anicet", "Ancelin", "Sixte", "Thierry", "Prosper", "Tancrède", "Cyriaque", "Philémon", "Manassé", "Josselin", "Armin", "Albéric", "Elisée", "Andéol", "Fabrice", "Serge", "Hubert", "Renaud", "Claude", "Pascal", "Jérôme", "Athanase", "Pierrick", "Gonzague", "Amédée", "René", "Aldric", "Romuald", "Cleandre", "Odilon", "Néhémie", "Clotaire", "Valère", "Gilles", "Gédéon", "Guy", "Cassien", "Bernard", "Joanny", "Candide", "Elzear", "Ignace", "Amiel", "Sidoine", "Nestor", "Patrice", "Alcide", "Roger", "Edgard", "Calliste", "Zéphir", "Audran", "Donatien", "Tanguy", "Victorin", "Rodolphe", "Arian", "Geoffroy", "Médéric", "Job", "Apollinaire", "Alderic", "Stephan", "Désiré", "Didier", "Adelin", "Lothaire", "Fantin", "Cyrille", "Ghislain", "Béranger", "Wilfried", "Japhet", "Guénolé", "Jeannot", "Séverin", "Evrard", "Bérenger", "Amélien", "Archange", "Gilbert", "Géraud", "Sauveur", "Foulques", "Valéry", "Timoleon", "Eustache", "Baudoin", "Eudes", "Yvon", "Arnould", "Hilaire", "Sylvestre", "Aurian", "Perceval", "Josse", "Bertrand", "Hervé", "Gérard", "Marcellin", "Adhemar"], + "female": ["m", "mlle", "madame", "mademoiselle", "baronesse", "comtesse", "marquise", "duchesse", "princesse", "emperesse", "reine", "dame", "femme", "meuf", "nana", "tante", "fille", "fillette", "gamine", "gonzesse", "amie", "pote", "compagne", "maîtresse", "amante", "soeur ", "épouse", "mariée", "mère", "maman", "daronne", "nièce", "tante", "grand-mère", "mamie", "femmes", "meufs", "nanas", "tantes", "filles", "fillettes", "gamines", "gonzesses", "amies", "potes", "compagnes", "maîtresses", "amantes", "soeurs", "épouses", "mariées", "mères", "mamans", "daronnes", "nièces", "tantes", "grand-mères", "mamies", "actrice", "comédienne", "héroïne", "amatrice", "réalisatrice", "productrice", "opératrice", "danseuse", "chanteuse", "musicienne", "animatrice", "étudiante", "enseignante", "chercheuse", "innovatrice", "proprietrice", "directrice", "dirigeante", "conductrice", "programmeuse", "informaticienne", "ingénieure", "technicienne", "aiguilleuse", "moniale", "nonne", "hôtesse", "serveuse", "servante", "cuisinière", "boulangère", "policière", "gardienne", "infermière", "pharmacienne", "acheteuse", "vendeuse", "éleveuse", "agricultrice", "actrices", "comédiennes", "héroïnes", "amatrices", "réalisatrices", "productrices", "opératrices", "danseuses", "chanteuses", "musiciennes", "animatrices", "étudiantes", "enseignantes", "chercheuses", "innovatrices", "proprietrices", "directrices", "dirigeantes", "conductrices", "programmeuses", "informaticiennes", "ingénieures", "techniciennes", "aiguilleuses", "moniales", "nonnes", "hôtesses", "serveuses", "servantes", "cuisinières", "boulangères", "policières", "gardiennes", "infirmières", "pharmaciennes", "acheteuses", "vendeuses", "éleveuses", "agricultrices", "Emma", "Jade", "Louise", "Alice", "Lina", "Chloé", "Inès", "Léa", "Rose", "Ambre", "Anna", "Julia", "Léna", "Zoé", "Juliette", "Manon", "Eva", "Agathe", "Lou", "Camille", "Jeanne", "Léonie", "Romane", "Iris", "Lucie", "Adèle", "Sarah", "Charlotte", "Margaux", "Olivia", "Victoria", "Victoire", "Margot", "Elise", "Clémence", "Alix", "Clara", "Capucine", "Anaïs", "Gabrielle", "Mathilde", "Apolline", "Lise", "Noémie", "Eléonore", "Maria", "Héloïse", "Valentine", "Marie", "Elsa", "Lisa", "Thaïs", "Mélina", "Andréa", "Zélie", "Salomé", "Eloïse", "Constance", "Soline", "Elisa", "Elina", "Océane", "Céleste", "Joséphine", "Elya", "Garance", "Faustine", "Sara", "Pauline", "Diane", "Suzanne", "Mélissa", "Lison", "Maeva", "Ninon", "Laura", "Emilie", "Albane", "Charline", "Elia", "Justine", "Célestine", "Coline", "Julie", "Cassandre", "Myriam", "Sophie", "Augustine", "Judith", "Louison", "Noëlie", "Raphaëlle", "Madeleine", "Axelle", "Daphné", "Aurore", "Hortense", "Clémentine", "Rachel", "Line", "Clarisse", "Olympe", "Violette", "Angèle", "Aliénor", "Lucile", "Esther", "Bérénice", "Ariane", "Blanche", "Brune", "Marion", "Athénaïs", "Isaure", "Rosalie", "Morgane", "Eve", "Flora", "Elisabeth", "Amélie", "Solène", "Marilou", "Lyne", "Claire", "Aline", "Abigaëlle", "Luce", "Céline", "Philippine", "Estelle", "Amandine", "Sibylle", "Adélaïde", "Fanny", "Eugénie", "Lydia", "Prune", "Marylou", "Colette", "Mélodie", "Sixtine", "Bertille", "Grâce", "Hélène", "Pénélope", "Agnès", "Abigaïl", "Adélie", "Laure", "Léontine", "Flore", "Mahaut", "Maud", "Fleur", "Flavie", "Ombeline", "Emeline", "Anne", "Annabelle", "Laurine", "Ophélie", "Oriane", "Gaëlle", "Azélie", "Mélanie", "Déborah", "Marguerite", "Marianne", "Clotilde", "Caroline", "Irène", "Orlane", "Philomène", "Maÿlis", "Isabelle", "Coralie", "Elodie", "Tiphaine", "Marine", "Sidonie", "Colombe", "Béatrice", "Laëtitia", "Aimée", "Honorine", "Léopoldine", "Arielle", "Sybille", "Angéline", "Félicie", "Ambrine", "Lauréline", "Cécile", "Eulalie", "Lucille", "Suzon", "Aurélia", "Ludivine", "Cyrielle", "Mahault", "Emmanuelle", "Nadine", "Michelle", "Perrine", "Domitille", "Tatiana", "Aglaé", "Simone", "Fantine", "Edith", "Auriane", "Eglantine", "Nicole", "Audrey", "Adeline", "Sandra", "Gladys", "Victorine", "Florine", "Hermione", "Alaïs", "Nathalie", "Hermine", "Miriam", "Solange", "Emelyne", "Auxane", "Laurène", "Aude", "Blandine", "Gwenaëlle", "Armance", "Lydie", "Alexane", "Daphnée", "Séraphine", "Angélique", "Barbara", "Armelle", "Marthe", "Eliane", "Gwendoline", "Stéphanie", "Octavie", "Mériam", "Ernestine", "Lauriane", "Evangéline", "Olive", "Gisèle", "Mélissandre", "Nine", "Florence", "Aurélie", "Marceline", "Priscille", "Théodora", "Aloïse", "Evelyne", "Catherine", "Noëlla", "Jacinthe", "Aubane", "Marlène", "Delphine", "Mégane", "Viviane", "Coraline", "Solenn", "Ariel", "Thérèse", "Marcelle", "Doriane", "Débora", "Alizé", "France", "Maureen", "Artémis", "Espérance", "Bénédicte", "Christine", "Mylène", "Annie", "Prisca", "Quitterie", "Alexine", "Prudence", "Astrée", "Albertine", "Edmée", "Lauren", "Christelle", "Ysaline", "Marjorie", "Aure", "Alexandrine", "Jeannette", "Eliette", "Valérie", "Loriane", "Denise", "Michèle", "Johane", "Françoise", "Clélie", "Azelle", "Azalée", "Sandrine", "Azéline", "Ismérie", "Anne-Lise", "Guillemette", "Odile", "Désirée", "Alexiane", "Léonce", "Mariette", "Virginie", "Jéhanne", "Ameline", "Abelle", "Lisette", "Laurianne", "Malaury", "Lorène", "Paule", "Avigaïl", "Dorine", "Mallaury", "Léocadie", "Priscilla", "Sabine", "Myriame", "Natalie", "Magali", "Henriette", "Jocelyne", "Karine", "Pétronille", "Reine", "Annette", "Pascaline", "Yseult", "Elvire", "Martine", "Jacqueline", "Amethyste", "Marie-Claire", "Francine", "Othilie", "Sylviane", "Melisandre", "Claudine", "Athina", "Amaryllis", "Brunehilde", "Yvonne", "Lorraine", "Alexanne", "Adrienne", "Antoinette", "Renée", "Noëlle", "Marjolaine", "Laurence", "Mélisande", "Christiane", "Roseline", "Lucienne", "Dorothée", "Danielle", "Aliette", "Chantal", "Geneviève", "Muriel", "Joëlle", "Sylvie", "Francia", "Carine", "Florie", "Edwige", "Ombline", "Emmeline", "Aube", "Belle", "Isabeau", "Corentine", "Léonne", "Marie-Hélène", "Julienne", "Monique", "Iseult", "Mélaine", "Géraldine", "Annick", "Giselle", "Bernadette", "Laurane", "Brigitte", "Corinne", "Nadège", "Anémone", "Ambroisine", "Dominique", "Gertrude", "Annonciade", "Murielle", "Yvette", "Philothée", "Anastasie", "Véronique", "Mariane", "Charlaine", "Aleth", "Maguelone", "Eudoxie"] }, "sexual_orientation": { - "hetero": [], - "homo": [] + "hetero": ["hétéro", "hétéros", "hétérosexuel", "hétérosexuelle", "hétérosexuels", "hétérosexuelles"], + "homo": ["homo", "homos", "homosexuel", "homosexuelle", "homosexuels", "homosexuelles"] }, "age": { - "young": [], - "old": [] + "young": ["jeun", "jeuns", "jeune", "jeunes", "mineur", "mineurs", "mineure", "mineures", "juvénile", "juvéniles", "adolescent", "adolescents", "adolescente", "adolescentes", "immature", "immatures", "puéril", "puérile", "puérils", "puériles", "infantile", "infantiles", "enfantin", "enfantine", "enfantins", "enfantines"], + "old": ["vieux", "vieil", "vieille", "vieilles", "âgé", "âgée", "âgés", "âgées", "sénile", "séniles", "mûr", "mûre", "mûrs", "mûres", "vétéran", "vétérans", "gériatrique"] }, "appearencence": { - "attractive": [], - "unattractive": [] + "attractive": ["bel", "belle", "bels", "belles", "joli", "jolie", "jolis", "jolies", "mignon", "mignons", "mignonne", "mignonnes", "attractif", "attractifs", "attractive", "attractives", "attrayant", "attrayante", "attrayants", "attrayantes", "attirant", "attirante", "attirants", "attirantes", "charmant", "charmante", "charmants", "charmantes", "mince", "minces"], + "unattractive": ["moche", "moches", "laid", "laide", "laids", "laides", "vilain", "vilaine", "vilains", "vilaines", "horrible", "horribles", "affreux", "affreuse", "affreuses", "repoussant", "repoussants", "repoussante", "repoussantes", "répugnant", "répugnants", "répugnante", "répugnantes", "dégoûtant", "dégoûtants", "dégoûtante", "dégoûtantes", "dégueulasse", "dégueulasses", "gros", "grosse", "grosses"] }, "disability": { From 9ed012cb63f63f89fcc71a1d591dd3c1061d2a0f Mon Sep 17 00:00:00 2001 From: Asnota Date: Fri, 3 Dec 2021 16:16:41 +0100 Subject: [PATCH 3/3] Update README.md Mentioned French laguage in the description --- filters/universal_bias/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/filters/universal_bias/README.md b/filters/universal_bias/README.md index cdfeee8ea..7bf8ce588 100644 --- a/filters/universal_bias/README.md +++ b/filters/universal_bias/README.md @@ -3,9 +3,10 @@ ## What type of a filter is this? This filter is currently contains lexical seeds for 10 categories (religion, race, ethnicity, gender, sexual orientation, age, appearance, disability, experience, education, economic status). -The lexical seeds representing these categories are currently available in English only, however the pool of languages can be extended by a simple addition of the lexical seeds in a desired language to the lexicals.json file. +The lexical seeds representing these categories are currently available in English and French languages, however the pool of languages can be extended by a simple addition of the lexical seeds in a desired language to the lexicals.json file, using a separate language key (Ex.: Polish language goes with "pl" key). The minority parameter is a potentially underrepresented group, defined with its own set of keywords; the majority parameter is a set of keywords, representing the dominating group. + The filter returns "True" if the minority group is indeed underrepresented, "False" otherwise. Author: Anna Shvets @@ -26,8 +27,8 @@ The beneficial impact of the current extrinsic filter is its complete transparen ## Example of use ``` -sentences = [ "He is going to make a cake.", - "Olivia is going to program", +sentences = [ "He is going home.", + "Olivia is going to work", "Nobody likes washing dishes", "He agreed to help me" ] @@ -68,8 +69,8 @@ print("This is a neutral group:", neutral_group) ``` Output: ``` -This is a minority group: ['She is going to program'] -This is a majority group: ['He is going to make a cake.', 'He agreed to help me'] +This is a minority group: ['She is going to work'] +This is a majority group: ['He is going home.', 'He agreed to help me'] This is a neutral group: ['Nobody likes washing dishes'] ```