diff --git a/test/mapper.py b/test/mapper.py index 09c3ed8b5..a63db9d74 100644 --- a/test/mapper.py +++ b/test/mapper.py @@ -71,6 +71,7 @@ "pinyin", "punctuation", "quora_trained_t5_for_qa", + "random_walk" "sentence_reordering", "synonym_substitution", "token_replacement", diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md new file mode 100644 index 000000000..184c1e3e5 --- /dev/null +++ b/transformations/random_walk/README.md @@ -0,0 +1,62 @@ +# Random Walk using Masked-Languange Modeling +This transformation performs a random walk on the original sentence by randomly masking a word and replacing it with a suggestion by the BERT languange model. + +Author names: + + - Sajant Anand (sajant@berkeley.edu, UC Berkeley) + - Roy Rinberg (royrinberg@gmail.com, Columbia University) + - Jamie Simon (james.simon@berkeley.edu, UC Berkeley) + - Chandan Singh (chandan_singh@berkeley.edu, UC Berkeley) + +## Data and Code Provenance + +This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers library. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran. Both libraries operates under the Apache 2.0 license. Additionally the Spacy library is necessary but is installed by default when using this benchmark. + +## What type of a transformation is this? +This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. We measure the similarity between the original and random-walked sentence by performing sentence embeddings and then calculate the cosine similarity bewteen the embedded vectors. + +## How it works +At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. Additionally, we can exclude named entities found by the Spacy model from random selection for repalcement. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences. Note that BERT has a max input token length of 512, so for long inputs, we split the sentence into chunks less than 512 tokens. + +The differences between original and generated sentences are generally controlled by two class initialization parameters, `steps` and `k`. + - `steps`: number of random walk steps to do + - `k`: number of high probability replacements for the masked word to consider + +This process generates $k^steps$ new sentences. We then randomly select a subset of these, as specified by `max_outputs`. + +The seed of the random generators (both from `numpy` and the `random` module) are set by the `seed` parameter in the class initializer. Choosing a fixed value will lead to reproducable results. + +The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter `sim_req` which is the minimum similarity score that a generated sentence must have to be considered valid. + +Finally a boolean `names` specifies whether or not we replace named entities and a boolean `descending` controls the order of the most probable tokens for masked-word replacement. + +## What tasks does it intend to benefit? +This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, text generation, etc. Evaluating the perturbation using Google Colab is currently in progress. + +## Robustness Evaluation + +This model was evaluated with the model aychang/roberta-base-imdb on the test[:20%] split of the imdb dataset. Note: due to the computational demands of this transformation and the lack of resources at our disposal (only GPU access is Colab), we evaluate the transformation with the following parameters: + - `seed = 0` + - `max_outputs = 1` : Produce a single sentence + - `steps = 5` : Randomly select a word to replace 5 times + - `k = 1`: Number of high probability replacements for the masked word to consider + - `sim_req = 0` : Similarity requirement for generated sentences (long sentences tend to have low similarity + - `named_entities = True` : Do not replace named entities + - `descending = True` : Choose most probable replacements (we will rarely use `False`; we included it for kicks.) + +Wall Time: 00:03:52 (DD:HH:MM) +Performance: Of 1000 original sentences, 985 successfully transformed and 15 unchanged (0.985 perturb rate). Accuracy: 96.0 -> 96.0 + +Performance is strongly affected by parameters `steps` and `k`, as larger values of each will lead to greater variation in generated sentences, at the expense of longer runtimes. + +## What are the limitations of this transformation? + +This transformation can generate nonsensical words when the random walk has many steps (steps >~ number of words in sentence). + +## References +1) Saketh Kotamraju; "How to use BERT from the Hugging face transformer library"; https://towardsdatascience.com/how-to-use-bert-from-the-hugging-face-transformer-library-d373a22b0209 + +As far as we know, this type of transformation where words are randomly perturbed has not been studied in published literature. Random walks have been used to measure sentence similarity, e.g. the papers listed below. + +2) Daniel Ramage, Anna N. Rafferty, and Christopher D. Manning; "Random Walks for Text Semantic Similarity"; https://nlp.stanford.edu/pubs/wordwalk-textgraphs09.pdf +3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf diff --git a/transformations/random_walk/__init__.py b/transformations/random_walk/__init__.py new file mode 100644 index 000000000..89ecd1199 --- /dev/null +++ b/transformations/random_walk/__init__.py @@ -0,0 +1,2 @@ +from .transformation import * + diff --git a/transformations/random_walk/requirements.txt b/transformations/random_walk/requirements.txt new file mode 100644 index 000000000..394f7600f --- /dev/null +++ b/transformations/random_walk/requirements.txt @@ -0,0 +1,2 @@ +sentence-transformers==2.0.0 +transformers==4.6.1 \ No newline at end of file diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json new file mode 100644 index 000000000..471237f4b --- /dev/null +++ b/transformations/random_walk/test.json @@ -0,0 +1,90 @@ +{ + "type": "random_walk", + "test_cases": [ + { + "class": "RandomWalk", + "inputs": { + "sentence": "Andrew finally returned the French book to Chris that I bought last week." + }, + "outputs": [ + { + "sentence": "She finally returned the French book to me that I bought last year." + }, + { + "sentence": "She finally returned the picture book for me that I bought last year." + }, + { + "sentence": "Andrew finally returned the French box to Chris that I bought last week." + } + ] + }, + { + "class": "RandomWalk", + "inputs": { + "sentence": "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments." + }, + "outputs": [ + { + "sentence": "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an explicit predicate to explain the relation between two or more arguments." + }, + { + "sentence": "Sentences involving gapping, such as John likes coffee and Mary tea, lack an appropriate predicate to indicate the relation between two or three arguments." + }, + { + "sentence": "Examples with gapping, Such as Paul likes coffee and Mary tea, lack an appropriate predicate to indicate the relation between two or more arguments." + } + ] + }, + { + "class": "RandomWalk", + "inputs": { + "sentence": "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film" + }, + "outputs": [ + { + "sentence": "Alice in Wonderland is a 2010 American live-Action/Animated romantic fantasy adventure film" + }, + { + "sentence": "Alice In Wonderland is a 2010 Canadian live-Action/animated dark fantasy adventure film" + }, + { + "sentence": "Alice in Wonderland is a 2010 American live-Action/Animated dark fantasy adventure film" + } + ] + }, + { + "class": "RandomWalk", + "inputs": { + "sentence": "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001" + }, + "outputs": [ + { + "sentence": "Ujjal Dev who served as the Premier of Sri Columbia from 2000 until 2001" + }, + { + "sentence": "Ram Dev Dosanjh served as 33rd Premier of British Columbia from 2000 until 2003" + }, + { + "sentence": "and Dev Dosanjh serving as Deputy Premier of British Columbia from 2000 to 2001" + } + ] + }, + { + "class": "RandomWalk", + "inputs": { + "sentence": "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization." + }, + "outputs": [ + { + "sentence": "Neuroplasticity is a continuous processing allowing short-term, mid-term, and long-term remodeling of the brain organization." + }, + { + "sentence": "Neuroplasticity is a neural processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization." + }, + { + "sentence": "It is a dynamic processing allowing short-duration, medium-term, and long-term remodeling of the neuronosynaptic organization." + } + ] + } + ] +} \ No newline at end of file diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py new file mode 100644 index 000000000..d9d4acaf9 --- /dev/null +++ b/transformations/random_walk/transformation.py @@ -0,0 +1,326 @@ +#import itertools +import random +import numpy as np +import re +import copy +import random +from typing import List + +from transformers import BertTokenizer, BertForMaskedLM +from sentence_transformers import SentenceTransformer, util +import spacy + +from torch.nn import functional as F +import torch + +from interfaces.SentenceOperation import SentenceOperation +from tasks.TaskTypes import TaskType + +def _mask_word(sentence, split_indices, mask): + """ helper function to replace word in a sentence with mask-token, as prep + for BERT tokenizer + + Args: + sentence (str): sentence with work to mask + split_indices ([int, int]): index of word to replace, begining and character + after the end indices + mask (BERT Token): token for a BERT mask + + """ + return sentence[0:split_indices[0]] + mask + sentence[ + split_indices[1]:] + + +def get_k_replacement_words(tokenized_text, tokenizer, model, k, descending=True): + """return k most similar words from the model, for a tokenized mask-word + in a sentence. + + Args: + tokenized_text (str): sentence with a word masked out + tokenizer ([type]): tokenizer + model ([type]): model + k (int): how many similar words to find for a given tokenized-word. Checks + that generated word is a composed of letters or numbers. + descending (bool): If true, sort tokens by in decreasing order of probability. + Else sort in increasing order of probability. We sample from sorted tokens. + + Returns: + [list]: list of top k words + bool: Whether or not we found the desired number of valid replacement words + """ + inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512) + index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id) + if index_to_mask[0].numel() == 0: # Since we are truncating the input to be + # 512 tokens (BERT's max), we need to make sure the mask is in these first 512. + # If not, return False so that we try again. + return None, False + # This should not occur since we split long sentences. + outputs = model(**inputs) + softmax = F.softmax(outputs.logits, dim=-1) + mask_word = softmax[0, index_to_mask, :] + + sorted_tokens = torch.argsort(mask_word[0], descending=descending) + i = 0 + valid_tokens = [] # The k most probable tokens are guaranteed to be words, + # so we make sure they are. + while len(valid_tokens) < k and i < len(sorted_tokens): + if tokenizer.decode([sorted_tokens[i]]).isalnum(): + valid_tokens.append(sorted_tokens[i]) + i += 1 + assert len(valid_tokens) == k or i == len(sorted_tokens) # We either have found k valid (non punctuation) tokens or we have looked through all the tokens. + if len(valid_tokens) < k: + valid_tokens += (k - len(valid_tokens)) * [valid_tokens[0]] + + return valid_tokens, True + + +def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k, descending): + """For a given sentence, choose a random word to mask, and + replace it with a word the top-k most similar words in BERT model. + Return k sentences, each with a different replacement word for the mask. + + Args: + sentence ([type]): sentence to perform random walk on + tokenizer ([type]): tokenizer + model ([type]): model + nlp ([type]): Spacy NLP model for finding named entities + names (list): Named entities to not replace. + k (int): how many replacement words to try. + descending (bool): If true, sort tokens by in decreasing order of probability. + Else sort in increasing order of probability. We sample from sorted tokens. + + Returns: + [list]: k-sentences with masked word replaced with top-k most similar words + """ + split_iter = re.finditer(r"[\w']+|[.,!?;]", sentence) # Split sentence on puctuation. + text_split = [] + split_indices = [] + sentence_parts = [] + text_split.append([]) + split_indices.append([]) + base_index = 0 + for m in split_iter: + text_split[-1].append(m.group(0)) + split_indices[-1].append((m.start() - base_index, m.end() - base_index)) + + if m.end() - base_index > 450: + sentence_parts.append(sentence[base_index:m.end()]) + base_index = m.end() + text_split.append([]) + split_indices.append([]) + + sentence_parts.append(sentence[base_index:]) + + # Remove any empty sentence parts. + valid_splits = [len(ts) > 0 for ts in text_split] + text_split = [ts for ts, vs in zip(text_split, valid_splits) if vs] + split_indices = [si for si, vs, in zip(split_indices, valid_splits) if vs] + sentence_parts = [sen for sen, vs in zip(sentence_parts, valid_splits) if vs] + + new_sentences = [] + for ts, si, sen in zip (text_split, split_indices, sentence_parts): + if len(ts) == 0: + print(text_split, split_indices, sentence_parts) + raise ValueError("Somehow we got a sentence part that is emtpy. Not good!") + rand_int = np.random.randint(len(ts)) # pick a random word to mask + word_to_mask = ts[rand_int] + iter_count = 1 + give_up = False + while len(word_to_mask) == 0 or not word_to_mask.isalnum() or word_to_mask in names: # Avoid empty strings in split text + rand_int = np.random.randint(len(ts)) + word_to_mask = ts[rand_int] + iter_count += 1 + if iter_count > len(ts): + print("In the sentence <" + sen + ">, no valid words to mask.") + give_up = True + break + + if not give_up: + # mask word + new_text = _mask_word(sen, si[rand_int], tokenizer.mask_token) + # get k replacement words + top_k, included_mask = get_k_replacement_words(new_text, tokenizer, + model, k=k, descending=descending) + assert included_mask == True + + replacement_words = [tokenizer.decode([token]) for token in top_k] + + # replace mask-token with the word from the top-k replacements + new_sentences.append([ + new_text.replace(tokenizer.mask_token, word) + for word in replacement_words + ]) + else: + new_sentences.append([ + sen for _ in range(k) + ]) + + final_sentences = new_sentences[0] + for sens in new_sentences[1:]: + final_sentences = [s + a for s, a in zip(final_sentences, sens)] + return final_sentences + + +def single_round(sentences: List[str], tokenizer, model, nlp, names, k, descending) -> List[str]: + """For a given list of sentences, do a random walk on each sentence. + + Args: + sentences ([type]): list of sentnces to perform random walk on + tokenizer ([type]): tokenizer + model ([type]): model + nlp ([type]): Spacy NLP model for finding named entities + names (list): Named entities to not replace. + k (int): how many words to sample to replace masked word + descending (bool): If true, sort tokens by in decreasing order of probability. + Else sort in increasing order of probability. We sample from sorted tokens. + + Returns: + [List]: list of k random-walked sentences + """ + new_sentences = [] + + for sentence in sentences: + new_sentences.extend( + single_sentence_random_step(sentence, tokenizer, model, nlp, names, k, descending)) + return new_sentences + + +def random_walk(original_text: str, steps: int, k: int, tokenizer, + model, nlp, names: bool, descending: bool) -> List[str]: + """For a sentence, perform a random walk sequence on the sentence, generating + new sentences at each step and perturbing these during the next step. + + Args: + original_text (str): original sentence we want to perturb + steps (int): how many random walks iterations we perform on the sentence + k (int): how many words to sample to replace masked word during each iteration + tokenizer ([type]): tokenizer + model ([type]): model + nlp ([type]): Spacy NLP model for finding named entities + names (bool): Whether or not to change named entities + descending (bool): If true, sort tokens by in decreasing order of probability. + Else sort in increasing order of probability. We sample from sorted tokens. + + Returns: + [List]: list of steps^k random-walked sentences + """ + if names: + doc = nlp(original_text) + entities = [ent.text for ent in doc.ents] + split_entities = [] + for ent in entities: + split_iter = re.finditer(r"[\w']+|[.,!?;]", ent) + for m in split_iter: + split_entities.append(m.group(0)) + print('Named entities: ', split_entities) + else: + split_entities = [] + old_sentences = [original_text] + # Do $steps$ steps of random walk procedure + for _ in range(steps): + sentences = single_round(old_sentences, tokenizer, model, nlp, split_entities, k, descending) + old_sentences = copy.deepcopy(sentences) + return sentences + +def sentence_similarity_metric(similarity_model, sen_A, sen_B): + """Compute the similarity between two sentences by embedding them using a + sentence transformer and computing the cosine similarity. + + Args: + similarity_model (type): sentence transformer + sen_A (str): first sentence + sen_B (str): second sentence + Returns: + float: sentence similarity + """ + + emb_A = similarity_model.encode(sen_A) + emb_B = similarity_model.encode(sen_B) + + score = util.pytorch_cos_sim(emb_A, emb_B) + return score + +class RandomWalk(SentenceOperation): + tasks = [ + TaskType.TEXT_TO_TEXT_GENERATION, + TaskType.TEXT_CLASSIFICATION + ] + languages = ["en"] + heavy = True + keywords = [ "model-based", "api-based", "transformer-based", "tokenizer-required", \ + "lexical", "possible-meaning-alteration", "low-precision", \ + "high-coverage", "high-generations" ] + + # Default parameters match those of the 'test.json' below. + def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, + named_entities=False, descending=True): + # For evaluation, use parameters that are less compute intensive. + # def __init__(self, seed=0, max_outputs=1, steps=5, k=1, sim_req=0, named_entities=True, descending=True): + random.seed(self.seed) + np.random.seed(self.seed) + + super().__init__(seed, max_outputs=max_outputs) + self.tokenizer = BertTokenizer.from_pretrained('bert-large-cased') + self.model = BertForMaskedLM.from_pretrained('bert-large-cased') + self.sim_model = SentenceTransformer('all-mpnet-base-v2') + self.spacy_nlp = spacy.load("en_core_web_sm") + self.max_outputs = max_outputs + self.steps = steps + self.k = k + self.sim_req = sim_req + self.named_entities = named_entities + self.descending = descending + + def generate(self, sentence: str): + print('Random walking on the sentence:', sentence) + perturbed_texts = random_walk( + original_text=sentence, + steps=self.steps, + k=self.k, + tokenizer=self.tokenizer, + model=self.model, + nlp=self.spacy_nlp, + names=self.named_entities, + descending=self.descending + ) + scores = [] + for o in perturbed_texts: + scores.append(sentence_similarity_metric(self.sim_model, sentence, o)) + valid_sentences = np.array(scores) > self.sim_req # Only sentences with a + assert np.sum(valid_sentences) > 0, "Similarity requirement too high; no valid sentences. Note: Long sentences have very low similarity." + + # high enough similarity score are kept. + perturbed_texts = [o for o,s in zip(perturbed_texts, valid_sentences) if s] + assert np.sum(valid_sentences) == len(perturbed_texts) + + if len(perturbed_texts) > self.max_outputs: + perturbed_texts = random.sample(perturbed_texts, self.max_outputs) + + return perturbed_texts + +""" +# The code to produce 'test.json' must be commented out so that pytest succeeds. +# Sample code to demonstrate usage. Can also assist in adding test cases. +# You don't need to keep this code in your transformation. +if __name__ == '__main__': + import json + #from TestRunner import convert_to_snake_case + + tf = RandomWalk(max_outputs=3, k=2, steps=5, sim_req=0.25, named_entities=True) + #sentence = "Andrew finally returned the French book to Chris that I bought last week" + test_cases = [] + for sentence in ["Andrew finally returned the French book to Chris that I bought last week.", + "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments.", + "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film", + "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001", + "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."]: + test_cases.append({ + "class": "RandomWalk",#tf.name(), + "inputs": {"sentence": sentence}, "outputs": [{"sentence": o} for o in tf.generate(sentence)]} + ) + json_file = {"type": "random_walk", "test_cases": test_cases} #convert_to_snake_case(tf.name()) + print(json.dumps(json_file, indent=2)) + + with open('test.json', 'w') as f: + json.dump(json_file, f, indent=2) +""" \ No newline at end of file