From e562770567e2e05725120b6db8b5306e2acfefa8 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Tue, 31 Aug 2021 23:20:25 -0700
Subject: [PATCH 01/19] Create branch.

---
 transformations/random_walk/README.md         |  70 +++++++++++
 transformations/random_walk/__init__.py       |   2 +
 transformations/random_walk/test.json         |  50 ++++++++
 transformations/random_walk/transformation.py | 117 ++++++++++++++++++
 4 files changed, 239 insertions(+)
 create mode 100644 transformations/random_walk/README.md
 create mode 100644 transformations/random_walk/__init__.py
 create mode 100644 transformations/random_walk/test.json
 create mode 100644 transformations/random_walk/transformation.py

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
new file mode 100644
index 000000000..31a4094bd
--- /dev/null
+++ b/transformations/random_walk/README.md
@@ -0,0 +1,70 @@
+# Butter Fingers Perturbation 🦎  + ⌨️ → 🐍
+This perturbation adds noise to all types of text sources (sentence, paragraph, etc.) proportional to noise erupting 
+from keyboard typos making common spelling errors.
+
+Author name: Kaustubh Dhole
+Author email: __
+Author Affiliation: __
+
+## What type of a transformation is this?
+This transformation acts like a perturbation to test robustness. Few letters picked at random are replaced with letters 
+which are at keyboard positions near the source letter. Generated transformations display high similarity to the 
+source sentences i.e. the code outputs highly precise generations. 
+
+## What tasks does it intend to benefit?
+This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 
+text generation, etc. 
+
+```python evaluate.py -t ButterFingersPerturbation -task TEXT_CLASSIFICATION```
+```model_name = "aychang/roberta-base-imdb"```
+The accuracy of a RoBERTa model (fine-tuned on IMDB) (model: "aychang/roberta-base-imdb") 
+on a subset of IMDB sentiment dataset = 95.74
+The accuracy of the same model on the perturbed set = 88.26
+
+The average bleu score of a distillbert model (fine-tuned on xsum) (model: "sshleifer/distilbart-xsum-12-6") 
+on a subset (10%) of xsum test dataset = 14.9104
+The average bleu score of same model on the pertubed set = 11.9221
+
+## Previous Work
+1) Butter Finger implementation borrowed from this code https://github.com/alexyorke/butter-fingers
+
+2) There has also been some recent work in the contrast sets of the GEM Benchmark (ACL 2021):
+```bibtex
+@article{DBLP:journals/corr/abs-2102-01672,
+  title     = {The {GEM} Benchmark: Natural Language Generation, its Evaluation and
+               Metrics},
+  journal   = {CoRR},
+  volume    = {abs/2102.01672},
+  year      = {2021},
+  url       = {https://arxiv.org/abs/2102.01672},
+  archivePrefix = {arXiv},
+  eprint    = {2102.01672},
+  timestamp = {Tue, 16 Feb 2021 16:58:52 +0100},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-2102-01672.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+
+3) There has been some recent work in NoiseQA too:
+```bibtex
+@inproceedings{ravichander-etal-2021-noiseqa,
+    title = "{N}oise{QA}: Challenge Set Evaluation for User-Centric Question Answering",
+    author = "Ravichander, Abhilasha  and
+      Dalmia, Siddharth  and
+      Ryskina, Maria  and
+      Metze, Florian  and
+      Hovy, Eduard  and
+      Black, Alan W",
+    booktitle = "Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume",
+    month = apr,
+    year = "2021",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2021.eacl-main.259",
+    pages = "2976--2992",
+    abstract = "When Question-Answering (QA) systems are deployed in the real world, users query them through a variety of interfaces, such as speaking to voice assistants, typing questions into a search engine, or even translating questions to languages supported by the QA system. While there has been significant community attention devoted to identifying correct answers in passages assuming a perfectly formed question, we show that components in the pipeline that precede an answering engine can introduce varied and considerable sources of error, and performance can degrade substantially based on these upstream noise sources even for powerful pre-trained QA models. We conclude that there is substantial room for progress before QA systems can be effectively deployed, highlight the need for QA evaluation to expand to consider real-world use, and hope that our findings will spur greater community interest in the issues that arise when our systems actually need to be of utility to humans.",
+}
+```
+## What are the limitations of this transformation?
+The transformation's outputs are too simple to be used for data augmentation. Unlike a paraphraser, it is not capable of
+ generating linguistically diverse text.
diff --git a/transformations/random_walk/__init__.py b/transformations/random_walk/__init__.py
new file mode 100644
index 000000000..89ecd1199
--- /dev/null
+++ b/transformations/random_walk/__init__.py
@@ -0,0 +1,2 @@
+from .transformation import *
+
diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json
new file mode 100644
index 000000000..a42c6d52c
--- /dev/null
+++ b/transformations/random_walk/test.json
@@ -0,0 +1,50 @@
+{
+  "type": "butter_fingers_perturbation",
+  "test_cases": [
+    {
+      "class": "ButterFingersPerturbation",
+      "inputs": {
+        "sentence": "Andrew finally returned the French book to Chris that I bought last week"
+      },
+      "outputs": [{
+        "sentence": "Andgew finally returned the French book to Chrus thav I bought last week"
+      }]
+    },
+    {
+      "class": "ButterFingersPerturbation",
+      "inputs": {
+        "sentence": "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments."
+      },
+      "outputs": [{
+        "sentence": "Senhences with gapping, such as Paul likes coffwe and Mary tea, lack an overt predicate to indicatx the relauion between two or more argumentd."
+      }]
+    },
+    {
+      "class": "ButterFingersPerturbation",
+      "inputs": {
+        "sentence": "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film"
+      },
+      "outputs": [{
+        "sentence": "Alife in Wonderland is a 2010 American live-action/abimatev dark fantasy adventure film"
+      }]
+    },
+    {
+      "class": "ButterFingersPerturbation",
+      "inputs": {
+        "sentence": "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001"
+      },
+      "outputs": [{
+        "sentence": "Ujjwl Dev Dosanjh served as 33rd Premier of Britush Conumbia from 2000 to 2001"
+      }]
+    },
+    {
+      "class": "ButterFingersPerturbation",
+      "inputs": {
+        "sentence": "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."
+      },
+      "outputs": [{
+        "sentence": "Neugoplasticity is a continuous processing alliwing vhort-term, medium-term, and long-term remodeling of the beuronosynaptic organization."
+      }]
+    }
+  ]
+}
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
new file mode 100644
index 000000000..082c137db
--- /dev/null
+++ b/transformations/random_walk/transformation.py
@@ -0,0 +1,117 @@
+import itertools
+import random
+
+from interfaces.SentenceOperation import SentenceOperation
+from tasks.TaskTypes import TaskType
+
+"""
+Base Class for implementing the different input transformations a generation should be robust against.
+"""
+
+
+def butter_finger(text, prob=0.1, keyboard="querty", seed=0, max_outputs=1):
+    random.seed(seed)
+    key_approx = {}
+
+    if keyboard == "querty":
+        key_approx["q"] = "qwasedzx"
+        key_approx["w"] = "wqesadrfcx"
+        key_approx["e"] = "ewrsfdqazxcvgt"
+        key_approx["r"] = "retdgfwsxcvgt"
+        key_approx["t"] = "tryfhgedcvbnju"
+        key_approx["y"] = "ytugjhrfvbnji"
+        key_approx["u"] = "uyihkjtgbnmlo"
+        key_approx["i"] = "iuojlkyhnmlp"
+        key_approx["o"] = "oipklujm"
+        key_approx["p"] = "plo['ik"
+
+        key_approx["a"] = "aqszwxwdce"
+        key_approx["s"] = "swxadrfv"
+        key_approx["d"] = "decsfaqgbv"
+        key_approx["f"] = "fdgrvwsxyhn"
+        key_approx["g"] = "gtbfhedcyjn"
+        key_approx["h"] = "hyngjfrvkim"
+        key_approx["j"] = "jhknugtblom"
+        key_approx["k"] = "kjlinyhn"
+        key_approx["l"] = "lokmpujn"
+
+        key_approx["z"] = "zaxsvde"
+        key_approx["x"] = "xzcsdbvfrewq"
+        key_approx["c"] = "cxvdfzswergb"
+        key_approx["v"] = "vcfbgxdertyn"
+        key_approx["b"] = "bvnghcftyun"
+        key_approx["n"] = "nbmhjvgtuik"
+        key_approx["m"] = "mnkjloik"
+        key_approx[" "] = " "
+    else:
+        print("Keyboard not supported.")
+
+    prob_of_typo = int(prob * 100)
+    perturbed_texts = []
+    for _ in itertools.repeat(None, max_outputs):
+        butter_text = ""
+        for letter in text:
+            lcletter = letter.lower()
+            if lcletter not in key_approx.keys():
+                new_letter = lcletter
+            else:
+                if random.choice(range(0, 100)) <= prob_of_typo:
+                    new_letter = random.choice(key_approx[lcletter])
+                else:
+                    new_letter = lcletter
+            # go back to original case
+            if not lcletter == letter:
+                new_letter = new_letter.upper()
+            butter_text += new_letter
+        perturbed_texts.append(butter_text)
+    return perturbed_texts
+
+
+"""
+Butter Finger implementation borrowed from https://github.com/alexyorke/butter-fingers.
+"""
+
+
+class ButterFingersPerturbation(SentenceOperation):
+    tasks = [
+        TaskType.TEXT_CLASSIFICATION,
+        TaskType.TEXT_TO_TEXT_GENERATION,
+        TaskType.TEXT_TAGGING,
+    ]
+    languages = ["en"]
+
+    def __init__(self, seed=0, max_outputs=1):
+        super().__init__(seed, max_outputs=max_outputs)
+
+    def generate(self, sentence: str):
+        perturbed_texts = butter_finger(
+            text=sentence,
+            prob=0.05,
+            seed=self.seed,
+            max_outputs=self.max_outputs,
+        )
+        return perturbed_texts
+
+
+"""
+# Sample code to demonstrate usage. Can also assist in adding test cases.
+# You don't need to keep this code in your transformation.
+if __name__ == '__main__':
+    import json
+    from TestRunner import convert_to_snake_case
+
+    tf = ButterFingersPerturbation(max_outputs=3)
+    sentence = "Andrew finally returned the French book to Chris that I bought last week"
+    test_cases = []
+    for sentence in ["Andrew finally returned the French book to Chris that I bought last week",
+                     "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments.",
+                     "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film",
+                     "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001",
+                     "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."]:
+        test_cases.append({
+            "class": tf.name(),
+            "inputs": {"sentence": sentence}, "outputs": [{"sentence": o} for o in tf.generate(sentence)]}
+        )
+    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
+    print(json.dumps(json_file, indent=2))
+"""

From beff00f8e4a9b55b77ae8fa960a0dce720a4e478 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Wed, 1 Sep 2021 11:25:59 -0700
Subject: [PATCH 02/19] Working barebones version.

---
 transformations/random_walk/README.md         |  76 ++++------
 transformations/random_walk/test.json         |  84 ++++++++---
 transformations/random_walk/transformation.py | 138 ++++++++----------
 3 files changed, 150 insertions(+), 148 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index 31a4094bd..dffe97630 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -1,15 +1,27 @@
-# Butter Fingers Perturbation 🦎  + ⌨️ → 🐍
-This perturbation adds noise to all types of text sources (sentence, paragraph, etc.) proportional to noise erupting 
-from keyboard typos making common spelling errors.
+# Random Walk using Masked-Languange Modeling
+This transformation performs a random walk on the original sentence by randomly masking a word and replacing it with a suggestion by the BERT languange model.
 
-Author name: Kaustubh Dhole
-Author email: __
-Author Affiliation: __
+Author names:
+ - Chandan Singh (chandan_singh@berkeley.edu, UC Berkeley)
+ - Jamie Simon (james.simon@berkeley.edu, UC Berkeley)
+ - Sajant Anand (sajant@berkeley.edu, UC Berkeley)
+ - Roy Rinberg (royrinberg@gmail.com, Columbia University)
+
+## Extras
+
+This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library. Provided that the library is installed, this model will be installed the first time this transformation is ran.
 
 ## What type of a transformation is this?
-This transformation acts like a perturbation to test robustness. Few letters picked at random are replaced with letters 
-which are at keyboard positions near the source letter. Generated transformations display high similarity to the 
-source sentences i.e. the code outputs highly precise generations. 
+This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. 
+
+## How it works
+At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences.
+
+The differences between original and generated sentences are controlled by two class initialization parameters, `steps` and `k`.
+ - `steps`: number of random walk steps to do
+ - `k`: number of high probability replacements for the masked word to consider
+
+This process generates $k^steps$ new sentences. We then randomly select a subset of these, as specified by `max_outputs`.
 
 ## What tasks does it intend to benefit?
 This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 
@@ -25,46 +37,10 @@ The average bleu score of a distillbert model (fine-tuned on xsum) (model: "sshl
 on a subset (10%) of xsum test dataset = 14.9104
 The average bleu score of same model on the pertubed set = 11.9221
 
-## Previous Work
-1) Butter Finger implementation borrowed from this code https://github.com/alexyorke/butter-fingers
+## What are the limitations of this transformation?
+
+This transformation can generate nonsensical words when the random walk has many steps (steps >~ number of words in sentence).
 
-2) There has also been some recent work in the contrast sets of the GEM Benchmark (ACL 2021):
-```bibtex
-@article{DBLP:journals/corr/abs-2102-01672,
-  title     = {The {GEM} Benchmark: Natural Language Generation, its Evaluation and
-               Metrics},
-  journal   = {CoRR},
-  volume    = {abs/2102.01672},
-  year      = {2021},
-  url       = {https://arxiv.org/abs/2102.01672},
-  archivePrefix = {arXiv},
-  eprint    = {2102.01672},
-  timestamp = {Tue, 16 Feb 2021 16:58:52 +0100},
-  biburl    = {https://dblp.org/rec/journals/corr/abs-2102-01672.bib},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
-}
-```
+## References
+1) Saketh Kotamraju, "How to use BERT from the Hugging face transformer library", https://towardsdatascience.com/how-to-use-bert-from-the-hugging-face-transformer-library-d373a22b0209
 
-3) There has been some recent work in NoiseQA too:
-```bibtex
-@inproceedings{ravichander-etal-2021-noiseqa,
-    title = "{N}oise{QA}: Challenge Set Evaluation for User-Centric Question Answering",
-    author = "Ravichander, Abhilasha  and
-      Dalmia, Siddharth  and
-      Ryskina, Maria  and
-      Metze, Florian  and
-      Hovy, Eduard  and
-      Black, Alan W",
-    booktitle = "Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume",
-    month = apr,
-    year = "2021",
-    address = "Online",
-    publisher = "Association for Computational Linguistics",
-    url = "https://www.aclweb.org/anthology/2021.eacl-main.259",
-    pages = "2976--2992",
-    abstract = "When Question-Answering (QA) systems are deployed in the real world, users query them through a variety of interfaces, such as speaking to voice assistants, typing questions into a search engine, or even translating questions to languages supported by the QA system. While there has been significant community attention devoted to identifying correct answers in passages assuming a perfectly formed question, we show that components in the pipeline that precede an answering engine can introduce varied and considerable sources of error, and performance can degrade substantially based on these upstream noise sources even for powerful pre-trained QA models. We conclude that there is substantial room for progress before QA systems can be effectively deployed, highlight the need for QA evaluation to expand to consider real-world use, and hope that our findings will spur greater community interest in the issues that arise when our systems actually need to be of utility to humans.",
-}
-```
-## What are the limitations of this transformation?
-The transformation's outputs are too simple to be used for data augmentation. Unlike a paraphraser, it is not capable of
- generating linguistically diverse text.
diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json
index a42c6d52c..17ff5b6a8 100644
--- a/transformations/random_walk/test.json
+++ b/transformations/random_walk/test.json
@@ -1,50 +1,90 @@
 {
-  "type": "butter_fingers_perturbation",
+  "type": "random_walk",
   "test_cases": [
     {
-      "class": "ButterFingersPerturbation",
+      "class": "RandomWalk",
       "inputs": {
         "sentence": "Andrew finally returned the French book to Chris that I bought last week"
       },
-      "outputs": [{
-        "sentence": "Andgew finally returned the French book to Chrus thav I bought last week"
-      }]
+      "outputs": [
+        {
+          "sentence": "He finally returned the comic book from Chris that she bought last week"
+        },
+        {
+          "sentence": "I finally returned my last book to Chris that I bought last !"
+        },
+        {
+          "sentence": "I have returned the French box to Chris that I bought last ."
+        }
+      ]
     },
     {
-      "class": "ButterFingersPerturbation",
+      "class": "RandomWalk",
       "inputs": {
         "sentence": "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments."
       },
-      "outputs": [{
-        "sentence": "Senhences with gapping, such as Paul likes coffwe and Mary tea, lack an overt predicate to indicatx the relauion between two or more argumentd."
-      }]
+      "outputs": [
+        {
+          "sentence": "Languages with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to describe the relation between two or more arguments."
+        },
+        {
+          "sentence": "Sentences with gapping, so as Paul likes John and Mary tea, lack an overt way of indicate the relation between two or more arguments."
+        },
+        {
+          "sentence": "Sentences with arguments, so as Paul likes Peter andd Mary tea, lack an overt predicate to indicate the relation between two / more arguments."
+        }
+      ]
     },
     {
-      "class": "ButterFingersPerturbation",
+      "class": "RandomWalk",
       "inputs": {
         "sentence": "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film"
       },
-      "outputs": [{
-        "sentence": "Alife in Wonderland is a 2010 American live-action/abimatev dark fantasy adventure film"
-      }]
+      "outputs": [
+        {
+          "sentence": "Alice in Wonderland is a 2010 Canadian live-action/animated / fantasy fantasy film"
+        },
+        {
+          "sentence": "Alice in Wonderland was a 2010 American live-action/animated musical - adventure film"
+        },
+        {
+          "sentence": "Alice Into Darkness , a 2010 American independent dark fantasy horror film"
+        }
+      ]
     },
     {
-      "class": "ButterFingersPerturbation",
+      "class": "RandomWalk",
       "inputs": {
         "sentence": "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001"
       },
-      "outputs": [{
-        "sentence": "Ujjwl Dev Dosanjh served as 33rd Premier of Britush Conumbia from 2000 to 2001"
-      }]
+      "outputs": [
+        {
+          "sentence": "Shankar Dev Dosanjh served in 33rd Premier of British Columbia from 1999 to 2001"
+        },
+        {
+          "sentence": "Jai Dev Dosanjh served as 33rd Premier of Canadian Columbia , 2000 to 2001"
+        },
+        {
+          "sentence": "Ram Dev Dosanjh served as 33rd Premier of Sri Columbia from 1997 to 2001"
+        }
+      ]
     },
     {
-      "class": "ButterFingersPerturbation",
+      "class": "RandomWalk",
       "inputs": {
         "sentence": "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."
       },
-      "outputs": [{
-        "sentence": "Neugoplasticity is a continuous processing alliwing vhort-term, medium-term, and long-term remodeling of the beuronosynaptic organization."
-      }]
+      "outputs": [
+        {
+          "sentence": "Neuroplasticity is a neural processing allowing rapid, intermediate, and long-term remodeling of the neuronosynaptic organization."
+        },
+        {
+          "sentence": "Th##R is a dynamic processing allowing short-term, intensive, and long-term remodeling of the neuronosynaptic organization."
+        },
+        {
+          "sentence": "Neuroplasticity is a continuous transformation allowing immediate, intermediate, and long-term remodeling for the entire organization."
+        }
+      ]
     }
   ]
-}
+}
\ No newline at end of file
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 082c137db..68248668b 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -1,5 +1,12 @@
 import itertools
 import random
+from transformers import BertTokenizer, BertForMaskedLM
+from torch.nn import functional as F
+import torch
+import numpy as np
+import re
+import copy
+import random
 
 from interfaces.SentenceOperation import SentenceOperation
 from tasks.TaskTypes import TaskType
@@ -8,99 +15,76 @@
 Base Class for implementing the different input transformations a generation should be robust against.
 """
 
-
-def butter_finger(text, prob=0.1, keyboard="querty", seed=0, max_outputs=1):
-    random.seed(seed)
-    key_approx = {}
-
-    if keyboard == "querty":
-        key_approx["q"] = "qwasedzx"
-        key_approx["w"] = "wqesadrfcx"
-        key_approx["e"] = "ewrsfdqazxcvgt"
-        key_approx["r"] = "retdgfwsxcvgt"
-        key_approx["t"] = "tryfhgedcvbnju"
-        key_approx["y"] = "ytugjhrfvbnji"
-        key_approx["u"] = "uyihkjtgbnmlo"
-        key_approx["i"] = "iuojlkyhnmlp"
-        key_approx["o"] = "oipklujm"
-        key_approx["p"] = "plo['ik"
-
-        key_approx["a"] = "aqszwxwdce"
-        key_approx["s"] = "swxadrfv"
-        key_approx["d"] = "decsfaqgbv"
-        key_approx["f"] = "fdgrvwsxyhn"
-        key_approx["g"] = "gtbfhedcyjn"
-        key_approx["h"] = "hyngjfrvkim"
-        key_approx["j"] = "jhknugtblom"
-        key_approx["k"] = "kjlinyhn"
-        key_approx["l"] = "lokmpujn"
-
-        key_approx["z"] = "zaxsvde"
-        key_approx["x"] = "xzcsdbvfrewq"
-        key_approx["c"] = "cxvdfzswergb"
-        key_approx["v"] = "vcfbgxdertyn"
-        key_approx["b"] = "bvnghcftyun"
-        key_approx["n"] = "nbmhjvgtuik"
-        key_approx["m"] = "mnkjloik"
-        key_approx[" "] = " "
-    else:
-        print("Keyboard not supported.")
-
-    prob_of_typo = int(prob * 100)
-    perturbed_texts = []
-    for _ in itertools.repeat(None, max_outputs):
-        butter_text = ""
-        for letter in text:
-            lcletter = letter.lower()
-            if lcletter not in key_approx.keys():
-                new_letter = lcletter
-            else:
-                if random.choice(range(0, 100)) <= prob_of_typo:
-                    new_letter = random.choice(key_approx[lcletter])
-                else:
-                    new_letter = lcletter
-            # go back to original case
-            if not lcletter == letter:
-                new_letter = new_letter.upper()
-            butter_text += new_letter
-        perturbed_texts.append(butter_text)
-    return perturbed_texts
-
-
-"""
-Butter Finger implementation borrowed from https://github.com/alexyorke/butter-fingers.
-"""
-
-
-class ButterFingersPerturbation(SentenceOperation):
+def random_walk(original_text, steps, k, tokenizer, model):
+    sentences = []
+    old_sentences = [original_text]
+    for i in range(steps): # Do k steps of random walk procedure
+        sentences = []
+        for text in old_sentences:
+            text_split = re.split('[ ?.,!;"]', text)
+            splits = len(text_split)
+            index_to_mask = np.random.randint(splits)
+            while len(text_split[index_to_mask]) == 0:
+                index_to_mask = np.random.randint(splits)
+            word_to_mask = text_split[index_to_mask]
+            # print('Word to mask:', word_to_mask)
+            start_index = text.find(word_to_mask)
+            new_text = text[0:start_index] + tokenizer.mask_token + text[start_index + len(word_to_mask):]
+
+            inputs = tokenizer.encode_plus(new_text, return_tensors='pt')
+            index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
+            outputs = model(**inputs)
+            logits = outputs.logits
+            softmax = F.softmax(logits, dim=-1)
+            mask_word = softmax[0, index_to_mask, :]
+            top_k = torch.topk(mask_word, k)[1][0]
+
+            for token in top_k:
+                word = tokenizer.decode([token])
+                new_sentence = new_text.replace(tokenizer.mask_token, word)
+                sentences.append(new_sentence)
+
+        old_sentences = copy.deepcopy(sentences)
+    assert len(sentences) == k**steps
+    return sentences
+
+
+
+
+class RandomWalk(SentenceOperation):
     tasks = [
-        TaskType.TEXT_CLASSIFICATION,
         TaskType.TEXT_TO_TEXT_GENERATION,
-        TaskType.TEXT_TAGGING,
     ]
     languages = ["en"]
 
-    def __init__(self, seed=0, max_outputs=1):
+    def __init__(self, seed=0, max_outputs=1, steps=5, k=2):
         super().__init__(seed, max_outputs=max_outputs)
+        self.tokenizer = BertTokenizer.from_pretrained('bert-large-cased')
+        self.model = BertForMaskedLM.from_pretrained('bert-large-cased')
+        self.max_outputs = max_outputs
+        self.steps = steps
+        self.k = k
 
     def generate(self, sentence: str):
-        perturbed_texts = butter_finger(
-            text=sentence,
-            prob=0.05,
-            seed=self.seed,
-            max_outputs=self.max_outputs,
+        perturbed_texts = random_walk(
+            original_text=sentence,
+            steps=self.steps,
+            k=self.k,
+            tokenizer=self.tokenizer,
+            model=self.model
         )
+        if len(perturbed_texts) > self.max_outputs:
+            perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
         return perturbed_texts
 
 
-"""
 # Sample code to demonstrate usage. Can also assist in adding test cases.
 # You don't need to keep this code in your transformation.
 if __name__ == '__main__':
     import json
     from TestRunner import convert_to_snake_case
 
-    tf = ButterFingersPerturbation(max_outputs=3)
+    tf = RandomWalk(max_outputs=3, k=2, steps=5)
     sentence = "Andrew finally returned the French book to Chris that I bought last week"
     test_cases = []
     for sentence in ["Andrew finally returned the French book to Chris that I bought last week",
@@ -114,4 +98,6 @@ def generate(self, sentence: str):
         )
     json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
     print(json.dumps(json_file, indent=2))
-"""
+
+    with open('test.json', 'w') as f:
+        json.dump(json_file, f, indent=2)

From 90b1f43bf47200f3a55715430966840c7a212cee Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Wed, 1 Sep 2021 15:31:38 -0700
Subject: [PATCH 03/19] No generated punctuation.

---
 transformations/random_walk/requirements.txt  |   2 +
 transformations/random_walk/test.json         |  30 ++--
 transformations/random_walk/transformation.py | 166 ++++++++++++++----
 3 files changed, 145 insertions(+), 53 deletions(-)
 create mode 100644 transformations/random_walk/requirements.txt

diff --git a/transformations/random_walk/requirements.txt b/transformations/random_walk/requirements.txt
new file mode 100644
index 000000000..0d3b780b8
--- /dev/null
+++ b/transformations/random_walk/requirements.txt
@@ -0,0 +1,2 @@
+sentence-transformers==2.0.0
+transformers==4.6.0
diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json
index 17ff5b6a8..aa840ffe0 100644
--- a/transformations/random_walk/test.json
+++ b/transformations/random_walk/test.json
@@ -8,13 +8,13 @@
       },
       "outputs": [
         {
-          "sentence": "He finally returned the comic book from Chris that she bought last week"
+          "sentence": "She finally gives the French book to Chris that I bought last and"
         },
         {
-          "sentence": "I finally returned my last book to Chris that I bought last !"
+          "sentence": "Andrew finally returned the French book to Chris which he bought last the"
         },
         {
-          "sentence": "I have returned the French box to Chris that I bought last ."
+          "sentence": "Andrew finally returned the French book from France which I bought last week"
         }
       ]
     },
@@ -25,13 +25,13 @@
       },
       "outputs": [
         {
-          "sentence": "Languages with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to describe the relation between two or more arguments."
+          "sentence": "Sentences with gapping, such as Paul likes coffee and Mary leaves, lack an overt way to indicate the difference between two or more arguments."
         },
         {
-          "sentence": "Sentences with gapping, so as Paul likes John and Mary tea, lack an overt way of indicate the relation between two or more arguments."
+          "sentence": "Words with gapping, such as Paul likes coffee and Mary leaves, lack an overt meaning to indicate the relation between two or more arguments."
         },
         {
-          "sentence": "Sentences with arguments, so as Paul likes Peter andd Mary tea, lack an overt predicate to indicate the relation between two / more arguments."
+          "sentence": "Lines with gapping, such as Paul likes coffee and Mary leaves, lack an overt meaning to indicate the relation between two or more arguments."
         }
       ]
     },
@@ -42,13 +42,13 @@
       },
       "outputs": [
         {
-          "sentence": "Alice in Wonderland is a 2010 Canadian live-action/animated / fantasy fantasy film"
+          "sentence": "Alice In Wonderland is a 2010 American live-action/animated musical fantasy adventure film"
         },
         {
-          "sentence": "Alice in Wonderland was a 2010 American live-action/animated musical - adventure film"
+          "sentence": "Alice in Wonderlandnd is a 2010 American live-action/animated romantic fantasy adventure film"
         },
         {
-          "sentence": "Alice Into Darkness , a 2010 American independent dark fantasy horror film"
+          "sentence": "Alice in Wonderland is a 2010 American animated musical fantasy adventure film"
         }
       ]
     },
@@ -59,13 +59,13 @@
       },
       "outputs": [
         {
-          "sentence": "Shankar Dev Dosanjh served in 33rd Premier of British Columbia from 1999 to 2001"
+          "sentence": "Ujjal Singh Dosanjh served as 33rd Premier of British Columbia from 2000 until 2003"
         },
         {
-          "sentence": "Jai Dev Dosanjh served as 33rd Premier of Canadian Columbia , 2000 to 2001"
+          "sentence": "Ujjal Dev Dosanjh served the 33rd Premier of British Columbia in 2000 to 2003"
         },
         {
-          "sentence": "Ram Dev Dosanjh served as 33rd Premier of Sri Columbia from 1997 to 2001"
+          "sentence": "Ujjal Dev Dosanjh served the 33rd Governor of British Columbia in 2000 to 2002"
         }
       ]
     },
@@ -76,13 +76,13 @@
       },
       "outputs": [
         {
-          "sentence": "Neuroplasticity is a neural processing allowing rapid, intermediate, and long-term remodeling of the neuronosynaptic organization."
+          "sentence": "Neuroplandndsticity is a cognitive processing allowing short-term, spontaneous, and rapid remodeling of the neuronosynaptic organization."
         },
         {
-          "sentence": "Th##R is a dynamic processing allowing short-term, intensive, and long-term remodeling of the neuronosynaptic organization."
+          "sentence": "Neuroplasticity involves a continuous processing and short, medium-term, or continuous remodeling of the neuronosynaptic organization."
         },
         {
-          "sentence": "Neuroplasticity is a continuous transformation allowing immediate, intermediate, and long-term remodeling for the entire organization."
+          "sentence": "Neuroplorsticity is a continuous processing allowing immediate, medium-term, and continuous remodeling of the neural organization."
         }
       ]
     }
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 68248668b..9d15e84b3 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -1,55 +1,130 @@
-import itertools
+#import itertools
 import random
-from transformers import BertTokenizer, BertForMaskedLM
-from torch.nn import functional as F
-import torch
 import numpy as np
 import re
 import copy
 import random
+from typing import List
+
+from transformers import BertTokenizer, BertForMaskedLM
+from sentence_transformers import SentenceTransformer, util
+
+from torch.nn import functional as F
+import torch
 
 from interfaces.SentenceOperation import SentenceOperation
 from tasks.TaskTypes import TaskType
 
-"""
-Base Class for implementing the different input transformations a generation should be robust against.
-"""
 
-def random_walk(original_text, steps, k, tokenizer, model):
-    sentences = []
+def _mask_word(sentence, word_to_mask, tokenizer):
+    """ helper function,
+    replace word in a sentence with mask-token, as prep for BERT tokenizer"""
+    start_index = sentence.find(word_to_mask)
+    return sentence[0:start_index] + tokenizer.mask_token + sentence[
+        start_index + len(word_to_mask):]
+
+
+def get_k_replacement_words(tokenized_text, tokenizer, model, k):
+    """return k most similar words from the model, for a tokenized mask-word in a sentence.
+
+    Args:
+        tokenized_text (str): sentence with a word masked out
+        model ([type]): model
+        tokenizer ([type]): tokenizer
+        k (int, optional): how many similar words to find for a given tokenized-word. Defaults to 5.
+
+    Returns:
+        [list]: list of top k words
+    """
+    inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt')
+    index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
+    outputs = model(**inputs)
+    softmax = F.softmax(outputs.logits, dim=-1)
+    mask_word = softmax[0, index_to_mask, :]
+
+    #return torch.topk(mask_word, k)[1][0]
+
+    sorted_tokens = torch.argsort(mask_word[0], descending=True)
+    i = 0
+    valid_tokens = []
+    while len(valid_tokens) < k and i < len(sorted_tokens):
+        if tokenizer.decode([sorted_tokens[i]]).isalnum():
+            valid_tokens.append(sorted_tokens[i])
+        i += 1
+    assert len(valid_tokens) == k or i == len(sorted_tokens)    # We either have found k valid (non punctuation) tokens or we have looked through all the tokens.
+    return valid_tokens
+
+def single_sentence_random_step(sentence, tokenizer, model, k):
+    """For a given sentence, choose a random word to mask, and
+    replace it with a word the top-k most similar words in BERT model.
+    Return k sentences, each with a different replacement word for the mask.
+
+    Args:
+        sentence ([type]): sentence to perform random walk on
+        tokenizer ([type]): tokenizer
+        model ([type]): model
+        k (int, optional): how many replacement words to try. Defaults to 5.
+
+    Returns:
+        [list]: k-sentences with masked word replaced with top-k most similar words
+    """
+    text_split = re.split('[ ?.,!;"]', sentence)
+
+    # pick a random word to mask
+    word_to_mask = random.choice(text_split)
+    while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
+        word_to_mask = random.choice(text_split)
+    # mask word
+    new_text = _mask_word(sentence, word_to_mask, tokenizer)
+
+    # get k replacement words
+    top_k = get_k_replacement_words(new_text, tokenizer, model, k=k)
+
+    # replace mask-token with the word from the top-k replacements
+    return [
+        new_text.replace(tokenizer.mask_token, tokenizer.decode([token]))
+        for token in top_k if tokenizer.decode([token]).isalnum()
+    ]
+
+
+def single_round(sentences: List[str], tokenizer, model, k) -> List[str]:
+    """For a given list of sentences, do a random walk on each sentence.
+
+    Args:
+        sentences ([type]): list of sentnces to perform random walk on
+        tokenizer ([type]): tokenizer
+        model ([type]): model
+
+    Returns:
+        [List]: list of random-walked sentences
+    """
+    new_sentences = []
+
+    for sentence in sentences:
+        new_sentences.extend(
+            single_sentence_random_step(sentence, tokenizer, model, k))
+
+    return new_sentences
+
+
+def random_walk(original_text: str, steps: int, k: int, tokenizer,
+                model) -> List[str]:
     old_sentences = [original_text]
-    for i in range(steps): # Do k steps of random walk procedure
-        sentences = []
-        for text in old_sentences:
-            text_split = re.split('[ ?.,!;"]', text)
-            splits = len(text_split)
-            index_to_mask = np.random.randint(splits)
-            while len(text_split[index_to_mask]) == 0:
-                index_to_mask = np.random.randint(splits)
-            word_to_mask = text_split[index_to_mask]
-            # print('Word to mask:', word_to_mask)
-            start_index = text.find(word_to_mask)
-            new_text = text[0:start_index] + tokenizer.mask_token + text[start_index + len(word_to_mask):]
-
-            inputs = tokenizer.encode_plus(new_text, return_tensors='pt')
-            index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
-            outputs = model(**inputs)
-            logits = outputs.logits
-            softmax = F.softmax(logits, dim=-1)
-            mask_word = softmax[0, index_to_mask, :]
-            top_k = torch.topk(mask_word, k)[1][0]
-
-            for token in top_k:
-                word = tokenizer.decode([token])
-                new_sentence = new_text.replace(tokenizer.mask_token, word)
-                sentences.append(new_sentence)
 
+    # Do k steps of random walk procedure
+    for _ in range(steps):
+        sentences = single_round(old_sentences, tokenizer, model, k)
         old_sentences = copy.deepcopy(sentences)
-    assert len(sentences) == k**steps
-    return sentences
 
+    #assert len(sentences) == k**steps  # This may not be possible if we cannot find k non-punctuation suggestions.
+    return sentences
 
+def sentence_similarity_metric(similarity_model, sen_A, sen_B):
+    emb_A = similarity_model.encode(sen_A)
+    emb_B = similarity_model.encode(sen_B)
 
+    score = util.pytorch_cos_sim(emb_A, emb_B)
+    return score
 
 class RandomWalk(SentenceOperation):
     tasks = [
@@ -57,15 +132,20 @@ class RandomWalk(SentenceOperation):
     ]
     languages = ["en"]
 
-    def __init__(self, seed=0, max_outputs=1, steps=5, k=2):
+    def __init__(self, seed=0, max_outputs=1, steps=5, k=2, sim_req=0.75):
+        random.seed(seed)
+        np.random.seed(seed)
         super().__init__(seed, max_outputs=max_outputs)
         self.tokenizer = BertTokenizer.from_pretrained('bert-large-cased')
         self.model = BertForMaskedLM.from_pretrained('bert-large-cased')
+        self.sim_model = SentenceTransformer('all-mpnet-base-v2')
         self.max_outputs = max_outputs
         self.steps = steps
         self.k = k
+        self.sim_req = sim_req
 
     def generate(self, sentence: str):
+        print('Random walking on the sentence:', sentence)
         perturbed_texts = random_walk(
             original_text=sentence,
             steps=self.steps,
@@ -73,6 +153,16 @@ def generate(self, sentence: str):
             tokenizer=self.tokenizer,
             model=self.model
         )
+
+        scores = []
+        for o in perturbed_texts:
+            scores.append(sentence_similarity_metric(self.sim_model, sentence, o))
+            # print(o, scores[-1])
+        valid_sentences = np.array(scores) > self.sim_req
+        perturbed_texts = [o for o,s in zip(perturbed_texts, valid_sentences) if s]
+        assert np.sum(valid_sentences) == len(perturbed_texts)
+        #print('Number of valid sentences:', len(perturbed_texts))   # Could return 0 sentences if none are high enough in quality.
+
         if len(perturbed_texts) > self.max_outputs:
             perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
         return perturbed_texts
@@ -84,7 +174,7 @@ def generate(self, sentence: str):
     import json
     from TestRunner import convert_to_snake_case
 
-    tf = RandomWalk(max_outputs=3, k=2, steps=5)
+    tf = RandomWalk(max_outputs=3, k=2, steps=5, sim_req=0.25)
     sentence = "Andrew finally returned the French book to Chris that I bought last week"
     test_cases = []
     for sentence in ["Andrew finally returned the French book to Chris that I bought last week",

From 18c4ef9677117244eb1feef72021f15b95c91fb3 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Thu, 2 Sep 2021 13:52:45 -0700
Subject: [PATCH 04/19] Update requirements.txt

---
 transformations/random_walk/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformations/random_walk/requirements.txt b/transformations/random_walk/requirements.txt
index 0d3b780b8..ebe278d3c 100644
--- a/transformations/random_walk/requirements.txt
+++ b/transformations/random_walk/requirements.txt
@@ -1,2 +1,2 @@
 sentence-transformers==2.0.0
-transformers==4.6.0
+transformers==4.6.1

From 5b8def20748f9d58e679bea74f3c7d1e101bfbe7 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Sat, 4 Sep 2021 15:51:14 +0000
Subject: [PATCH 05/19] Passing pytest.

---
 transformations/random_walk/transformation.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 9d15e84b3..d887641f8 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -132,7 +132,8 @@ class RandomWalk(SentenceOperation):
     ]
     languages = ["en"]
 
-    def __init__(self, seed=0, max_outputs=1, steps=5, k=2, sim_req=0.75):
+    # Default parameters match those of the 'test.json' below.
+    def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25):
         random.seed(seed)
         np.random.seed(seed)
         super().__init__(seed, max_outputs=max_outputs)
@@ -157,17 +158,16 @@ def generate(self, sentence: str):
         scores = []
         for o in perturbed_texts:
             scores.append(sentence_similarity_metric(self.sim_model, sentence, o))
-            # print(o, scores[-1])
         valid_sentences = np.array(scores) > self.sim_req
         perturbed_texts = [o for o,s in zip(perturbed_texts, valid_sentences) if s]
         assert np.sum(valid_sentences) == len(perturbed_texts)
-        #print('Number of valid sentences:', len(perturbed_texts))   # Could return 0 sentences if none are high enough in quality.
 
         if len(perturbed_texts) > self.max_outputs:
             perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
         return perturbed_texts
 
-
+"""
+# The code to produce 'test.json' must be commented out so that pytest succeeds.
 # Sample code to demonstrate usage. Can also assist in adding test cases.
 # You don't need to keep this code in your transformation.
 if __name__ == '__main__':
@@ -191,3 +191,4 @@ def generate(self, sentence: str):
 
     with open('test.json', 'w') as f:
         json.dump(json_file, f, indent=2)
+"""
\ No newline at end of file

From e06451e89c09cf57d469814350c119f83b45bcac Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Sat, 4 Sep 2021 09:38:29 -0700
Subject: [PATCH 06/19] Update README.md

---
 transformations/random_walk/README.md | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index dffe97630..55f7502e2 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -9,10 +9,10 @@ Author names:
 
 ## Extras
 
-This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library. Provided that the library is installed, this model will be installed the first time this transformation is ran.
+This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers model. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran.
 
 ## What type of a transformation is this?
-This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. 
+This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. We measure the similarity between the original and random-walked sentence by performing sentence embeddings and then calculate the cosine similarity bewteen the embedded vectors.
 
 ## How it works
 At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences.
@@ -23,19 +23,13 @@ The differences between original and generated sentences are controlled by two c
 
 This process generates $k^steps$ new sentences. We then randomly select a subset of these, as specified by `max_outputs`.
 
-## What tasks does it intend to benefit?
-This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 
-text generation, etc. 
+The seed of the random generators (both from 'numpy' and the 'random' module) are set by the 'seed' parameter in the class initializer. Choosing a fixed value will lead to reproducable results.
 
-```python evaluate.py -t ButterFingersPerturbation -task TEXT_CLASSIFICATION```
-```model_name = "aychang/roberta-base-imdb"```
-The accuracy of a RoBERTa model (fine-tuned on IMDB) (model: "aychang/roberta-base-imdb") 
-on a subset of IMDB sentiment dataset = 95.74
-The accuracy of the same model on the perturbed set = 88.26
+The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter 'sim_req' which is the minimum similarity score that a generated sentence must have to be considered valid.
 
-The average bleu score of a distillbert model (fine-tuned on xsum) (model: "sshleifer/distilbart-xsum-12-6") 
-on a subset (10%) of xsum test dataset = 14.9104
-The average bleu score of same model on the pertubed set = 11.9221
+## What tasks does it intend to benefit?
+This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 
+text generation, etc. Evaluating the perturbation using Google Colab is currently in progress.
 
 ## What are the limitations of this transformation?
 

From 2fd47531f6125075767ba410409d1705d915cc5c Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Sat, 4 Sep 2021 09:39:57 -0700
Subject: [PATCH 07/19] Update README.md

---
 transformations/random_walk/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index 55f7502e2..ef6594f40 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -2,10 +2,11 @@
 This transformation performs a random walk on the original sentence by randomly masking a word and replacing it with a suggestion by the BERT languange model.
 
 Author names:
- - Chandan Singh (chandan_singh@berkeley.edu, UC Berkeley)
- - Jamie Simon (james.simon@berkeley.edu, UC Berkeley)
+        
  - Sajant Anand (sajant@berkeley.edu, UC Berkeley)
  - Roy Rinberg (royrinberg@gmail.com, Columbia University)
+ - Jamie Simon (james.simon@berkeley.edu, UC Berkeley)
+ - Chandan Singh (chandan_singh@berkeley.edu, UC Berkeley)
 
 ## Extras
 

From 3d61fde921333528b6049da2a2cc28ca5392c910 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Sat, 4 Sep 2021 09:40:23 -0700
Subject: [PATCH 08/19] Update transformation.py

---
 transformations/random_walk/transformation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index d887641f8..67a8310ee 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -129,6 +129,7 @@ def sentence_similarity_metric(similarity_model, sen_A, sen_B):
 class RandomWalk(SentenceOperation):
     tasks = [
         TaskType.TEXT_TO_TEXT_GENERATION,
+        TaskType.TEXT_CLASSIFICATION
     ]
     languages = ["en"]
 
@@ -191,4 +192,4 @@ def generate(self, sentence: str):
 
     with open('test.json', 'w') as f:
         json.dump(json_file, f, indent=2)
-"""
\ No newline at end of file
+"""

From 41cb2fff54688b5600f94084b77bb52fc3a8eb06 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Sat, 4 Sep 2021 17:03:21 +0000
Subject: [PATCH 09/19] Truncating long sequences.

---
 transformations/random_walk/transformation.py | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index d887641f8..81a559e2d 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -36,7 +36,7 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
     Returns:
         [list]: list of top k words
     """
-    inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt')
+    inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512)
     index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
     outputs = model(**inputs)
     softmax = F.softmax(outputs.logits, dim=-1)
@@ -52,7 +52,7 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
             valid_tokens.append(sorted_tokens[i])
         i += 1
     assert len(valid_tokens) == k or i == len(sorted_tokens)    # We either have found k valid (non punctuation) tokens or we have looked through all the tokens.
-    return valid_tokens
+    return valid_tokens, True
 
 def single_sentence_random_step(sentence, tokenizer, model, k):
     """For a given sentence, choose a random word to mask, and
@@ -69,16 +69,19 @@ def single_sentence_random_step(sentence, tokenizer, model, k):
         [list]: k-sentences with masked word replaced with top-k most similar words
     """
     text_split = re.split('[ ?.,!;"]', sentence)
-
-    # pick a random word to mask
-    word_to_mask = random.choice(text_split)
-    while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
-        word_to_mask = random.choice(text_split)
-    # mask word
-    new_text = _mask_word(sentence, word_to_mask, tokenizer)
-
-    # get k replacement words
-    top_k = get_k_replacement_words(new_text, tokenizer, model, k=k)
+    included_mask = False
+    while not included_mask:
+      # pick a random word to mask
+      word_to_mask = random.choice(text_split)
+      start_index = 0
+      while len(word_to_mask) == 0 and not word_to_mask.isalnum() and start_index < 512: # Avoid empty strings in split text
+          word_to_mask = random.choice(text_split)
+          #start_index = sentence.find(word_to_mask)
+      # mask word
+      new_text = _mask_word(sentence, word_to_mask, tokenizer)
+
+      # get k replacement words
+      top_k, inluded_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
 
     # replace mask-token with the word from the top-k replacements
     return [
@@ -129,6 +132,7 @@ def sentence_similarity_metric(similarity_model, sen_A, sen_B):
 class RandomWalk(SentenceOperation):
     tasks = [
         TaskType.TEXT_TO_TEXT_GENERATION,
+        TaskType.TEXT_CLASSIFICATION
     ]
     languages = ["en"]
 

From 38663ed5c9fad02834f36b34ed7f1e3934d71fd5 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Sat, 4 Sep 2021 17:58:06 +0000
Subject: [PATCH 10/19] Working truncation of very long messages.

---
 transformations/random_walk/transformation.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index f2354a5eb..9e54e82ec 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -38,12 +38,12 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
     """
     inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512)
     index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
+    if index_to_mask[0].numel() == 0: # Since we are truncating the input to be 512 tokens (BERT's max), we need to make sure the mask is in these first 512.
+        return None, False
     outputs = model(**inputs)
     softmax = F.softmax(outputs.logits, dim=-1)
     mask_word = softmax[0, index_to_mask, :]
 
-    #return torch.topk(mask_word, k)[1][0]
-
     sorted_tokens = torch.argsort(mask_word[0], descending=True)
     i = 0
     valid_tokens = []
@@ -70,18 +70,16 @@ def single_sentence_random_step(sentence, tokenizer, model, k):
     """
     text_split = re.split('[ ?.,!;"]', sentence)
     included_mask = False
-    while not included_mask:
+    while not included_mask: # Loop until the masked word is one of the first 512 tokens of input, since all else are discarded.
       # pick a random word to mask
       word_to_mask = random.choice(text_split)
-      start_index = 0
-      while len(word_to_mask) == 0 and not word_to_mask.isalnum() and start_index < 512: # Avoid empty strings in split text
+      while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
           word_to_mask = random.choice(text_split)
-          #start_index = sentence.find(word_to_mask)
       # mask word
       new_text = _mask_word(sentence, word_to_mask, tokenizer)
 
       # get k replacement words
-      top_k, inluded_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+      top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
 
     # replace mask-token with the word from the top-k replacements
     return [

From bc5ff32b52b3116f88c43e5881ac4651c9f480b2 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Sat, 4 Sep 2021 11:11:05 -0700
Subject: [PATCH 11/19] Update README.md

---
 transformations/random_walk/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index ef6594f40..c668e3210 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -16,7 +16,7 @@ This transformation requires the 'bert-large-cased' pretrained model (~1 GB) fro
 This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. We measure the similarity between the original and random-walked sentence by performing sentence embeddings and then calculate the cosine similarity bewteen the embedded vectors.
 
 ## How it works
-At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences.
+At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences. Note that BERT has a max input token length of 512, so for long inputs, we truncate to the first 512 tokens.
 
 The differences between original and generated sentences are controlled by two class initialization parameters, `steps` and `k`.
  - `steps`: number of random walk steps to do
@@ -24,9 +24,9 @@ The differences between original and generated sentences are controlled by two c
 
 This process generates $k^steps$ new sentences. We then randomly select a subset of these, as specified by `max_outputs`.
 
-The seed of the random generators (both from 'numpy' and the 'random' module) are set by the 'seed' parameter in the class initializer. Choosing a fixed value will lead to reproducable results.
+The seed of the random generators (both from `numpy` and the `random` module) are set by the `seed` parameter in the class initializer. Choosing a fixed value will lead to reproducable results.
 
-The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter 'sim_req' which is the minimum similarity score that a generated sentence must have to be considered valid.
+The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter `sim_req` which is the minimum similarity score that a generated sentence must have to be considered valid.
 
 ## What tasks does it intend to benefit?
 This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 

From 0f8569526877363404d18f0a7b8aea82b2fb5ce1 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Sun, 12 Sep 2021 01:36:34 +0000
Subject: [PATCH 12/19] Fix partial word masking.

---
 transformations/random_walk/transformation.py | 45 +++++++++++++------
 1 file changed, 32 insertions(+), 13 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 9e54e82ec..200f2a19a 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -15,13 +15,20 @@
 from interfaces.SentenceOperation import SentenceOperation
 from tasks.TaskTypes import TaskType
 
-
+'''
 def _mask_word(sentence, word_to_mask, tokenizer):
     """ helper function,
     replace word in a sentence with mask-token, as prep for BERT tokenizer"""
     start_index = sentence.find(word_to_mask)
     return sentence[0:start_index] + tokenizer.mask_token + sentence[
         start_index + len(word_to_mask):]
+'''
+
+def _mask_word(sentence, split_indices, mask):
+    """ helper function,
+    replace word in a sentence with mask-token, as prep for BERT tokenizer"""
+    return sentence[0:split_indices[0]] + mask + sentence[
+        split_indices[1]:]
 
 
 def get_k_replacement_words(tokenized_text, tokenizer, model, k):
@@ -68,23 +75,35 @@ def single_sentence_random_step(sentence, tokenizer, model, k):
     Returns:
         [list]: k-sentences with masked word replaced with top-k most similar words
     """
-    text_split = re.split('[ ?.,!;"]', sentence)
+    #text_split = re.split('[ ?.,!;"]', sentence)
+    split_iter = re.finditer(r"[\w']+|[.,!?;]", sentence)
+    text_split = []
+    split_indices = []
+    for m in split_iter:
+      text_split.append(m.group(0))
+      split_indices.append((m.start(), m.end()))
+
     included_mask = False
     while not included_mask: # Loop until the masked word is one of the first 512 tokens of input, since all else are discarded.
       # pick a random word to mask
-      word_to_mask = random.choice(text_split)
+      
+      rand_int = np.random.randint(len(text_split))
+      word_to_mask = text_split[rand_int]
       while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
-          word_to_mask = random.choice(text_split)
+          rand_int = np.random.randint(len(text_split))
+          word_to_mask = text_split[rand_int]
+      
       # mask word
-      new_text = _mask_word(sentence, word_to_mask, tokenizer)
+      new_text = _mask_word(sentence, split_indices[rand_int], tokenizer.mask_token)
 
       # get k replacement words
       top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+    replacement_words = [tokenizer.decode([token]) for token in top_k]
 
     # replace mask-token with the word from the top-k replacements
     return [
-        new_text.replace(tokenizer.mask_token, tokenizer.decode([token]))
-        for token in top_k if tokenizer.decode([token]).isalnum()
+        new_text.replace(tokenizer.mask_token, word)
+        for word in replacement_words
     ]
 
 
@@ -168,7 +187,7 @@ def generate(self, sentence: str):
             perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
         return perturbed_texts
 
-"""
+
 # The code to produce 'test.json' must be commented out so that pytest succeeds.
 # Sample code to demonstrate usage. Can also assist in adding test cases.
 # You don't need to keep this code in your transformation.
@@ -177,9 +196,9 @@ def generate(self, sentence: str):
     from TestRunner import convert_to_snake_case
 
     tf = RandomWalk(max_outputs=3, k=2, steps=5, sim_req=0.25)
-    sentence = "Andrew finally returned the French book to Chris that I bought last week"
+    #sentence = "Andrew finally returned the French book to Chris that I bought last week"
     test_cases = []
-    for sentence in ["Andrew finally returned the French book to Chris that I bought last week",
+    for sentence in ["Andrew finally returned the French book to Chris that I bought last week.",
                      "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments.",
                      "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film",
                      "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001",
@@ -191,6 +210,6 @@ def generate(self, sentence: str):
     json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
     print(json.dumps(json_file, indent=2))
 
-    with open('test.json', 'w') as f:
-        json.dump(json_file, f, indent=2)
-"""
+    #with open('test.json', 'w') as f:
+    #    json.dump(json_file, f, indent=2)
+

From 4a1450d5003670b787090cd632bf2f0f911e1765 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Sun, 12 Sep 2021 01:39:08 +0000
Subject: [PATCH 13/19] New json.

---
 transformations/random_walk/test.json         | 32 +++++++++----------
 transformations/random_walk/transformation.py |  8 ++---
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json
index aa840ffe0..f6c7a6239 100644
--- a/transformations/random_walk/test.json
+++ b/transformations/random_walk/test.json
@@ -4,17 +4,17 @@
     {
       "class": "RandomWalk",
       "inputs": {
-        "sentence": "Andrew finally returned the French book to Chris that I bought last week"
+        "sentence": "Andrew finally returned the French book to Chris that I bought last week."
       },
       "outputs": [
         {
-          "sentence": "She finally gives the French book to Chris that I bought last and"
+          "sentence": "She finally returned the French book to me that I bought last year."
         },
         {
-          "sentence": "Andrew finally returned the French book to Chris which he bought last the"
+          "sentence": "She finally returned the French book for me that I bought last year."
         },
         {
-          "sentence": "Andrew finally returned the French book from France which I bought last week"
+          "sentence": "Andrew finally returned the French box to Chris that I bought last week."
         }
       ]
     },
@@ -25,13 +25,13 @@
       },
       "outputs": [
         {
-          "sentence": "Sentences with gapping, such as Paul likes coffee and Mary leaves, lack an overt way to indicate the difference between two or more arguments."
+          "sentence": "Lines with gappinglines such as Paul likes coffee and Mary tea, lack an overt attempt to indicate a relation between two or more arguments."
         },
         {
-          "sentence": "Words with gapping, such as Paul likes coffee and Mary leaves, lack an overt meaning to indicate the relation between two or more arguments."
+          "sentence": "Sentences with gapping, in that Paul drinks coffee and Mary tea, lack an overt attempt to show the relation between two or more arguments."
         },
         {
-          "sentence": "Lines with gapping, such as Paul likes coffee and Mary leaves, lack an overt meaning to indicate the relation between two or more arguments."
+          "sentence": "Sentences with women, such that Paul likes coffee and Mary tea, lack any overt attempt to explain the relation between two or more arguments."
         }
       ]
     },
@@ -42,13 +42,13 @@
       },
       "outputs": [
         {
-          "sentence": "Alice In Wonderland is a 2010 American live-action/animated musical fantasy adventure film"
+          "sentence": "Alice In Wonderland is a 2014 American live-action/animated dark fantasy adventure film"
         },
         {
-          "sentence": "Alice in Wonderlandnd is a 2010 American live-action/animated romantic fantasy adventure film"
+          "sentence": "Alice in Wonderland is a 2015 Canadian live-action/animation dark fantasy adventure film"
         },
         {
-          "sentence": "Alice in Wonderland is a 2010 American animated musical fantasy adventure film"
+          "sentence": "Alice In Wonderland is a 2015 American live-action/animated dark fantasy adventure film"
         }
       ]
     },
@@ -59,13 +59,13 @@
       },
       "outputs": [
         {
-          "sentence": "Ujjal Singh Dosanjh served as 33rd Premier of British Columbia from 2000 until 2003"
+          "sentence": "Ujjal Dev Dosanjh served the 13th Premier of British Columbia from 2000 until 2001"
         },
         {
-          "sentence": "Ujjal Dev Dosanjh served the 33rd Premier of British Columbia in 2000 to 2003"
+          "sentence": "Ujjal Dev Dosanjh was the 33rd Premier of British Columbia from 2000 to 2001"
         },
         {
-          "sentence": "Ujjal Dev Dosanjh served the 33rd Governor of British Columbia in 2000 to 2002"
+          "sentence": "Ujjal Dev has served as 33rd premier in British Columbia from 2000 to 2001"
         }
       ]
     },
@@ -76,13 +76,13 @@
       },
       "outputs": [
         {
-          "sentence": "Neuroplandndsticity is a cognitive processing allowing short-term, spontaneous, and rapid remodeling of the neuronosynaptic organization."
+          "sentence": "Neuroplasticity is a continuous processing allowing short-term, medium-term, and longer-term remodeling of the neuronosynaptic organization."
         },
         {
-          "sentence": "Neuroplasticity involves a continuous processing and short, medium-term, or continuous remodeling of the neuronosynaptic organization."
+          "sentence": "Neuroplasticity is a continuous processing allowing short-duration, medium-termstability and long-term remodeling of the neuronosynaptic network."
         },
         {
-          "sentence": "Neuroplorsticity is a continuous processing allowing immediate, medium-term, and continuous remodeling of the neural organization."
+          "sentence": "It is a dynamic processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."
         }
       ]
     }
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 200f2a19a..251b30329 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -187,7 +187,7 @@ def generate(self, sentence: str):
             perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
         return perturbed_texts
 
-
+"""
 # The code to produce 'test.json' must be commented out so that pytest succeeds.
 # Sample code to demonstrate usage. Can also assist in adding test cases.
 # You don't need to keep this code in your transformation.
@@ -210,6 +210,6 @@ def generate(self, sentence: str):
     json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
     print(json.dumps(json_file, indent=2))
 
-    #with open('test.json', 'w') as f:
-    #    json.dump(json_file, f, indent=2)
-
+    with open('test.json', 'w') as f:
+        json.dump(json_file, f, indent=2)
+"""
\ No newline at end of file

From 0f361047ba6ac635753aeba31cb0eb72fc79cd66 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Tue, 21 Sep 2021 00:14:21 -0700
Subject: [PATCH 14/19] Address reviewer comments and improve documentatation.

---
 transformations/random_walk/transformation.py | 113 +++++++++++-------
 1 file changed, 72 insertions(+), 41 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 251b30329..b80172844 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -15,37 +15,40 @@
 from interfaces.SentenceOperation import SentenceOperation
 from tasks.TaskTypes import TaskType
 
-'''
-def _mask_word(sentence, word_to_mask, tokenizer):
-    """ helper function,
-    replace word in a sentence with mask-token, as prep for BERT tokenizer"""
-    start_index = sentence.find(word_to_mask)
-    return sentence[0:start_index] + tokenizer.mask_token + sentence[
-        start_index + len(word_to_mask):]
-'''
-
 def _mask_word(sentence, split_indices, mask):
-    """ helper function,
-    replace word in a sentence with mask-token, as prep for BERT tokenizer"""
+    """ helper function to replace word in a sentence with mask-token, as prep 
+    for BERT tokenizer
+    
+    Args:
+    sentence (str): sentence with work to mask
+    split_indices ([int, int]): index of word to replace, begining and character 
+      after the end indices
+    mask (BERT Token): token for a BERT mask
+    
+    """
     return sentence[0:split_indices[0]] + mask + sentence[
         split_indices[1]:]
 
 
 def get_k_replacement_words(tokenized_text, tokenizer, model, k):
-    """return k most similar words from the model, for a tokenized mask-word in a sentence.
+    """return k most similar words from the model, for a tokenized mask-word 
+    in a sentence.
 
     Args:
         tokenized_text (str): sentence with a word masked out
-        model ([type]): model
         tokenizer ([type]): tokenizer
-        k (int, optional): how many similar words to find for a given tokenized-word. Defaults to 5.
+        model ([type]): model
+        k (int): how many similar words to find for a given tokenized-word. Checks
+          that generated word is a composed of letters or numbers.
 
     Returns:
         [list]: list of top k words
     """
     inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512)
     index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
-    if index_to_mask[0].numel() == 0: # Since we are truncating the input to be 512 tokens (BERT's max), we need to make sure the mask is in these first 512.
+    if index_to_mask[0].numel() == 0: # Since we are truncating the input to be 
+        # 512 tokens (BERT's max), we need to make sure the mask is in these first 512.
+        # If not, return False so that we try again.
         return None, False
     outputs = model(**inputs)
     softmax = F.softmax(outputs.logits, dim=-1)
@@ -53,7 +56,8 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
 
     sorted_tokens = torch.argsort(mask_word[0], descending=True)
     i = 0
-    valid_tokens = []
+    valid_tokens = [] # The k most probable tokens are guaranteed to be words,
+    # so we make sure they are.
     while len(valid_tokens) < k and i < len(sorted_tokens):
         if tokenizer.decode([sorted_tokens[i]]).isalnum():
             valid_tokens.append(sorted_tokens[i])
@@ -70,34 +74,34 @@ def single_sentence_random_step(sentence, tokenizer, model, k):
         sentence ([type]): sentence to perform random walk on
         tokenizer ([type]): tokenizer
         model ([type]): model
-        k (int, optional): how many replacement words to try. Defaults to 5.
+        k (int): how many replacement words to try.
 
     Returns:
         [list]: k-sentences with masked word replaced with top-k most similar words
     """
-    #text_split = re.split('[ ?.,!;"]', sentence)
-    split_iter = re.finditer(r"[\w']+|[.,!?;]", sentence)
+
+    split_iter = re.finditer(r"[\w']+|[.,!?;]", sentence) # Split sentence on puctuation.
     text_split = []
     split_indices = []
     for m in split_iter:
-      text_split.append(m.group(0))
-      split_indices.append((m.start(), m.end()))
+        text_split.append(m.group(0))
+        split_indices.append((m.start(), m.end()))
 
     included_mask = False
-    while not included_mask: # Loop until the masked word is one of the first 512 tokens of input, since all else are discarded.
-      # pick a random word to mask
+    while not included_mask: # Loop until the masked word is one of the first 
+        # 512 tokens of input, since all else are discarded.
       
-      rand_int = np.random.randint(len(text_split))
-      word_to_mask = text_split[rand_int]
-      while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
-          rand_int = np.random.randint(len(text_split))
-          word_to_mask = text_split[rand_int]
-      
-      # mask word
-      new_text = _mask_word(sentence, split_indices[rand_int], tokenizer.mask_token)
-
-      # get k replacement words
-      top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+        rand_int = np.random.randint(len(text_split)) # pick a random word to mask
+        word_to_mask = text_split[rand_int]
+        while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
+            rand_int = np.random.randint(len(text_split))
+            word_to_mask = text_split[rand_int]
+        
+        # mask word
+        new_text = _mask_word(sentence, split_indices[rand_int], tokenizer.mask_token)
+
+        # get k replacement words
+        top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
     replacement_words = [tokenizer.decode([token]) for token in top_k]
 
     # replace mask-token with the word from the top-k replacements
@@ -114,9 +118,9 @@ def single_round(sentences: List[str], tokenizer, model, k) -> List[str]:
         sentences ([type]): list of sentnces to perform random walk on
         tokenizer ([type]): tokenizer
         model ([type]): model
-
+        k (int): how many words to sample to replace masked word
     Returns:
-        [List]: list of random-walked sentences
+        [List]: list of k random-walked sentences
     """
     new_sentences = []
 
@@ -129,17 +133,39 @@ def single_round(sentences: List[str], tokenizer, model, k) -> List[str]:
 
 def random_walk(original_text: str, steps: int, k: int, tokenizer,
                 model) -> List[str]:
-    old_sentences = [original_text]
+    """For a sentence, perform a random walk sequence on the sentence, generating
+    new sentences at each step and perturbing these during the next step.
 
-    # Do k steps of random walk procedure
+    Args:
+        original_text (str): original sentence we want to perturb
+        steps (int): how many random walks iterations we perform on the sentence
+        k (int): how many words to sample to replace masked word during each iteration
+        tokenizer ([type]): tokenizer
+        model ([type]): model
+    Returns:
+        [List]: list of steps^k random-walked sentences
+    """
+
+    old_sentences = [original_text]
+    # Do $steps$ steps of random walk procedure
     for _ in range(steps):
         sentences = single_round(old_sentences, tokenizer, model, k)
         old_sentences = copy.deepcopy(sentences)
 
-    #assert len(sentences) == k**steps  # This may not be possible if we cannot find k non-punctuation suggestions.
     return sentences
 
 def sentence_similarity_metric(similarity_model, sen_A, sen_B):
+    """Compute the similarity between two sentences by embedding them using a
+    sentence transformer and computing the cosine similarity.
+
+    Args:
+        similarity_model (type): sentence transformer
+        sen_A (str): first sentence
+        sen_B (str): second sentence
+    Returns:
+        float: sentence similarity
+    """
+
     emb_A = similarity_model.encode(sen_A)
     emb_B = similarity_model.encode(sen_B)
 
@@ -152,6 +178,10 @@ class RandomWalk(SentenceOperation):
         TaskType.TEXT_CLASSIFICATION
     ]
     languages = ["en"]
+    heavy = True
+    keywords = [ "model-based", "api-based", "transformer-based", "tokenizer-requried", \
+        "lexical", "possible-meaning-alteration", "low-precision", \
+        "high-coverage", "high-generations" ]
 
     # Default parameters match those of the 'test.json' below.
     def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25):
@@ -179,7 +209,8 @@ def generate(self, sentence: str):
         scores = []
         for o in perturbed_texts:
             scores.append(sentence_similarity_metric(self.sim_model, sentence, o))
-        valid_sentences = np.array(scores) > self.sim_req
+        valid_sentences = np.array(scores) > self.sim_req # Only sentences with a 
+        # high enough similarity score are kept.
         perturbed_texts = [o for o,s in zip(perturbed_texts, valid_sentences) if s]
         assert np.sum(valid_sentences) == len(perturbed_texts)
 
@@ -212,4 +243,4 @@ def generate(self, sentence: str):
 
     with open('test.json', 'w') as f:
         json.dump(json_file, f, indent=2)
-"""
\ No newline at end of file
+"""

From 296ecdab64d881318988e6e1c6a8aba13d555669 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Tue, 21 Sep 2021 00:25:53 -0700
Subject: [PATCH 15/19] Update README.md

---
 transformations/random_walk/README.md | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index c668e3210..f585c4017 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -8,9 +8,9 @@ Author names:
  - Jamie Simon (james.simon@berkeley.edu, UC Berkeley)
  - Chandan Singh (chandan_singh@berkeley.edu, UC Berkeley)
 
-## Extras
+## Data and Code Provenance
 
-This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers model. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran.
+This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers library. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran. Both libraries operates under the Apache 2.0 license
 
 ## What type of a transformation is this?
 This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. We measure the similarity between the original and random-walked sentence by performing sentence embeddings and then calculate the cosine similarity bewteen the embedded vectors.
@@ -37,5 +37,10 @@ text generation, etc. Evaluating the perturbation using Google Colab is currentl
 This transformation can generate nonsensical words when the random walk has many steps (steps >~ number of words in sentence).
 
 ## References
-1) Saketh Kotamraju, "How to use BERT from the Hugging face transformer library", https://towardsdatascience.com/how-to-use-bert-from-the-hugging-face-transformer-library-d373a22b0209
+1) Saketh Kotamraju; "How to use BERT from the Hugging face transformer library"; https://towardsdatascience.com/how-to-use-bert-from-the-hugging-face-transformer-library-d373a22b0209
+
+As far as we know, this type of transformation where words are randomly perturbed has not been studied in published literature. Random walks have been used to measure sentence similarity, e.g. the papers listed below.
+
+2) Daniel Ramage, Anna N. Rafferty, and Christopher D. Manning; "Random Walks for Text Semantic Similarity"; https://nlp.stanford.edu/pubs/wordwalk-textgraphs09.pdf
+3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf
 

From f5ca532630b9aa2f128c5ed0522fb532d6098569 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Tue, 5 Oct 2021 05:44:18 +0000
Subject: [PATCH 16/19] Adding named entities.

---
 transformations/random_walk/test.json         |  26 ++--
 transformations/random_walk/transformation.py | 114 ++++++++++++------
 2 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/transformations/random_walk/test.json b/transformations/random_walk/test.json
index f6c7a6239..471237f4b 100644
--- a/transformations/random_walk/test.json
+++ b/transformations/random_walk/test.json
@@ -11,7 +11,7 @@
           "sentence": "She finally returned the French book to me that I bought last year."
         },
         {
-          "sentence": "She finally returned the French book for me that I bought last year."
+          "sentence": "She finally returned the picture book for me that I bought last year."
         },
         {
           "sentence": "Andrew finally returned the French box to Chris that I bought last week."
@@ -25,13 +25,13 @@
       },
       "outputs": [
         {
-          "sentence": "Lines with gappinglines such as Paul likes coffee and Mary tea, lack an overt attempt to indicate a relation between two or more arguments."
+          "sentence": "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an explicit predicate to explain the relation between two or more arguments."
         },
         {
-          "sentence": "Sentences with gapping, in that Paul drinks coffee and Mary tea, lack an overt attempt to show the relation between two or more arguments."
+          "sentence": "Sentences involving gapping, such as John likes coffee and Mary tea, lack an appropriate predicate to indicate the relation between two or three arguments."
         },
         {
-          "sentence": "Sentences with women, such that Paul likes coffee and Mary tea, lack any overt attempt to explain the relation between two or more arguments."
+          "sentence": "Examples with gapping, Such as Paul likes coffee and Mary tea, lack an appropriate predicate to indicate the relation between two or more arguments."
         }
       ]
     },
@@ -42,13 +42,13 @@
       },
       "outputs": [
         {
-          "sentence": "Alice In Wonderland is a 2014 American live-action/animated dark fantasy adventure film"
+          "sentence": "Alice in Wonderland is a 2010 American live-Action/Animated romantic fantasy adventure film"
         },
         {
-          "sentence": "Alice in Wonderland is a 2015 Canadian live-action/animation dark fantasy adventure film"
+          "sentence": "Alice In Wonderland is a 2010 Canadian live-Action/animated dark fantasy adventure film"
         },
         {
-          "sentence": "Alice In Wonderland is a 2015 American live-action/animated dark fantasy adventure film"
+          "sentence": "Alice in Wonderland is a 2010 American live-Action/Animated dark fantasy adventure film"
         }
       ]
     },
@@ -59,13 +59,13 @@
       },
       "outputs": [
         {
-          "sentence": "Ujjal Dev Dosanjh served the 13th Premier of British Columbia from 2000 until 2001"
+          "sentence": "Ujjal Dev who served as the Premier of Sri Columbia from 2000 until 2001"
         },
         {
-          "sentence": "Ujjal Dev Dosanjh was the 33rd Premier of British Columbia from 2000 to 2001"
+          "sentence": "Ram Dev Dosanjh served as 33rd Premier of British Columbia from 2000 until 2003"
         },
         {
-          "sentence": "Ujjal Dev has served as 33rd premier in British Columbia from 2000 to 2001"
+          "sentence": "and Dev Dosanjh serving as Deputy Premier of British Columbia from 2000 to 2001"
         }
       ]
     },
@@ -76,13 +76,13 @@
       },
       "outputs": [
         {
-          "sentence": "Neuroplasticity is a continuous processing allowing short-term, medium-term, and longer-term remodeling of the neuronosynaptic organization."
+          "sentence": "Neuroplasticity is a continuous processing allowing short-term, mid-term, and long-term remodeling of the brain organization."
         },
         {
-          "sentence": "Neuroplasticity is a continuous processing allowing short-duration, medium-termstability and long-term remodeling of the neuronosynaptic network."
+          "sentence": "Neuroplasticity is a neural processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."
         },
         {
-          "sentence": "It is a dynamic processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."
+          "sentence": "It is a dynamic processing allowing short-duration, medium-term, and long-term remodeling of the neuronosynaptic organization."
         }
       ]
     }
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index b80172844..166672a7b 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -8,6 +8,7 @@
 
 from transformers import BertTokenizer, BertForMaskedLM
 from sentence_transformers import SentenceTransformer, util
+import spacy
 
 from torch.nn import functional as F
 import torch
@@ -65,7 +66,8 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
     assert len(valid_tokens) == k or i == len(sorted_tokens)    # We either have found k valid (non punctuation) tokens or we have looked through all the tokens.
     return valid_tokens, True
 
-def single_sentence_random_step(sentence, tokenizer, model, k):
+
+def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
     """For a given sentence, choose a random word to mask, and
     replace it with a word the top-k most similar words in BERT model.
     Return k sentences, each with a different replacement word for the mask.
@@ -74,50 +76,76 @@ def single_sentence_random_step(sentence, tokenizer, model, k):
         sentence ([type]): sentence to perform random walk on
         tokenizer ([type]): tokenizer
         model ([type]): model
+        nlp ([type]): Spacy NLP model for finding named entities
+        names (bool): Whether or not to change named entities
         k (int): how many replacement words to try.
 
     Returns:
         [list]: k-sentences with masked word replaced with top-k most similar words
     """
-
     split_iter = re.finditer(r"[\w']+|[.,!?;]", sentence) # Split sentence on puctuation.
     text_split = []
     split_indices = []
+    sentence_parts = []
+    text_split.append([])
+    split_indices.append([])
+    base_index = 0
     for m in split_iter:
-        text_split.append(m.group(0))
-        split_indices.append((m.start(), m.end()))
-
-    included_mask = False
-    while not included_mask: # Loop until the masked word is one of the first 
-        # 512 tokens of input, since all else are discarded.
-      
-        rand_int = np.random.randint(len(text_split)) # pick a random word to mask
-        word_to_mask = text_split[rand_int]
-        while len(word_to_mask) == 0 and not word_to_mask.isalnum(): # Avoid empty strings in split text
-            rand_int = np.random.randint(len(text_split))
-            word_to_mask = text_split[rand_int]
-        
-        # mask word
-        new_text = _mask_word(sentence, split_indices[rand_int], tokenizer.mask_token)
-
-        # get k replacement words
-        top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
-    replacement_words = [tokenizer.decode([token]) for token in top_k]
-
-    # replace mask-token with the word from the top-k replacements
-    return [
-        new_text.replace(tokenizer.mask_token, word)
-        for word in replacement_words
-    ]
+        text_split[-1].append(m.group(0))
+        split_indices[-1].append((m.start(), m.end()))
+        if m.end() - base_index > 450:
+          sentence_parts.append(sentence[base_index:m.end()])
+          base_index = m.end()
+          text_split.append([])
+          split_indices.append([])
+    
+    sentence_parts.append(sentence[base_index:])
 
+    new_sentences = []
+    for ts, si, sen in zip (text_split, split_indices, sentence_parts):
+      included_mask = False
+      while not included_mask: # Loop until the masked word is one of the first 
+          # 512 tokens of input, since all else are discarded.
+          if names:
+              doc = nlp(sen)
+              entities = [ent.text for ent in doc.ents]
+          else:
+              entities = []
+          
+          rand_int = np.random.randint(len(ts)) # pick a random word to mask
+          word_to_mask = ts[rand_int]
+          while len(word_to_mask) == 0 or not word_to_mask.isalnum() or word_to_mask in entities: # Avoid empty strings in split text
+              rand_int = np.random.randint(len(ts))
+              word_to_mask = ts[rand_int]
+          
+          # mask word
+          new_text = _mask_word(sen, si[rand_int], tokenizer.mask_token)
+
+          # get k replacement words
+          top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+          assert included_mask == True
+      replacement_words = [tokenizer.decode([token]) for token in top_k]
+
+      # replace mask-token with the word from the top-k replacements
+      new_sentences.append([
+          new_text.replace(tokenizer.mask_token, word)
+          for word in replacement_words
+      ])
+    
+    final_sentences = new_sentences[0]
+    for sens in new_sentences[1:]:
+       sentences = [s + a for s, a in zip(final_sentences, sens)]
 
-def single_round(sentences: List[str], tokenizer, model, k) -> List[str]:
+    return final_sentences
+def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[str]:
     """For a given list of sentences, do a random walk on each sentence.
 
     Args:
         sentences ([type]): list of sentnces to perform random walk on
         tokenizer ([type]): tokenizer
         model ([type]): model
+        nlp ([type]): Spacy NLP model for finding named entities
+        names (bool): Whether or not to change named entities
         k (int): how many words to sample to replace masked word
     Returns:
         [List]: list of k random-walked sentences
@@ -126,13 +154,13 @@ def single_round(sentences: List[str], tokenizer, model, k) -> List[str]:
 
     for sentence in sentences:
         new_sentences.extend(
-            single_sentence_random_step(sentence, tokenizer, model, k))
+            single_sentence_random_step(sentence, tokenizer, model, nlp, names, k))
 
     return new_sentences
 
 
 def random_walk(original_text: str, steps: int, k: int, tokenizer,
-                model) -> List[str]:
+                model, nlp, names) -> List[str]:
     """For a sentence, perform a random walk sequence on the sentence, generating
     new sentences at each step and perturbing these during the next step.
 
@@ -142,6 +170,9 @@ def random_walk(original_text: str, steps: int, k: int, tokenizer,
         k (int): how many words to sample to replace masked word during each iteration
         tokenizer ([type]): tokenizer
         model ([type]): model
+        nlp ([type]): Spacy NLP model for finding named entities
+        names (bool): Whether or not to change named entities
+
     Returns:
         [List]: list of steps^k random-walked sentences
     """
@@ -149,9 +180,8 @@ def random_walk(original_text: str, steps: int, k: int, tokenizer,
     old_sentences = [original_text]
     # Do $steps$ steps of random walk procedure
     for _ in range(steps):
-        sentences = single_round(old_sentences, tokenizer, model, k)
+        sentences = single_round(old_sentences, tokenizer, model, nlp, names, k)
         old_sentences = copy.deepcopy(sentences)
-
     return sentences
 
 def sentence_similarity_metric(similarity_model, sen_A, sen_B):
@@ -179,22 +209,24 @@ class RandomWalk(SentenceOperation):
     ]
     languages = ["en"]
     heavy = True
-    keywords = [ "model-based", "api-based", "transformer-based", "tokenizer-requried", \
+    keywords = [ "model-based", "api-based", "transformer-based", "tokenizer-required", \
         "lexical", "possible-meaning-alteration", "low-precision", \
         "high-coverage", "high-generations" ]
 
     # Default parameters match those of the 'test.json' below.
-    def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25):
+    def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, named_entities=False):
         random.seed(seed)
         np.random.seed(seed)
         super().__init__(seed, max_outputs=max_outputs)
         self.tokenizer = BertTokenizer.from_pretrained('bert-large-cased')
         self.model = BertForMaskedLM.from_pretrained('bert-large-cased')
         self.sim_model = SentenceTransformer('all-mpnet-base-v2')
+        self.spacy_nlp = spacy.load("en_core_web_sm")
         self.max_outputs = max_outputs
         self.steps = steps
         self.k = k
         self.sim_req = sim_req
+        self.named_entities = named_entities
 
     def generate(self, sentence: str):
         print('Random walking on the sentence:', sentence)
@@ -203,7 +235,9 @@ def generate(self, sentence: str):
             steps=self.steps,
             k=self.k,
             tokenizer=self.tokenizer,
-            model=self.model
+            model=self.model,
+            nlp=self.spacy_nlp,
+            names = self.named_entities
         )
 
         scores = []
@@ -224,9 +258,9 @@ def generate(self, sentence: str):
 # You don't need to keep this code in your transformation.
 if __name__ == '__main__':
     import json
-    from TestRunner import convert_to_snake_case
+    #from TestRunner import convert_to_snake_case
 
-    tf = RandomWalk(max_outputs=3, k=2, steps=5, sim_req=0.25)
+    tf = RandomWalk(max_outputs=3, k=2, steps=5, sim_req=0.25, named_entities=True)
     #sentence = "Andrew finally returned the French book to Chris that I bought last week"
     test_cases = []
     for sentence in ["Andrew finally returned the French book to Chris that I bought last week.",
@@ -235,12 +269,12 @@ def generate(self, sentence: str):
                      "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001",
                      "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."]:
         test_cases.append({
-            "class": tf.name(),
+            "class": "RandomWalk",#tf.name(),
             "inputs": {"sentence": sentence}, "outputs": [{"sentence": o} for o in tf.generate(sentence)]}
         )
-    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
+    json_file = {"type": "random_walk", "test_cases": test_cases} #convert_to_snake_case(tf.name())
     print(json.dumps(json_file, indent=2))
 
     with open('test.json', 'w') as f:
         json.dump(json_file, f, indent=2)
-"""
+"""
\ No newline at end of file

From 3fc94ebd1ff7a7007ed76e464e2ace473e534e7f Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Wed, 13 Oct 2021 05:25:07 +0000
Subject: [PATCH 17/19] Evaluation results.:

---
 test/mapper.py                                |   1 +
 transformations/random_walk/README.md         |  27 +++-
 transformations/random_walk/requirements.txt  |   2 +-
 transformations/random_walk/transformation.py | 124 +++++++++++-------
 4 files changed, 101 insertions(+), 53 deletions(-)

diff --git a/test/mapper.py b/test/mapper.py
index 09c3ed8b5..a63db9d74 100644
--- a/test/mapper.py
+++ b/test/mapper.py
@@ -71,6 +71,7 @@
         "pinyin",
         "punctuation",
         "quora_trained_t5_for_qa",
+        "random_walk"
         "sentence_reordering",
         "synonym_substitution",
         "token_replacement",
diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index f585c4017..c847f365a 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -10,13 +10,13 @@ Author names:
 
 ## Data and Code Provenance
 
-This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers library. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran. Both libraries operates under the Apache 2.0 license
+This transformation requires the 'bert-large-cased' pretrained model (~1 GB) from the Hugging Face Transformers library and the 'all-mpnet-base-v2' pretrained model (~400 GB) from the Sentence Transformers library. Provided that the libraries are installed (as they should be from 'requirements.txt'), these models will be installed the first time this transformation is ran. Both libraries operates under the Apache 2.0 license. Additionally the Spacy library is necessary but is installed by default when using this benchmark.
 
 ## What type of a transformation is this?
 This transformation acts like a perturbation to test robustness and generate sentences with similar syntactic content. By randomly replacing words with their mostly likely replacements, as determined by a bidirectional model that incorporates context clues from prevous and later words, we hope to generate similar sentences that make grammatical sense. We measure the similarity between the original and random-walked sentence by performing sentence embeddings and then calculate the cosine similarity bewteen the embedded vectors.
 
 ## How it works
-At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences. Note that BERT has a max input token length of 512, so for long inputs, we truncate to the first 512 tokens.
+At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. Additionally, we can exclude named entities found by the Spacy model from random selection for repalcement. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences. Note that BERT has a max input token length of 512, so for long inputs, we split the sentence into chunks less than 512 tokens.
 
 The differences between original and generated sentences are controlled by two class initialization parameters, `steps` and `k`.
  - `steps`: number of random walk steps to do
@@ -28,9 +28,25 @@ The seed of the random generators (both from `numpy` and the `random` module) ar
 
 The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter `sim_req` which is the minimum similarity score that a generated sentence must have to be considered valid.
 
+Finally a boolean `names` specifies whether or not we replace named entities.
+
 ## What tasks does it intend to benefit?
-This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, 
-text generation, etc. Evaluating the perturbation using Google Colab is currently in progress.
+This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, text generation, etc. Evaluating the perturbation using Google Colab is currently in progress.
+
+## Robustness Evaluation
+
+This model was evaluated with the model aychang/roberta-base-imdb on the test[:20%] split of the imdb dataset. Note: due to the computational demands of this transformation and the lack of resources at our disposal (only GPU access is Colab), we evaluate the transformation with the following parameters:
+ - `seed = 0`
+ - `max_outputs = 1` : Produce a single sentence
+ - `steps = 5` : Randomly select a word to replace 5 times
+ - `k = 1`: Number of high probability replacements for the masked word to consider
+ - `sim_req = 0` : Similarity requirement for generated sentences (long sentences tend to have low similarity
+ - `named_entities = True` : Do not replace named entities
+
+Wall Time: 00:03:52 (DD:HH:MM)
+Performance: Of 1000 original sentences, 985 successfully transformed and 15 unchanged (0.985 perturb rate). Accuracy: 96.0 -> 96.0
+
+Performance is strongly affected by parameters `steps` and `k`, as larger values of each will lead to greater variation in generated sentences, at the expense of longer runtimes.
 
 ## What are the limitations of this transformation?
 
@@ -42,5 +58,4 @@ This transformation can generate nonsensical words when the random walk has many
 As far as we know, this type of transformation where words are randomly perturbed has not been studied in published literature. Random walks have been used to measure sentence similarity, e.g. the papers listed below.
 
 2) Daniel Ramage, Anna N. Rafferty, and Christopher D. Manning; "Random Walks for Text Semantic Similarity"; https://nlp.stanford.edu/pubs/wordwalk-textgraphs09.pdf
-3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf
-
+3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf
\ No newline at end of file
diff --git a/transformations/random_walk/requirements.txt b/transformations/random_walk/requirements.txt
index ebe278d3c..394f7600f 100644
--- a/transformations/random_walk/requirements.txt
+++ b/transformations/random_walk/requirements.txt
@@ -1,2 +1,2 @@
 sentence-transformers==2.0.0
-transformers==4.6.1
+transformers==4.6.1
\ No newline at end of file
diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 166672a7b..7730932b6 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -23,7 +23,7 @@ def _mask_word(sentence, split_indices, mask):
     Args:
     sentence (str): sentence with work to mask
     split_indices ([int, int]): index of word to replace, begining and character 
-      after the end indices
+		after the end indices
     mask (BERT Token): token for a BERT mask
     
     """
@@ -44,6 +44,7 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
 
     Returns:
         [list]: list of top k words
+		[bool]: Whether or not we found the desired number of valid replacement words
     """
     inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512)
     index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
@@ -51,6 +52,7 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
         # 512 tokens (BERT's max), we need to make sure the mask is in these first 512.
         # If not, return False so that we try again.
         return None, False
+        # This should not occur since we split long sentences.
     outputs = model(**inputs)
     softmax = F.softmax(outputs.logits, dim=-1)
     mask_word = softmax[0, index_to_mask, :]
@@ -64,7 +66,9 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
             valid_tokens.append(sorted_tokens[i])
         i += 1
     assert len(valid_tokens) == k or i == len(sorted_tokens)    # We either have found k valid (non punctuation) tokens or we have looked through all the tokens.
-    return valid_tokens, True
+    if len(valid_tokens) < k:
+        valid_tokens += (k - len(valid_tokens)) * [valid_tokens[0]]
+	return valid_tokens, True
 
 
 def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
@@ -77,7 +81,7 @@ def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
         tokenizer ([type]): tokenizer
         model ([type]): model
         nlp ([type]): Spacy NLP model for finding named entities
-        names (bool): Whether or not to change named entities
+        names (list): Named entities to not replace.
         k (int): how many replacement words to try.
 
     Returns:
@@ -92,51 +96,65 @@ def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
     base_index = 0
     for m in split_iter:
         text_split[-1].append(m.group(0))
-        split_indices[-1].append((m.start(), m.end()))
+        split_indices[-1].append((m.start() - base_index, m.end() - base_index))
+        
         if m.end() - base_index > 450:
-          sentence_parts.append(sentence[base_index:m.end()])
-          base_index = m.end()
-          text_split.append([])
-          split_indices.append([])
-    
+            sentence_parts.append(sentence[base_index:m.end()])
+            base_index = m.end()
+            text_split.append([])
+            split_indices.append([])
+            
     sentence_parts.append(sentence[base_index:])
 
+    # Remove any empty sentence parts.
+    valid_splits = [len(ts) > 0 for ts in text_split]
+    text_split = [ts for ts, vs in zip(text_split, valid_splits) if vs]
+    split_indices = [si for si, vs, in zip(split_indices, valid_splits) if vs]
+    sentence_parts = [sen for sen, vs in zip(sentence_parts, valid_splits) if vs]
+
     new_sentences = []
-    for ts, si, sen in zip (text_split, split_indices, sentence_parts):
-      included_mask = False
-      while not included_mask: # Loop until the masked word is one of the first 
-          # 512 tokens of input, since all else are discarded.
-          if names:
-              doc = nlp(sen)
-              entities = [ent.text for ent in doc.ents]
-          else:
-              entities = []
+    for ts, si, sen in zip (text_split, split_indices, sentence_parts):		
+        if len(ts) == 0:
+            print(text_split, split_indices, sentence_parts)
+            raise ValueError("Somehow we got a sentence part that is emtpy. Not good!")
+        rand_int = np.random.randint(len(ts)) # pick a random word to mask
+        word_to_mask = ts[rand_int]
+        iter_count = 1
+        give_up = False
+        while len(word_to_mask) == 0 or not word_to_mask.isalnum() or word_to_mask in names: # Avoid empty strings in split text
+            rand_int = np.random.randint(len(ts))
+            word_to_mask = ts[rand_int]
+            iter_count += 1
+            if iter_count > len(ts):
+                print("In the sentence <" + sen + ">, no valid words to mask.")
+                give_up = True
+                break
           
-          rand_int = np.random.randint(len(ts)) # pick a random word to mask
-          word_to_mask = ts[rand_int]
-          while len(word_to_mask) == 0 or not word_to_mask.isalnum() or word_to_mask in entities: # Avoid empty strings in split text
-              rand_int = np.random.randint(len(ts))
-              word_to_mask = ts[rand_int]
-          
-          # mask word
-          new_text = _mask_word(sen, si[rand_int], tokenizer.mask_token)
-
-          # get k replacement words
-          top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
-          assert included_mask == True
-      replacement_words = [tokenizer.decode([token]) for token in top_k]
-
-      # replace mask-token with the word from the top-k replacements
-      new_sentences.append([
-          new_text.replace(tokenizer.mask_token, word)
-          for word in replacement_words
-      ])
+        if not give_up:
+            # mask word
+            new_text = _mask_word(sen, si[rand_int], tokenizer.mask_token)
+            # get k replacement words
+            top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+            assert included_mask == True
+            
+            replacement_words = [tokenizer.decode([token]) for token in top_k]
+
+            # replace mask-token with the word from the top-k replacements
+            new_sentences.append([
+                new_text.replace(tokenizer.mask_token, word)
+                for word in replacement_words
+            ])
+        else:
+            new_sentences.append([
+                sen for _ in range(k)
+            ])
     
     final_sentences = new_sentences[0]
     for sens in new_sentences[1:]:
-       sentences = [s + a for s, a in zip(final_sentences, sens)]
-
+        final_sentences = [s + a for s, a in zip(final_sentences, sens)]
     return final_sentences
+	
+	
 def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[str]:
     """For a given list of sentences, do a random walk on each sentence.
 
@@ -145,7 +163,7 @@ def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[
         tokenizer ([type]): tokenizer
         model ([type]): model
         nlp ([type]): Spacy NLP model for finding named entities
-        names (bool): Whether or not to change named entities
+        names (list): Named entities to not replace.
         k (int): how many words to sample to replace masked word
     Returns:
         [List]: list of k random-walked sentences
@@ -155,7 +173,6 @@ def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[
     for sentence in sentences:
         new_sentences.extend(
             single_sentence_random_step(sentence, tokenizer, model, nlp, names, k))
-
     return new_sentences
 
 
@@ -176,11 +193,21 @@ def random_walk(original_text: str, steps: int, k: int, tokenizer,
     Returns:
         [List]: list of steps^k random-walked sentences
     """
-
+    if names:
+        doc = nlp(original_text)
+        entities = [ent.text for ent in doc.ents]
+        split_entities = []
+        for ent in entities:
+            split_iter = re.finditer(r"[\w']+|[.,!?;]", ent)
+            for m in split_iter:
+                split_entities.append(m.group(0))
+        print('Named entities: ', split_entities)
+    else:
+        split_entities = []
     old_sentences = [original_text]
     # Do $steps$ steps of random walk procedure
     for _ in range(steps):
-        sentences = single_round(old_sentences, tokenizer, model, nlp, names, k)
+        sentences = single_round(old_sentences, tokenizer, model, nlp, split_entities, k)
         old_sentences = copy.deepcopy(sentences)
     return sentences
 
@@ -215,8 +242,11 @@ class RandomWalk(SentenceOperation):
 
     # Default parameters match those of the 'test.json' below.
     def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, named_entities=False):
-        random.seed(seed)
-        np.random.seed(seed)
+    # For evaluation, use parameters that are less compute intensive.
+    # def __init__(self, seed=0, max_outputs=1, steps=5, k=1, sim_req=0, named_entities=True):
+        random.seed(self.seed)
+        np.random.seed(self.seed)
+        
         super().__init__(seed, max_outputs=max_outputs)
         self.tokenizer = BertTokenizer.from_pretrained('bert-large-cased')
         self.model = BertForMaskedLM.from_pretrained('bert-large-cased')
@@ -239,17 +269,19 @@ def generate(self, sentence: str):
             nlp=self.spacy_nlp,
             names = self.named_entities
         )
-
         scores = []
         for o in perturbed_texts:
             scores.append(sentence_similarity_metric(self.sim_model, sentence, o))
         valid_sentences = np.array(scores) > self.sim_req # Only sentences with a 
+        assert np.sum(valid_sentences) > 0, "Similarity requirement too high; no valid sentences. Note: Long sentences have very low similarity."
+        
         # high enough similarity score are kept.
         perturbed_texts = [o for o,s in zip(perturbed_texts, valid_sentences) if s]
         assert np.sum(valid_sentences) == len(perturbed_texts)
 
         if len(perturbed_texts) > self.max_outputs:
             perturbed_texts = random.sample(perturbed_texts, self.max_outputs)
+          
         return perturbed_texts
 
 """

From 6f889995aa49432693e18370adc07adea8a664d7 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@aol.com>
Date: Wed, 13 Oct 2021 05:51:02 +0000
Subject: [PATCH 18/19] Fix tabs and add option to choose least probable
 replacement.

---
 transformations/random_walk/transformation.py | 42 ++++++++++++-------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/transformations/random_walk/transformation.py b/transformations/random_walk/transformation.py
index 7730932b6..d9d4acaf9 100644
--- a/transformations/random_walk/transformation.py
+++ b/transformations/random_walk/transformation.py
@@ -31,7 +31,7 @@ def _mask_word(sentence, split_indices, mask):
         split_indices[1]:]
 
 
-def get_k_replacement_words(tokenized_text, tokenizer, model, k):
+def get_k_replacement_words(tokenized_text, tokenizer, model, k, descending=True):
     """return k most similar words from the model, for a tokenized mask-word 
     in a sentence.
 
@@ -41,10 +41,12 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
         model ([type]): model
         k (int): how many similar words to find for a given tokenized-word. Checks
           that generated word is a composed of letters or numbers.
+        descending (bool): If true, sort tokens by  in decreasing order of probability.
+          Else sort in increasing order of probability. We sample from sorted tokens.
 
     Returns:
         [list]: list of top k words
-		[bool]: Whether or not we found the desired number of valid replacement words
+        bool: Whether or not we found the desired number of valid replacement words
     """
     inputs = tokenizer.encode_plus(tokenized_text, return_tensors='pt', truncation=True, max_length = 512)
     index_to_mask = torch.where(inputs.input_ids[0] == tokenizer.mask_token_id)
@@ -57,7 +59,7 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
     softmax = F.softmax(outputs.logits, dim=-1)
     mask_word = softmax[0, index_to_mask, :]
 
-    sorted_tokens = torch.argsort(mask_word[0], descending=True)
+    sorted_tokens = torch.argsort(mask_word[0], descending=descending)
     i = 0
     valid_tokens = [] # The k most probable tokens are guaranteed to be words,
     # so we make sure they are.
@@ -67,11 +69,12 @@ def get_k_replacement_words(tokenized_text, tokenizer, model, k):
         i += 1
     assert len(valid_tokens) == k or i == len(sorted_tokens)    # We either have found k valid (non punctuation) tokens or we have looked through all the tokens.
     if len(valid_tokens) < k:
-        valid_tokens += (k - len(valid_tokens)) * [valid_tokens[0]]
-	return valid_tokens, True
+      valid_tokens += (k - len(valid_tokens)) * [valid_tokens[0]]
+    
+    return valid_tokens, True
 
 
-def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
+def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k, descending):
     """For a given sentence, choose a random word to mask, and
     replace it with a word the top-k most similar words in BERT model.
     Return k sentences, each with a different replacement word for the mask.
@@ -83,6 +86,8 @@ def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
         nlp ([type]): Spacy NLP model for finding named entities
         names (list): Named entities to not replace.
         k (int): how many replacement words to try.
+        descending (bool): If true, sort tokens by  in decreasing order of probability.
+          Else sort in increasing order of probability. We sample from sorted tokens.
 
     Returns:
         [list]: k-sentences with masked word replaced with top-k most similar words
@@ -134,7 +139,8 @@ def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
             # mask word
             new_text = _mask_word(sen, si[rand_int], tokenizer.mask_token)
             # get k replacement words
-            top_k, included_mask = get_k_replacement_words(new_text, tokenizer, model, k=k)
+            top_k, included_mask = get_k_replacement_words(new_text, tokenizer, 
+                                                          model, k=k, descending=descending)
             assert included_mask == True
             
             replacement_words = [tokenizer.decode([token]) for token in top_k]
@@ -155,7 +161,7 @@ def single_sentence_random_step(sentence, tokenizer, model, nlp, names, k):
     return final_sentences
 	
 	
-def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[str]:
+def single_round(sentences: List[str], tokenizer, model, nlp, names, k, descending) -> List[str]:
     """For a given list of sentences, do a random walk on each sentence.
 
     Args:
@@ -165,6 +171,9 @@ def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[
         nlp ([type]): Spacy NLP model for finding named entities
         names (list): Named entities to not replace.
         k (int): how many words to sample to replace masked word
+        descending (bool): If true, sort tokens by  in decreasing order of probability.
+          Else sort in increasing order of probability. We sample from sorted tokens.
+
     Returns:
         [List]: list of k random-walked sentences
     """
@@ -172,12 +181,12 @@ def single_round(sentences: List[str], tokenizer, model, nlp, names, k) -> List[
 
     for sentence in sentences:
         new_sentences.extend(
-            single_sentence_random_step(sentence, tokenizer, model, nlp, names, k))
+            single_sentence_random_step(sentence, tokenizer, model, nlp, names, k, descending))
     return new_sentences
 
 
 def random_walk(original_text: str, steps: int, k: int, tokenizer,
-                model, nlp, names) -> List[str]:
+                model, nlp, names: bool, descending: bool) -> List[str]:
     """For a sentence, perform a random walk sequence on the sentence, generating
     new sentences at each step and perturbing these during the next step.
 
@@ -189,6 +198,8 @@ def random_walk(original_text: str, steps: int, k: int, tokenizer,
         model ([type]): model
         nlp ([type]): Spacy NLP model for finding named entities
         names (bool): Whether or not to change named entities
+        descending (bool): If true, sort tokens by  in decreasing order of probability.
+          Else sort in increasing order of probability. We sample from sorted tokens.
 
     Returns:
         [List]: list of steps^k random-walked sentences
@@ -207,7 +218,7 @@ def random_walk(original_text: str, steps: int, k: int, tokenizer,
     old_sentences = [original_text]
     # Do $steps$ steps of random walk procedure
     for _ in range(steps):
-        sentences = single_round(old_sentences, tokenizer, model, nlp, split_entities, k)
+        sentences = single_round(old_sentences, tokenizer, model, nlp, split_entities, k, descending)
         old_sentences = copy.deepcopy(sentences)
     return sentences
 
@@ -241,9 +252,10 @@ class RandomWalk(SentenceOperation):
         "high-coverage", "high-generations" ]
 
     # Default parameters match those of the 'test.json' below.
-    def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, named_entities=False):
+    def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, 
+                  named_entities=False, descending=True):
     # For evaluation, use parameters that are less compute intensive.
-    # def __init__(self, seed=0, max_outputs=1, steps=5, k=1, sim_req=0, named_entities=True):
+    # def __init__(self, seed=0, max_outputs=1, steps=5, k=1, sim_req=0, named_entities=True, descending=True):
         random.seed(self.seed)
         np.random.seed(self.seed)
         
@@ -257,6 +269,7 @@ def __init__(self, seed=0, max_outputs=3, steps=5, k=2, sim_req=0.25, named_enti
         self.k = k
         self.sim_req = sim_req
         self.named_entities = named_entities
+        self.descending = descending
 
     def generate(self, sentence: str):
         print('Random walking on the sentence:', sentence)
@@ -267,7 +280,8 @@ def generate(self, sentence: str):
             tokenizer=self.tokenizer,
             model=self.model,
             nlp=self.spacy_nlp,
-            names = self.named_entities
+            names=self.named_entities,
+            descending=self.descending
         )
         scores = []
         for o in perturbed_texts:

From ee57b6051c94872ec732657b53a0ba4b2411fa97 Mon Sep 17 00:00:00 2001
From: Sajant Anand <sajantanand@users.noreply.github.com>
Date: Tue, 12 Oct 2021 23:05:55 -0700
Subject: [PATCH 19/19] Add "descending" parameter to README.md

---
 transformations/random_walk/README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/transformations/random_walk/README.md b/transformations/random_walk/README.md
index c847f365a..184c1e3e5 100644
--- a/transformations/random_walk/README.md
+++ b/transformations/random_walk/README.md
@@ -18,7 +18,7 @@ This transformation acts like a perturbation to test robustness and generate sen
 ## How it works
 At each step in the random walk, we randomly choose a word and replace it by the mask token recognized by BERT. Care is take to preserve punctuation where possible so that the generated sentence has the same punctuation as the original sentence. Additionally, we can exclude named entities found by the Spacy model from random selection for repalcement. With a word masked, we run BERT on the sentence and perform a softmax on the output logits. Then we select the high probability replacement words for the masked token and use these to construct new sentences. Note that BERT has a max input token length of 512, so for long inputs, we split the sentence into chunks less than 512 tokens.
 
-The differences between original and generated sentences are controlled by two class initialization parameters, `steps` and `k`.
+The differences between original and generated sentences are generally controlled by two class initialization parameters, `steps` and `k`.
  - `steps`: number of random walk steps to do
  - `k`: number of high probability replacements for the masked word to consider
 
@@ -28,7 +28,7 @@ The seed of the random generators (both from `numpy` and the `random` module) ar
 
 The sentence similarity is done by first mapping the original and random-walked sentence to 768-dimensional vectors using a pre-trained sentence transformer. We then calculate the cosine similarity. We note that generated sentences with low similarity to the original sentence will still typically make grammatical sense; the meaning of the sentence may not be close to the original however (e.g. change the verb 'love' to 'hate'). The class initialization function takes a parameter `sim_req` which is the minimum similarity score that a generated sentence must have to be considered valid.
 
-Finally a boolean `names` specifies whether or not we replace named entities.
+Finally a boolean `names` specifies whether or not we replace named entities and a boolean `descending` controls the order of the most probable tokens for masked-word replacement.
 
 ## What tasks does it intend to benefit?
 This perturbation would benefit all tasks which have a sentence/paragraph/document as input like text classification, text generation, etc. Evaluating the perturbation using Google Colab is currently in progress.
@@ -42,6 +42,7 @@ This model was evaluated with the model aychang/roberta-base-imdb on the test[:2
  - `k = 1`: Number of high probability replacements for the masked word to consider
  - `sim_req = 0` : Similarity requirement for generated sentences (long sentences tend to have low similarity
  - `named_entities = True` : Do not replace named entities
+ - `descending = True` : Choose most probable replacements (we will rarely use `False`; we included it for kicks.)
 
 Wall Time: 00:03:52 (DD:HH:MM)
 Performance: Of 1000 original sentences, 985 successfully transformed and 15 unchanged (0.985 perturb rate). Accuracy: 96.0 -> 96.0
@@ -58,4 +59,4 @@ This transformation can generate nonsensical words when the random walk has many
 As far as we know, this type of transformation where words are randomly perturbed has not been studied in published literature. Random walks have been used to measure sentence similarity, e.g. the papers listed below.
 
 2) Daniel Ramage, Anna N. Rafferty, and Christopher D. Manning; "Random Walks for Text Semantic Similarity"; https://nlp.stanford.edu/pubs/wordwalk-textgraphs09.pdf
-3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf
\ No newline at end of file
+3) Ahmed Hassan, Amjad Abu-Jbara, Wanchen Lu, and Dragomir Radev; "A Random Walk–Based Model for Identifying Semantic Orientation"; https://aclanthology.org/J14-3003.pdf