[Task Submission] Quantifier Understanding (quantifier_understanding)

src/genbench/tasks/quantifier_understanding/config.jsonnet

@@ -0,0 +1,50 @@
+{
+    name: 'Quantifier Understanding',
+    description: 'The task evaluates generalization in the understanding of quantifiers. It aims to measure
+                  how well can language models capture the semantics of logical quantifiers in language.
+                  ',
+    keywords: [
+        'quantifiers',
+        'LLM',
+        'prompting',
+        'semantics'
+    ],
+
+    authors: [
+        'Leroy Wang',
+        'Shane Steinert-Threlkeld'
+    ],
+
+    data_source: {
+        type: 'manual',
+        test: 'https://raw.githubusercontent.com/lerow/genbench_cbt/quantifier_understanding/src/genbench/tasks/quantifier_understanding/test_data.jsonl',
+    },
+
+    has_validation_set: false,
+    has_train_set: false,
+
+    task_type: 'free_form',
+
+    evaluation_metrics: [
+        {
+            hf_id: 'exact_match',
+            git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742",
+            best_score: 1.0,
+        }
+    ],
+
+    preparation_strategies: {
+        // A recipe for preparing the model to perform the task by configuring its prompt.
+        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
+        // We provide a few options for configuring the prompt. But, the task creator can
+        // also provide a custom prompt preparation in the task's Python class.
+        prompt_based_testing: {
+            prompt_builder: {
+                instruction_zero_shot: '',  // Left empty because the prompt is in the data
+                instruction_few_shot: '',   // Left empty because the prompt is in the data
+                input_prefix: 'Q: ',
+                output_prefix: '\nA: ',
+            }
+        },
+    },
+}

src/genbench/tasks/quantifier_understanding/doc.md

@@ -0,0 +1,24 @@
+# Quantifier Understanding
+
+The task evaluates generalization in the understanding of quantifiers. It aims to measure
+how well can language models capture the semantics of logical quantifiers in natural language. It tests compositional and robustness generalisation.
+
+
+## Examples
+
+Given a question about a quantifier, determine if the answer is true or false.
+
+```
+There are 10 tables. 7 of the tables are blue. 3 of the tables are red. Are less than half of the tables red? Answer with only one word, true or false.
+```
+
+## Data Source
+
+The test data is fully generated using rule-based methods.
+
+The data is hosted here [https://github.com/lerow/genbench_cbt/blob/quantifier_understanding/src/genbench/tasks/quantifier_understanding/test_data.jsonl].
+
+
+## GenBench eval card
+
+![GenBench Eval Card](eval_card.png)

src/genbench/tasks/quantifier_understanding/task.py

@@ -0,0 +1,43 @@
+import re
+from typing import Any, Dict, List
+
+import datasets
+
+from genbench import Task
+from genbench.utils.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class QuantifierUnderstandingTask(Task):
+    def parse_output(s: str) -> str:
+        # if true return 1 else return 0
+
+        s = s.strip().split()
+
+        for token in s:
+            if len(token) < 7:
+                token = re.sub(r"[^a-zA-Z]", "", token).lower()
+                if token == "true":
+                    return "true"
+                if token == "false":
+                    return "false"
+
+        return "false"
+
+    def evaluate_predictions(
+        self, *, predictions: List[Dict[str, Any]] = None, gold: datasets.Dataset = None
+    ) -> float:
+        preds = [pred["target"] for pred in predictions]
+        gold_labels = [g["target"] for g in gold]
+
+        count = 0
+        total = len(preds)
+
+        for n in range(total):
+            label = self.parse_output(preds[n])
+            if label == gold_labels[n]:
+                count += 1
+
+        return count / total