From e2603ca98294ac8781bb338ddaab0742b6785fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Mon, 31 Jul 2023 18:39:48 -0700 Subject: [PATCH 01/12] add task skeleton --- .../tasks/multilingual_scan/__init__.py | 5 ++ .../tasks/multilingual_scan/cmn/__init__.py | 0 .../multilingual_scan/cmn/config.jsonnet | 51 +++++++++++++++++++ .../tasks/multilingual_scan/cmn/doc.md | 19 +++++++ .../tasks/multilingual_scan/cmn/task.py | 5 ++ .../tasks/multilingual_scan/config.jsonnet | 42 +++++++++++++++ src/genbench/tasks/multilingual_scan/doc.md | 17 +++++++ .../tasks/multilingual_scan/fra/__init__.py | 0 .../multilingual_scan/fra/config.jsonnet | 51 +++++++++++++++++++ .../tasks/multilingual_scan/fra/doc.md | 19 +++++++ .../tasks/multilingual_scan/fra/task.py | 5 ++ .../tasks/multilingual_scan/hin/__init__.py | 0 .../multilingual_scan/hin/config.jsonnet | 51 +++++++++++++++++++ .../tasks/multilingual_scan/hin/doc.md | 19 +++++++ .../tasks/multilingual_scan/hin/task.py | 5 ++ .../tasks/multilingual_scan/rus/__init__.py | 0 .../multilingual_scan/rus/config.jsonnet | 51 +++++++++++++++++++ .../tasks/multilingual_scan/rus/doc.md | 19 +++++++ .../tasks/multilingual_scan/rus/task.py | 5 ++ .../tasks/multilingual_scan/tur/__init__.py | 0 .../multilingual_scan/tur/config.jsonnet | 51 +++++++++++++++++++ .../tasks/multilingual_scan/tur/doc.md | 19 +++++++ .../tasks/multilingual_scan/tur/task.py | 5 ++ 23 files changed, 439 insertions(+) create mode 100644 src/genbench/tasks/multilingual_scan/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/cmn/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/cmn/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/cmn/task.py create mode 100644 src/genbench/tasks/multilingual_scan/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus/task.py create mode 100644 src/genbench/tasks/multilingual_scan/tur/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/tur/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/tur/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/tur/task.py diff --git a/src/genbench/tasks/multilingual_scan/__init__.py b/src/genbench/tasks/multilingual_scan/__init__.py new file mode 100644 index 0000000..e3f7abc --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/__init__.py @@ -0,0 +1,5 @@ +from genbench import TaskDict + + +class MultilingualScan(TaskDict): + pass diff --git a/src/genbench/tasks/multilingual_scan/cmn/__init__.py b/src/genbench/tasks/multilingual_scan/cmn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet new file mode 100644 index 0000000..d73f7de --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet @@ -0,0 +1,51 @@ +{ + name: 'Multilingual SCAN (cmn)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + + ], + + data_source: { + type: 'manual', + test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + }, + + has_validation_set: false, + has_train_set: false, + + task_type: 'free_form', + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ], + + preparation_strategies: { + // A recipe for preparing the model to perform the task by configuring its prompt. + // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. + // We provide a few options for configuring the prompt. But, the task creator can + // also provide a custom prompt preparation in the task's Python class. + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: 'Add two numbers together\n\n', + instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', + input_prefix: 'Q: ', + output_prefix: '\nA: ', + } + }, + }, +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn/doc.md b/src/genbench/tasks/multilingual_scan/cmn/doc.md new file mode 100644 index 0000000..56d3000 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn/doc.md @@ -0,0 +1,19 @@ +# Multilingual SCAN (cmn) + +## Abstract +*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (cmn).* + +## Examples +*Give some examples of the Multilingual SCAN (cmn).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Multilingual SCAN (cmn).* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN (cmn) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn/task.py b/src/genbench/tasks/multilingual_scan/cmn/task.py new file mode 100644 index 0000000..9a48092 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MultilingualScanCmn(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/config.jsonnet b/src/genbench/tasks/multilingual_scan/config.jsonnet new file mode 100644 index 0000000..bbea4c1 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/config.jsonnet @@ -0,0 +1,42 @@ +{ + name: 'Multilingual SCAN', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'multilingual', + 'compositional', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + test: 'https://somewebsite.com/path/to/test.jsonl', + }, + + has_train_set: true, + + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + } + + subtasks_order: [ + 'fra', + 'rus', + 'cmn', + 'tur', + 'hin', + + ], +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/doc.md b/src/genbench/tasks/multilingual_scan/doc.md new file mode 100644 index 0000000..24c042c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/doc.md @@ -0,0 +1,17 @@ +## Motivation +*Describe the motivation for this Multilingual SCAN.* + +## Examples +*Give examples of the Multilingual SCAN.* + +## Data Source +*Describe the data source for this Multilingual SCAN.* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN has, with links and references if possible.* + +## Citation +*Cite the source where this Multilingual SCAN was introduced.* + +## Further References +*Add any useful further references.* \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra/__init__.py b/src/genbench/tasks/multilingual_scan/fra/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet new file mode 100644 index 0000000..7ff4c16 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet @@ -0,0 +1,51 @@ +{ + name: 'Multilingual SCAN (fra)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + + ], + + data_source: { + type: 'manual', + test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + }, + + has_validation_set: false, + has_train_set: false, + + task_type: 'free_form', + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ], + + preparation_strategies: { + // A recipe for preparing the model to perform the task by configuring its prompt. + // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. + // We provide a few options for configuring the prompt. But, the task creator can + // also provide a custom prompt preparation in the task's Python class. + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: 'Add two numbers together\n\n', + instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', + input_prefix: 'Q: ', + output_prefix: '\nA: ', + } + }, + }, +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra/doc.md b/src/genbench/tasks/multilingual_scan/fra/doc.md new file mode 100644 index 0000000..f41c6f6 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra/doc.md @@ -0,0 +1,19 @@ +# Multilingual SCAN (fra) + +## Abstract +*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (fra).* + +## Examples +*Give some examples of the Multilingual SCAN (fra).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Multilingual SCAN (fra).* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN (fra) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra/task.py b/src/genbench/tasks/multilingual_scan/fra/task.py new file mode 100644 index 0000000..7ba7d80 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MultilingualScanFra(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin/__init__.py b/src/genbench/tasks/multilingual_scan/hin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet new file mode 100644 index 0000000..f7794f3 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet @@ -0,0 +1,51 @@ +{ + name: 'Multilingual SCAN (hin)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + + ], + + data_source: { + type: 'manual', + test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + }, + + has_validation_set: false, + has_train_set: false, + + task_type: 'free_form', + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ], + + preparation_strategies: { + // A recipe for preparing the model to perform the task by configuring its prompt. + // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. + // We provide a few options for configuring the prompt. But, the task creator can + // also provide a custom prompt preparation in the task's Python class. + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: 'Add two numbers together\n\n', + instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', + input_prefix: 'Q: ', + output_prefix: '\nA: ', + } + }, + }, +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin/doc.md b/src/genbench/tasks/multilingual_scan/hin/doc.md new file mode 100644 index 0000000..4780f3c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin/doc.md @@ -0,0 +1,19 @@ +# Multilingual SCAN (hin) + +## Abstract +*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (hin).* + +## Examples +*Give some examples of the Multilingual SCAN (hin).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Multilingual SCAN (hin).* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN (hin) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin/task.py b/src/genbench/tasks/multilingual_scan/hin/task.py new file mode 100644 index 0000000..5029129 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MultilingualScanHin(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus/__init__.py b/src/genbench/tasks/multilingual_scan/rus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet new file mode 100644 index 0000000..37c8b4b --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet @@ -0,0 +1,51 @@ +{ + name: 'Multilingual SCAN (rus)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + + ], + + data_source: { + type: 'manual', + test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + }, + + has_validation_set: false, + has_train_set: false, + + task_type: 'free_form', + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ], + + preparation_strategies: { + // A recipe for preparing the model to perform the task by configuring its prompt. + // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. + // We provide a few options for configuring the prompt. But, the task creator can + // also provide a custom prompt preparation in the task's Python class. + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: 'Add two numbers together\n\n', + instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', + input_prefix: 'Q: ', + output_prefix: '\nA: ', + } + }, + }, +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus/doc.md b/src/genbench/tasks/multilingual_scan/rus/doc.md new file mode 100644 index 0000000..b972d20 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus/doc.md @@ -0,0 +1,19 @@ +# Multilingual SCAN (rus) + +## Abstract +*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (rus).* + +## Examples +*Give some examples of the Multilingual SCAN (rus).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Multilingual SCAN (rus).* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN (rus) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus/task.py b/src/genbench/tasks/multilingual_scan/rus/task.py new file mode 100644 index 0000000..b9c0912 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MultilingualScanRus(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/tur/__init__.py b/src/genbench/tasks/multilingual_scan/tur/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet new file mode 100644 index 0000000..7595ba9 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet @@ -0,0 +1,51 @@ +{ + name: 'Multilingual SCAN (tur)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (tur) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + ' Shane Steinert-Threlkeld', + + ], + + data_source: { + type: 'manual', + test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + }, + + has_validation_set: false, + has_train_set: false, + + task_type: 'free_form', + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ], + + preparation_strategies: { + // A recipe for preparing the model to perform the task by configuring its prompt. + // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. + // We provide a few options for configuring the prompt. But, the task creator can + // also provide a custom prompt preparation in the task's Python class. + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: 'Add two numbers together\n\n', + instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', + input_prefix: 'Q: ', + output_prefix: '\nA: ', + } + }, + }, +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/tur/doc.md b/src/genbench/tasks/multilingual_scan/tur/doc.md new file mode 100644 index 0000000..3c00a3a --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/tur/doc.md @@ -0,0 +1,19 @@ +# Multilingual SCAN (tur) + +## Abstract +*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (tur).* + +## Examples +*Give some examples of the Multilingual SCAN (tur).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Multilingual SCAN (tur).* + +## Limitations and Bias +*Note any known limitations or biases that the Multilingual SCAN (tur) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/tur/task.py b/src/genbench/tasks/multilingual_scan/tur/task.py new file mode 100644 index 0000000..8f08a88 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/tur/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MultilingualScanTur(Task): + pass From e08e107e7b0e3707fca9d74333d41671fd2a3c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Mon, 31 Jul 2023 18:58:06 -0700 Subject: [PATCH 02/12] Add fra sample --- src/genbench/tasks/multilingual_scan/fra/sample.jsonl | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/genbench/tasks/multilingual_scan/fra/sample.jsonl diff --git a/src/genbench/tasks/multilingual_scan/fra/sample.jsonl b/src/genbench/tasks/multilingual_scan/fra/sample.jsonl new file mode 100644 index 0000000..bab8227 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra/sample.jsonl @@ -0,0 +1,10 @@ +{"input": "marcher en face à droite trois fois ensuite courir en face à droite", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +{"input": "tourner autour à droite trois fois ensuite courir en face à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} +{"input": "regarder en face à gauche deux fois ensuite marcher autour à droite deux fois", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "courir en face à gauche et regarder autour à gauche deux fois", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} +{"input": "regarder à droite trois fois et regarder en face à gauche deux fois", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "marcher en face à gauche deux fois et courir", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} +{"input": "regarder trois fois ensuite sauter en face à droite trois fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} +{"input": "tourner en face à gauche deux fois ensuite regarder en face à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} +{"input": "sauter à gauche deux fois ensuite tourner autour à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} +{"input": "regarder autour à droite et courir en face à gauche trois fois", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} \ No newline at end of file From e9933588aee0b627fc61580989ec93128a251a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Mon, 31 Jul 2023 19:00:59 -0700 Subject: [PATCH 03/12] Add URL --- src/genbench/tasks/multilingual_scan/fra/config.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet index 7ff4c16..54b1df3 100644 --- a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet @@ -18,7 +18,7 @@ data_source: { type: 'manual', - test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/fra/sample.jsonl', }, has_validation_set: false, From 5ac8dcf7b8fbacf172817b315a22a2d91ab4a226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Mon, 31 Jul 2023 19:07:59 -0700 Subject: [PATCH 04/12] Fix subtask preparation strategy --- .../tasks/multilingual_scan/cmn/config.jsonnet | 17 +---------------- .../tasks/multilingual_scan/fra/config.jsonnet | 17 +---------------- .../tasks/multilingual_scan/hin/config.jsonnet | 17 +---------------- .../tasks/multilingual_scan/rus/config.jsonnet | 17 +---------------- .../tasks/multilingual_scan/tur/config.jsonnet | 17 +---------------- 5 files changed, 5 insertions(+), 80 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet index d73f7de..e0df0a0 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet @@ -32,20 +32,5 @@ git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", best_score: 1.0, } - ], - - preparation_strategies: { - // A recipe for preparing the model to perform the task by configuring its prompt. - // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. - // We provide a few options for configuring the prompt. But, the task creator can - // also provide a custom prompt preparation in the task's Python class. - prompt_based_testing: { - prompt_builder: { - instruction_zero_shot: 'Add two numbers together\n\n', - instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', - input_prefix: 'Q: ', - output_prefix: '\nA: ', - } - }, - }, + ] } \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet index 54b1df3..bddbe2b 100644 --- a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet @@ -32,20 +32,5 @@ git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", best_score: 1.0, } - ], - - preparation_strategies: { - // A recipe for preparing the model to perform the task by configuring its prompt. - // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. - // We provide a few options for configuring the prompt. But, the task creator can - // also provide a custom prompt preparation in the task's Python class. - prompt_based_testing: { - prompt_builder: { - instruction_zero_shot: 'Add two numbers together\n\n', - instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', - input_prefix: 'Q: ', - output_prefix: '\nA: ', - } - }, - }, + ] } \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet index f7794f3..4fcdc71 100644 --- a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet @@ -32,20 +32,5 @@ git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", best_score: 1.0, } - ], - - preparation_strategies: { - // A recipe for preparing the model to perform the task by configuring its prompt. - // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. - // We provide a few options for configuring the prompt. But, the task creator can - // also provide a custom prompt preparation in the task's Python class. - prompt_based_testing: { - prompt_builder: { - instruction_zero_shot: 'Add two numbers together\n\n', - instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', - input_prefix: 'Q: ', - output_prefix: '\nA: ', - } - }, - }, + ] } \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet index 37c8b4b..59f3b59 100644 --- a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet @@ -32,20 +32,5 @@ git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", best_score: 1.0, } - ], - - preparation_strategies: { - // A recipe for preparing the model to perform the task by configuring its prompt. - // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. - // We provide a few options for configuring the prompt. But, the task creator can - // also provide a custom prompt preparation in the task's Python class. - prompt_based_testing: { - prompt_builder: { - instruction_zero_shot: 'Add two numbers together\n\n', - instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', - input_prefix: 'Q: ', - output_prefix: '\nA: ', - } - }, - }, + ] } \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet index 7595ba9..419464c 100644 --- a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet @@ -32,20 +32,5 @@ git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", best_score: 1.0, } - ], - - preparation_strategies: { - // A recipe for preparing the model to perform the task by configuring its prompt. - // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc. - // We provide a few options for configuring the prompt. But, the task creator can - // also provide a custom prompt preparation in the task's Python class. - prompt_based_testing: { - prompt_builder: { - instruction_zero_shot: 'Add two numbers together\n\n', - instruction_few_shot: 'Add two numbers together. Here are some examples: \n\n', - input_prefix: 'Q: ', - output_prefix: '\nA: ', - } - }, - }, + ] } \ No newline at end of file From b6debe73be528db5de0c263d3652607a1c393919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 14:33:47 -0700 Subject: [PATCH 05/12] Add doc.md --- src/genbench/tasks/multilingual_scan/doc.md | 28 ++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/doc.md b/src/genbench/tasks/multilingual_scan/doc.md index 24c042c..95c1621 100644 --- a/src/genbench/tasks/multilingual_scan/doc.md +++ b/src/genbench/tasks/multilingual_scan/doc.md @@ -1,17 +1,33 @@ ## Motivation -*Describe the motivation for this Multilingual SCAN.* +It is widely acknowledged that fine-tuning a pretrained model generally results in better performance on a given task, compared to training the model from scratch. Evidence suggest that this is also the case for compositional generalization tasks. However, it has also been shown that multilingual models may not exhibit consistent performance across languages, with low resource languages often doing worse. Can we expect similar variations between languages when testing a multilingual model for compositionality? + +The majority of research on compositional generalisation has focussed on English data and models. With the ambition to gain a deeper understanding on this issue from a multilingual perspective, we aim to adapt SCAN an existing compositionality benchmark into multiple languages, in order to evaluate multilingual LLMs for compositional generalization. + ## Examples -*Give examples of the Multilingual SCAN.* +We have five subtasks, corresponding each to five languages: French (fra), Hindi (hin), Russian (ru), Turkish (tur), and Mandarin Chinese (cmn). + +Each subtasks consists of the adapted SCAN dataset into the aforementioned languages. + +Example: + +ENG-SCAN + +IN: walk opposite right thrice after run opposite right +OUT: I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK + + +FRA-SCAN +IN: marcher à l'envers par la droite trois fois après courir à l'envers par la droite +OUT: I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK ## Data Source -*Describe the data source for this Multilingual SCAN.* +To generate the data, native speakers of the five selected languages have been asked to manually translate the vocabulary of the original SCAN dataset as well as either the equivalent interpretation function or grammar in their own language. ## Limitations and Bias *Note any known limitations or biases that the Multilingual SCAN has, with links and references if possible.* ## Citation -*Cite the source where this Multilingual SCAN was introduced.* - +** ## Further References -*Add any useful further references.* \ No newline at end of file +Original SCAN benchmark paper (Lake & Baroni, 2018): https://arxiv.org/abs/1711.00350 From fc9de2b753ad6e2ae652d9a26f2604ec48f490d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 15:02:48 -0700 Subject: [PATCH 06/12] Add jsonl for rus, cmn and hin --- src/genbench/tasks/multilingual_scan/cmn/sample.jsonl | 5 +++++ src/genbench/tasks/multilingual_scan/hin/sample.jsonl | 10 ++++++++++ src/genbench/tasks/multilingual_scan/rus/sample.jsonl | 10 ++++++++++ 3 files changed, 25 insertions(+) create mode 100644 src/genbench/tasks/multilingual_scan/cmn/sample.jsonl create mode 100644 src/genbench/tasks/multilingual_scan/hin/sample.jsonl create mode 100644 src/genbench/tasks/multilingual_scan/rus/sample.jsonl diff --git a/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl b/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl new file mode 100644 index 0000000..dbb8a27 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl @@ -0,0 +1,5 @@ +{"input": "xiang you fan fangxiang pao hou, xiang you fan fangxiang xingzou san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +{"input": "xiang you fan fangxiang pao liang ci hou, xiang you zhuan san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} +{"input": "xiang you zou liang quan hou, xiang zuo fan fangxiang kan liang ci", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "xiang zuo fan fangxiang pao, ranhou xiang zuo kan liang quan", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} +{"input": "xiang you kan san ci, ranhou xiang zuo fan fangxiang kan liang ci", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} diff --git a/src/genbench/tasks/multilingual_scan/hin/sample.jsonl b/src/genbench/tasks/multilingual_scan/hin/sample.jsonl new file mode 100644 index 0000000..fa931bd --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin/sample.jsonl @@ -0,0 +1,10 @@ +{"input": "daaye doosra bhaag ke baad theenbaar daaye doosra chal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +{"input": "dobaar daaye doosra bhaag ke baad theenbaar pura muro daaye", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} +{"input": "dobaar pura chal daaye ke baad dobaar baaye doosra dekh", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "baaye doosra bhaag or dobaar pura dekh baaye", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} +{"input": "theenbaar daaye dekh or dobaar baaye doosra dekh", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "dobaar baaye doosra chal or bhaag", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} +{"input": "theenbaar daaye doosra uchal ke baad theenbaar dekh", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} +{"input": "dobaar daaye doosra dekh ke baad dobaar muro baaye doosra", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} +{"input": "dobaar pura muro daaye ke baad dobaar baaye uchal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} +{"input": "pura dekh daaye or theenbaar baaye doosra bhaag", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus/sample.jsonl b/src/genbench/tasks/multilingual_scan/rus/sample.jsonl new file mode 100644 index 0000000..c8f7117 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus/sample.jsonl @@ -0,0 +1,10 @@ +{"input": "Идти напротив справа трижды после бега справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +{"input": "Повернуть кругом направо посла бега напротив справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} +{"input": "Смотреть напротив слева дважды после езды кругом справа дважды", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "Бегать напротив слева и смотреть вокруг налево дважды", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} +{"input": "Смотреть справа трижды и смотреть напротив слева дважды", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} +{"input": "Идти напротив слева дважды и бегать", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} +{"input": "Смотреть трижды после прыжка напротив справа трижды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} +{"input": "Повернуть напротив слева дважды после осмотра напротив справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} +{"input": "Прыгать влево дважды после поворота кругом направо дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} +{"input": "Смотреть вокруг направо и бегать напротив налево трижды", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} From cea6f78a69dcf3fdbf3f89dd6799ad96be024ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 15:11:03 -0700 Subject: [PATCH 07/12] Add data URL to jsonnet --- src/genbench/tasks/multilingual_scan/cmn/config.jsonnet | 4 ++-- src/genbench/tasks/multilingual_scan/hin/config.jsonnet | 2 +- src/genbench/tasks/multilingual_scan/rus/config.jsonnet | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet index e0df0a0..77a5d81 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet @@ -6,7 +6,7 @@ // @TODO: Add a list of keywords that describe the task keywords: [ - 'keyword1', + '', 'keyword2', ], @@ -18,7 +18,7 @@ data_source: { type: 'manual', - test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl', }, has_validation_set: false, diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet index 4fcdc71..e500ab2 100644 --- a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet @@ -18,7 +18,7 @@ data_source: { type: 'manual', - test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/hin/sample.jsonl', }, has_validation_set: false, diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet index 59f3b59..803c3db 100644 --- a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet @@ -18,7 +18,7 @@ data_source: { type: 'manual', - test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', + test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/rus/sample.jsonl', }, has_validation_set: false, From 9bc575ac987677ac323485978fae6e90c0bab94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 15:40:15 -0700 Subject: [PATCH 08/12] Small jsonnet edits --- .../tasks/multilingual_scan/cmn/config.jsonnet | 9 +++++---- src/genbench/tasks/multilingual_scan/config.jsonnet | 10 +++++----- .../tasks/multilingual_scan/fra/config.jsonnet | 12 ++++++------ .../tasks/multilingual_scan/hin/config.jsonnet | 11 +++++------ .../tasks/multilingual_scan/rus/config.jsonnet | 10 ++++------ 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet index 77a5d81..f546239 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet @@ -2,17 +2,18 @@ name: 'Multilingual SCAN (cmn)', // @TODO: Add a description of the task - description: 'Multilingual SCAN (cmn) aims to measure ...', + description: 'Multilingual SCAN (cmn) aims to measure compositional and cross-lingual generalization in multilingual LLMs', // @TODO: Add a list of keywords that describe the task keywords: [ - '', - 'keyword2', + 'cross-lingual', + 'compositional', + 'Mandarin Chinese' ], authors: [ 'Amélie Reymond', - ' Shane Steinert-Threlkeld', + 'Shane Steinert-Threlkeld', ], diff --git a/src/genbench/tasks/multilingual_scan/config.jsonnet b/src/genbench/tasks/multilingual_scan/config.jsonnet index bbea4c1..286c271 100644 --- a/src/genbench/tasks/multilingual_scan/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/config.jsonnet @@ -1,18 +1,17 @@ { name: 'Multilingual SCAN', - // @TODO: Add a description of the task - description: 'Multilingual SCAN aims to measure ...', + description: 'Multilingual SCAN aims to measure compositional and cross-lingual generalization in multilingual LLMs.', - // @TODO: Add a list of keywords that describe the task keywords: [ - 'multilingual', + 'cross-lingual', 'compositional', + 'LLMs', ], authors: [ 'Amélie Reymond', - ' Shane Steinert-Threlkeld', + 'Shane Steinert-Threlkeld', ], data_source: { @@ -20,6 +19,7 @@ test: 'https://somewebsite.com/path/to/test.jsonl', }, + # TODO: provide finetuning set has_train_set: true, diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet index bddbe2b..100fbef 100644 --- a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet @@ -1,18 +1,18 @@ { name: 'Multilingual SCAN (fra)', - // @TODO: Add a description of the task - description: 'Multilingual SCAN (fra) aims to measure ...', - // @TODO: Add a list of keywords that describe the task + description: 'Multilingual SCAN (fra) aims to measure compositional and cross-lingual generalization in multilingual LLMs', + keywords: [ - 'keyword1', - 'keyword2', + 'cross-lingual', + 'compositional', + 'french' ], authors: [ 'Amélie Reymond', - ' Shane Steinert-Threlkeld', + 'Shane Steinert-Threlkeld', ], diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet index e500ab2..ecb5862 100644 --- a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet @@ -1,18 +1,17 @@ { name: 'Multilingual SCAN (hin)', - // @TODO: Add a description of the task - description: 'Multilingual SCAN (hin) aims to measure ...', + description: 'Multilingual SCAN (hin) aims to measure compositional and cross-lingual generalization in multilingual LLMs', - // @TODO: Add a list of keywords that describe the task keywords: [ - 'keyword1', - 'keyword2', + 'cross-lingual', + 'compositional', + 'hindi', ], authors: [ 'Amélie Reymond', - ' Shane Steinert-Threlkeld', + 'Shane Steinert-Threlkeld', ], diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet index 803c3db..c4d9d71 100644 --- a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet @@ -1,18 +1,16 @@ { name: 'Multilingual SCAN (rus)', - // @TODO: Add a description of the task - description: 'Multilingual SCAN (rus) aims to measure ...', + description: 'Multilingual SCAN (rus) aims to measure aims to measure compositional and cross-lingual generalization in multilingual LLMs', - // @TODO: Add a list of keywords that describe the task keywords: [ - 'keyword1', - 'keyword2', + 'cross-lingual', + 'compositional', ], authors: [ 'Amélie Reymond', - ' Shane Steinert-Threlkeld', + 'Shane Steinert-Threlkeld', ], From f2402ce3afc30471a249310ef63752cbce6c8bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 16:09:23 -0700 Subject: [PATCH 09/12] Add eval card --- .../multilingual_scan/GenBench_eval_card.png | Bin 0 -> 30445 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/genbench/tasks/multilingual_scan/GenBench_eval_card.png diff --git a/src/genbench/tasks/multilingual_scan/GenBench_eval_card.png b/src/genbench/tasks/multilingual_scan/GenBench_eval_card.png new file mode 100644 index 0000000000000000000000000000000000000000..e2488bece3780d37c4fdb99fdd2fe78aeac9fa0c GIT binary patch literal 30445 zcmeFZWmHw&8a51@mIje7k?!u2lI{)(Q9$YL5|9ozAuSz}k^<78bO<6LrF4UIymNV; z=Zx?B`Tl&zIF2D>v)OyqoNL}!-78W48vRViTuxAJeNF?Z};2rDm zt6BsELd_S3si|aYUkAdIxC^R0p3G7L5?B|>aHPJyh$hD0@Gn;s)7}aFx zS2zZ$PZD?&ZVAh_4uRbF8^BGcZ`aZD8?b;K97g|Vj=f*R7I?1ZsLvF9?=xsL<>tuMvN#EDXbw19soo}Qjx z1Al%hzd2KT{#Mn_!}s}V#k%72hT`(ylj4($m1UJ=woddYZ(;YCa)Y2BQnoO<)Q0E! zAO8LZ{lG)P`XYvZUqcEJ1^ft30>3IrAXJ+D{T2M+qB*Ac=Z}aeN)NbbST7Koe4x)l zpXdl0a{bRY!Ea249YG&{AuVkY{rd$Zp3#4|qmlaz5veu_H>h0y-!Fh2qy2Bk|GU)x zOp*UhuK(Rm|64%*w`Bf5Q6xxoVJKV72YW61;l_?__1V#GhJS199U%#qxm4P-B*haO zX;APUh?Q9HP0**B5HO_M zO?eA1rUF|}XI}aX_`)qOjhHHn{e>ZEf z_o)!nf9mgkzL6h4D`E63QLc+XnQi>k^Tndy`EtZ^`pZh&4FB4%44AAHby{hDM_=cP zF66<9e<1>=T$y`8q>-zPjP9c*{2T7H-*y&Wm}(|+eb9}&+)o{dwHhxO;FqI!*-QMi z$6Mm!*ITYdzvHmxwRF}@u%+cV(LHn>_Rn{>R~Co8#JKR*#8iSx(fw9l4gY3vDZK{ohw6TOW&vZei_Av*vVWQS9@7{T z%qm{o!@weps4H3Dj8t&5Va_3%{zdQZ6_`a`r*n?f&$GSPNZ2Sn=be|ABdBpQUezjQ z+F$%mjqjYX$&Y=|2ma!;=zkVgT9nRG{p!;Ti`gf1vG?!r-N3bu-24W{l4_mY zDux@@NfNmf%{Rc5kE~Q40FKt_yQwD%tgU|9h;@i>QpKX}&ot(XEO)*0+pB%6O_Jj{ zk*yML+(P3I<@!}-_JjL2H&(IydP9xhw3K@<3_WWu`5fNdF@tM|0`4DJxFZ<-8f2R) zIrr6Z1!~F%rf%c30uHT*ebG~?aEHVa`Ap|NQmaVfKy7R95#jZN2GfTndB3)q3>T>0=a$HmVU6N-$nQ;C8lB>Gf^ED#$*mr`Uy|`%9Hv0&u9N0I zZP@a{x>7Fq4Vp80`pOt0*iY_gR?q#rzpNV`IeEyKG!HRs5^?)Id)>fSe*AcA?Y zTyVT6&x2>O7==-fdT;jTsp*%GNaCku@LSQUCJ2l3O%aXIBa(C<@di2 zd7%$5ah*JUx03n!zcBy)1_7e-mrt+j*E!(DUrUY*ZQh6ZaD@0)0{Y<=+yU-k3O|hk zOmp&QiKx%8U1*G~d2P;B$RoD)FpRA4=5e8ovLR;UtSMS*zr071P`~E9tLf_wu30}f zruV0})E}?~=bf6)+Gi3nxbZemY5qK&8t>YO3Z@NM%kumg_uO`CGluTsa=WUZ5rq+~ z!qwpP8veBNi;13gVRH7MiIc!08w~0iw>}&md?+?t^FJ9^=mrZHd5W<^bM>zz<>~pI zl?#JRTEhp$?k53WYr8h2!+cqcc%wqAFke<$oC=0{tMaZ zSd&E-Q7+-=W^s=J14e;Yq@VMTAB4lPvx?-o}|gu`rcJQc4u77!?f) z(T`16Zdv2+blmobt}D23tulG0H$hp!S8y~>tJwNRXj+DS-QtV_mZEv9)f9KFAc+z+ zu!0GV(B7zVp9~-NQ&T?&M|3dnIvQr*xFCRIA+xA+vM3Y36EA*hfc0}@HHs!!O)|%O z?VS`kc1Re?0|S~AEK`@S^oN_`BAA7aYyrPEC!+*h;8dF8b9RxpADu)uazj@txrgu& z$s%u?4)P>9PH$R|M=;^?U>V_YHh6e>mUqX6denFMno#YH4B3L~s5m#L4C%O}0nhUT zC)Ca7PaZ0bRJZ+3F}BQkYGPo_%`L##NkT7z6{Kd%_S1K#Mqx)ecFA+W{X4e0#Kp56 zg;)U@v5T+G3ORQ5N8GqF(iZK1rj7gekyag&gv^jxus*3*C;SSon0+#ZIaPnYKb33<`)$(WmTKw|>A&o} zvzG0!x49=x6+3mU5d1n%^@^15y)iWIk)}Ui6JfGAA!9q~->d z$py$ZcW8ET*^K2HtaRh@Nv$rmTDzhd%^bk$)I+wSDKNuE_E{#mwS4R=hWXx>} z&aK9WXP12H1#4BS3yl!HQ{kheGMdXwE3SP(78!DGO5?JXrZiKp<*;Q?x!*+7oLj$A z=nHz5;8d2xwU$g1x+>4iNGuVD(TUPuqDzRBK{&P&icK%NBRSPHFUw_zaHE3XH1UpI znk}+H2HZn3!+ux^0kgKny*LgR90>*jl#>&O>{G*;?G>5mtxPpPefQg}ireAou5{9y zf}z92qY6JIZ_ih`NuEDL3#kR=O#t=d=2iQ8wr`(R*9)OqwvcttpWci(PjXuAC#MEp z=$`FmAR3bF3q_GH-l*U?;5Z8?*_*I1X-5BR2P_*aF3oNA3x~u0dBbrIEpk>Id4n}3bfRmuznIMvcPl7xO|KF#?v zfR&{+uQOfK_YshO==&D`J};As3zA{{z46Wy9kHyY~Q7c6sX>TpiX`xJw~ zE>8Ny42FV*(UbJl=PBRwX1RX5-ICFhXxNzlB=3xR(y}} zxz2h#)Hdgps5XgwM>Ie87v#C8M{I8Tt@}IsRJ6aFp4tSMBwo5jhS9y=O?a?TGf;T1 zvOXF`i>uGBiKI}9twI3B>UtQ3!++!op6bV(X8zfd>egY(JjUN@(f4d`c+P{F6z}!8 zUgQ=ZRXR;!G`aVM0?TFLnVi)cvxr{Cv3EHGg;oOm)y(%_iDHwt#m7nQFMqRge44E33?&zf#-#J!WGhQpzD=D&{Ow%eNQ$}%aeK(T0rlz)fI{TOnHXk8nBs&|GSoO<`bH=Ray< za0i_S1!DpEx9cl08{E9+kv}*a?z-VuqGFVmaG(>1OTy zjb)hR&53pf0)3(NBL{RXme1;w^1tfM21yg$GZYX}e8Pk1pNX#uwWQv2dxtQ^^cOmV z3fvfr|FZQFSp7M0HFwf_aa+3AYi+qF+3OvEnHh&od0BNs{DEO^1JgHtN5@Bzz@A@ zg{olpP7pvv^XZWGAST*i3FiEK{t)a@!s6&FBQCN;%igD#Gl6$ER#MvX7(W0n`q5DC zMqD$Y#M*2BGGiuEFkbBFiPu(X@$A~pT%)6bb1xCnE}iS-QfFv4KtS0oB;YzrVcs)% zJRO2IWvZ3ig6sg$(ZTQ3r{RD!l-noM#qw#*06@|Os0w+p=7K`d@q{vG*v~fK2oz!y zE-}FUmVdm+!H#BcFz^DbuKNpJJcaTkx{t)o$>ae9m&+uH4WA`)^^sV;2DOt%MR37w z84LxEwkp4_0e~&ts)7smA+ng}xmlPRH)-gpq%L{m_G7!_bvlve7J9 z3+1Q(*+SDT24iORg2#}^_6PeZ7l1E4u(C zzA@m|oK$RAS?ah03HP;hC@PB5Scx9}jHUmpJ9%>Z`$&)0O#Kdft;U4Do4AGn0>a4g zwoExhSm_%52uJu=f)tGa;OM<9njw$Z08xM0l^-ubrU$T?(-3!mk5<=;JpQZNAx(Kw z`pUOSf-oW2;0fMPhT~__#$cA=?aCGoJ*3sX1}ealot^8&qPGv*VE6cngBo>*z31oc zP{$fCw-nqYvMko8_XiA^xDQszH~%O znVo3PI#zVAiQ6gfS7b*-N33i;o&CglCIJtO;kG6d&|5y+-w5e!;OJ}|O*@AT7qy?3I< zDBCvRDIme#5Ptt@wVuMN@d;td0hvLEISHL&8Wi4x5z+nE&Fk5ssE?3Ikjda`yI}Av z#^s3Z=l1TO6krS~SIcFvuPuTZWy#gI4u4%JyCjbkpcKzkN~V|h=vDj8X*9jQPJEJN za+y8QVRp&3wRgV?XspN`7^on>&~Z;H33=oN@$9+hv||q)6_|BPwRWEb$*pvvDSf%U zJ!_8u^Q6sm`WsHXomz-egTGDoF3(j27?tD>CNw=QZTlMXxiOedg$mEtN7!w#4kkCq zOq+QZ*M9n%6A@$^>kUF5O(I2a`8Tkp;V_K@M@M?zaDc zx+wQ-xu&(^SIZh=4<&S5(Vc3NpsDVU(;h+<&B$Kn&ps(l-GI6G-GqE^+CwS#rb?r@ zRtOS$xC#)0vD@B!zeknk>lMPiMyKrT?5T?zAYhUycgWwwx{J+|q1@k)hPmNtv5tqw1u}r& zAUq+yZs0kXtz|0r2haYc%HN{;hcK72kUsy<@vAy|(gglleA}bl_s%?1kKMe@|1Gb$ z#RWL2+l@M@-Iq2oDtqH0;pS|vl}n1Uah+UD-)yn0I1bkG zx{CO^=kwH!hKMLzoaXtrml>aWR1iPW&wv_eNsNACEcaBSUgUPjB=!f<#skDl(fl&o zeoD`nn`&-8?Jb%HV}Xuj;ErmegQ9dlqV$FRFY3GV{*D9T7_=vO3huM28ajcX(DZA4 z1otmbrte;J%|1hT#yh~^R>y=+e0#5v&~RcFIGtz<7~OnS+Bfo|Ef=-1l4VLW6oV@t z@SM731fW2U|zSWG~~;hG)djF^sN_v9}}Q7(*K~Z-0IhoG}kQSeLvL#gF&Ut?67oensu>3B|HaQK(J12T*|aEFb~W(0P#3mN%;ASyNC zNM0YDD>0K2Gy5n$1zPZ`&})+(Dxiy1U#w;HN0W7ut9$c&rlqv!Avp=bo&HSe zNAQja(e1RI{j8zoQzyv+vPVrpG|LxA-NT`MtT=Oz^#fi+zQvFvnHq1b9koxx8gFY+Hb%{jksXB3t;OkEXzVeh!a1ZnK70u>^99pPZ6ee{+0PGQ$4(` zOdld@U6AI>Bz!%Yb94+zh4;12EBg^xTx|H1hq_>51w;Yy6B6X15D-6m^ncCXGw`!Yp!l!!NV`d>>_lO zbt*BGm3Xr=nYzEh+gsZ^?^$3nd55!mRTa8bl#e!2wM z`Fne2pys=Z$x{^9srnf(Q&L#AYca$*|2h1{yCYPYjCr*L4pL$y;Pl6f{_iWb?-EWk zj>{(Go!{KVzx%W#8K+v9Wz_;0R&?p*JX`iUMe)`}wD(e$Hx*uTkN~q^VA9RA z+XkE>9!Xz6z-MLBM{G=r`QcRM$%VXVf%w_{M|MYgjYw~1#2HXTte6KSu678^Wl4L| zCmtx-mxy$~0A z8M7+AKb5>Ah!moIqk7d)xw&HtQPLSDimQ3FK z*Q){)PCUCD+)hu$PCrC@)0`}{pEt|?eNP3!oBoD^Ox_QvtC>_q zHjVhnl0vCmzkMI`hAZU~mQVa>Sn>XHdgL??nSP_3eOuNU=h-^2+L`I(<9A8kRPrLr zuL&(NXohj^{8Mc(zci12uDyegX%j>VFXA^u|0cpN<>`w=lihJ@xqQ+vy{Ko{Fn#Fe z$O03?lcKvbl6(`3EbFA(Oc%F1*Ho=RDGES+goOX)>nS~Bc0EVu@fpi#_Ac583IZ0- zLkpng#S*iJ!~!B9prgdOvVQ%A_*jd*hvZE%K}5r;h|V|`2^WnSUQ@~8BUW9@kHZ4S zegp|i)juQc9!+Nu5Ob8J69dCVsN|IT_8>xv>oJ1rl43Gzq@Ng(vZ&PTK_&qAHD$c) z#g{~wWrd1jonOMDPYt{h_dab`8Waw@{d9FU_Sr0K5hICabxgWLe4_It zUp!nD8x=e5S&F%xX);d)K(^H&U8WHoo<5Efg?XPCokJy=WyHhVR%Ij%DExbGB(Ljg z-q%G%&_03RJK6ruIaU;r8%H1jqf^@0de_8N>p(y(G%JHI^!aFj)!x3OX^AV=K9`bY z_jcoF@SdWLfts&Wiu9&oMbD}0d*xu{!bp+{j%dtBg)CypgG&o0I~ezvh2P7DZ8k?r z1^KUr%nn5&1WSeZ(`sr7oBpOHop0h^dRn-}hDr?=aLHtfWTDD=mi{4{s!5wi)~D^r z2`!is5tH7CA`sPx@y$y3YpW;ZqApjQYw$%_h%1kIO zf-Ce_EQjK!Az7rd^=N==`@_dRIc>YH_|-R$IB?ujJ1JcZBXeKVgxusSMVAqrCfZ?g z$eZAKJ$CEDV2sK55}@tg(;HE&ic!hrock=aBR2ZQz+i9aQYcMuP_U>YOfW^^AxZu~ z9KxGbIYngR1|LC+;@83UvNb~tlfHH&aLPVvz=;K)^Q z7L7JT7388!!VjUpUGV!NbCd0!62z`?_oe2BPTb!?f5a5|$ZJ>;nE>8ea|>D3aV&`+ zTWgP8I949#_SCGP2Jds@S~~?0Fgd?s>IFyf=nXlF1J=ahvKMj1K2|o$HyR6x!3HXYcH-M5M9Qk6ebz z2@AWCvnV*r=r9g8N>b6pqt9yKq`r)lPCTm03 z*47qWX?8|ksid^wm#B*Q7Z=8N&yoaWlq#AP%j+1UaShC+zFOCOYxcRgU1){T>8f3K z4!2x5c;{n`^@U(s?1g0^#I&dejGc3@L}Ww@=_ZO@^z>$oo@>&L_k~=M>Pd$3Y_=2h&u*~LB7lFp* zih*UAJ!>+zKi!avP^e#u|I)5n83VBp_2h*dKjM_uHMvpf(zohVE!1qw0({Qhf|BVx zyS%_cmYxEsN~uGWyn$>FNu6WOus&?ritu?ISFH&-QQ=u4%|B{Hq`E$tlRAZ(1SWn5 zc_O&Sx`L>q3&fDJ<_(C`?;kB|WUx8~W&ul8Y!B}b*{6^B-q4%py9G%_+YQB$As2_Y z!Ob;PWkXHMxV(Mh|I9pT=tFdxaKgk((3~A;R#NE8%!ltx_+zXy1Z%s?ILwM2lG59V zRVX0HM{?Syl<2KfR0JG^H0bw=>C6$6hvJ@vXGjB5s02tMw!2Vzpp^^xA8mp@6gGa# zLTO*S>cOz6z9zohqI(geHm~P3|DLx47gH<^dcIq1``1Cjr&BjF{CD_HxSzCx`gW1! znLCQhQOgFJF>+~rwg!U8VPw9=;q!IEtF6D8^{Lq2I||Jz9@SB=)8q0wWfN1kmW*^G zA@}wz@l^}fHy}uVioG;GBbh5{caOJc_SLJBzbSmx%Bq{2flM-)RIFk-B}neY0IgQKAt!46WQEH z%Zfb8_V~Y%hLUiOb$j!0T3SNxG>|qf23}49q>j3m6s%7Dxs>+@`$ z1Lgi3*o#bO`6Y}V06!USGhSqScR^W~IH8p{)j$i4fCPrDZxxL{$xv}9Nd6{lKz7Ax zr=~v~3InwQ0caUwp66TTRb7DH&U2_A;%56n;%)R>zbdddw`On>se~;sl=~!_h-EgY z039PSrzD2(!fFj)WYsS$&D&D?U!7t@ZWqP+38Ym6c~e0M2u?_=nUJpSEOi+NtHqVP z<}z^|=0|;oF{uwGkpQ^r4*4E(K-p=9!K1i01PV>yG1vgqHE5~~zHeIr{F4Owx`B}Z zkh}lFZ@;^nvzn-+r6{)_)~R8D_zPva3@|%OIhJt&?R&K@?feqx-w`sn;(T$jN*LtZ z5PevYVZmgp{+n!D`GkIM=Kp8v0210DB@FOfw3@4Axd(=iIM)AKsmfnZKv^5CG@NCI zWI(q^=H#_;&oV*eqY#q~@J@?&)+v#N#{CqoDl9n>a~ls%(A- z$~!euEpY!i)QrLTT9q2^|7^2Ls2dU4E|9BhJNf}&?t`K<%c$ycdGcO_xXyME>Y4bf z_(M8%EAalgxppRt)!7s?fYNTcbXpa7!#eLasp@PPcro~N<@5v4VU_`Uh@tnhXFUW{ zm}x8+j1n4`njUnUL+-%e90m{bK&^aszCm?0IENxj(U<;&IFW( zS_C<(H0~11M-kwN*gvZu7kq~#>=+Pt3f5nHVOA!`{H_H6x$fP@MJh<`P7|yMU2+6^ zYxJ==@N1UA<&35G0RcC&CXmTPqW6Hqf#@O?YymJQ_TM!rmoFIjJA4g@`TO)`*!mz# zCdGNij4zakic6Krfc+;qATuqsYzPuP-gsYRGV(nI;M-vL3l^l8Gyz+_i%?w*9||yd zcu%~|0P(eeH2a1hT_A!%g4FE2o+YTk(g@;cXiAfiWY6y!-cp1EovuK=iXu&sqeL-f>!7_*{A>@z_6E_VqkC8>tjW=gwXXU61y<A!O6f|XA~VSuu`Vk~5LGAZQAHH3Od zoP4hXc1+zDs%CO9NK#hlW4Kh^LGO&Tm+l0kY=&=Kt9*hG+G^XOtVB!5sN`KM+K%%@ z%M)g%yE3#xHDh_1vv`S1rW@fNR&`mId6jw;r(Uj`a(;atiWK*X)(Wv*FhlpI7gojg zH+;t`$X5o^%|MOV#I+b*K+H}N1ulO#q)>siBDVS~Lq^Ygdlxycu03FFRW_$l2VU23 zY?m}2FuZ$0Q{PASQuYc&)@s-ECKr%zJl)|NIorUC+}l{ClDxqUJeQfVE3fDR;Z2I1 zlGS<<`fdwD{q>i>+krteEc8&ubC(WjYH~4Cka&Uq=j=8&&CA`@xd~s-F=@134uOqS zyTW9>d#*#=3I<@NuS#ftQ16gm5tn|@siUM*Q6zhYe62j`wE$G$iGKUZcT&`P@V>$x zTxazb(Y>j1eYWNwo{Rog3MF43UK|SUnmo_(Kb=v~zihRUD;)vZuuO@gdZi$=djY^> zMU#J++kzeLzCuA?$RV6$QCn#w8-M*aV4372STQMc`2-7fe)pdVWPSX~ccYpsSN3_C zDZBfZ#Uu1S-okg`ioqtpPXB!`z5PC72q!fpfz2jFEa5RT6P<%2UyB`mVi@a7QISX3Gve%SrJTsP>8xf z0Ek3QX!$kd?4hD26piwWkXUnV)5Qrrw3i=juiZBp@(QA(4;Nrbhm6AD=vhQG3OwKP zeqlq(fb(T;p$2D{3G$=l5vR>pDOx>m)kK)FK45u_9H?T^xdXc;{N{A7YWM)~dQ4^Z z`fmxCn-;Q3@4RE7~= zW>G>;P>6;rac=GfG(0QZJ!CUD51tT|R4|)=o@Np5CFt9YE!ZO!okZ0qZQ?dAw;D8A zn*N4S76Kv;Abx5P)y*Xo^ZrXbXQ;|Pu*PbN&w+`}YIYxE-bc%FNPg%^%)a=6z-`XrytiOQv0ZgVFH5y|q-!|g~JjqrPrYj6!kPk&@!WOF1G%x`~mmnPwr?i_(Y@YjVNx%~CUJ1Ad4~44vrf85Yt3F@a z`4zqnvBM1*yBL3-CC)s~%tB0ydaO2;GF7CrjyJ>e0l)B`94m}3#DXKHh!&;kptu;` z2xKIKtt(&zE4V7TV+ia4FO(g(i6#rj+RwRoen7&-mDhw0w+3JkiK-_BO^7^nfOz4 zxDx+R@I|yU#=UF(AiJPFroQ55uK+T^22!C5TZJWsbQw<5X|tA8n8>xef;)OM4Ehe99^Ud0gBc+uNzQ%Gdr*1WN3+mh%1e4 zO-1|V=O`c-o@*U_>5W2c$Dk@xUcB3=J1UKJ$wHmS30x{1iH!lIp~G* zeNxFqQ2p1}f|3;EEz$rCTy;dzO|HXO^^KEkCxl`PEv{bg!yZJf3-$*uiuQ@@#>vt= zu}7G*vlB<`uYJ{KWwb>#UKxeZ7j0h}(V6@=F9E^H8n>S{Y{mymdLCa{2B*a;_mcmU zSGmUr1-J<**#vd}P0VP6C7VCK+_w;v7zin=J_02~7Hv@VA1V+$4SM@t4zF&RLE&ep zkPrf#!9U*VqWw?*UM@dm_Z&BU(*L^wfRAJWWl?pjp4a&gQ@C%V3gGAe?O|})5!#n$ z%p$|EYKJ=(u|C=V$NQ^7KpM?3`p;3nwDkop%+oNa6+#RMoQ3wGtWf`fk>(PxxiKeI z1x;D^(yw&zs|O|~p|FkG5^zA`=AdqfdVAwT|GA!ljIGU3G+Z2zPqcD=0N6O-3Y;J< z(M|nOP$M{-&S{Xn@pY=15A)suC;`MzJpsCdLcD(i5EKB#>~j4KxZ14-X^b!=TrsX8 zfw@=9KsCmWm%Q~d>4Qu)sHuPp#7G+lS3$l%SBegbw7gO1wb#l;yZ5(0A%=GZeeTsq z)$^PEKr$fJzdIlv%Pp|wy5i2u6@uL>v|yD4&t^AI?M68v0$zG?R( zBwT|$h~xRMx9VM>785rvgZw^j-GU)l^(}QbKn)>I1wp3C2q<9Fuz3Nzpl%&)RiYFi zu^)g`LKxI!QRxeo?D%7FEXB*6Xw)>bcPkPzrGF!5zdl*=V8u(%^7B62N!5p*yIY zTpfUGs7Iw=ipib=@f%Zq3;`$^MNUw4{0bHOCZL{nK}n|vAfCwuI#(j(h)wxW<)J$Z zPy*&ID237oR>8WohVOr4sU{#cC6Con@^AqRYB%6$6iCoUIG`+%#ow-o1rK8{K#aP< zn2|vAtkF3HgW*l*JcM+hCJ3ob3j6Q3C&iMzVh~XSB=Pwv!)y$Eic|s%qm3S%>GF>v0 zwRn~K4HyR>*38$5thS&6pfbG-DA|yE7An**c+G59O8SI<7_~u3}OfWJ^type5QbW6c}F z1-i_0t-xRGTXKPN)WIeLs}Md8>R?JPev1Qt9PERkF(9P~VA3*CF0QCjZ?^yG4=I}N zE6^z+r~>B0GStukkJ99SLi8IXHpoNkAu60=IgS&v6ivWhhHe`upX;_7E7;e5rB8)d z!jUKflF3NYG~0#n=m?||rB=X!y2`6QlO03CevM8Hi{@-Oc=d7_q#(EJTd3g241Au^<2x`3wgqO|7~}7{5!aw|pEA=gq zn!Hy)6S39pQ}90du`Zg4t-p@?^d+19+*pe62WyfTdRj;%GV0(tw!;UE>7P>Y$q>ja z4ycZQrZk6pCS6XxKKpOlqPd{h@Bf>&FnT8H#S@zwBa!Rv_?Ox{ zMD(_$!F*Mlip@R_Vv=TJatbTVZFQf0D)SRl9chTYWzHlRu1>K{v07w zsQbHoPIk}69!xt9DSj-fBo@o6H$Z^OpmwE1 zO@5MFf|7A=tWtR`FcIWJ!ILf7{dKpeTn{%-Ce?9I3T{CH`!&>P^f~T}i##8$%uPXO zt%4hA)^NlmNWc+7jNA$2pu#E|X3Mk@@rak8mb=QLrZy~0*_DT?cjm&8G-1op=no{nBWSX~369}Y~$`UShF{!C5O z4V-5N+jfmX9G9yQml2Bq)O+PzhqkI$=UVOiY}HEZ!k^FQre*bNpHvKib@kH!7avLp z;yXrZTiA(#+KZg)H{${!W|NzgSvnL(;m(M6{^6_$2>d+IvjCogMO!F~+JZnD7FmgY zuAkX{o=uPhHNSC5@voxsyjJ_evl4Re&biSPi{d;3Q%9+^Csy>luCh+NRD`w-LO{W^ zZU$7x34+6eM`=}Nh4KcqAAr5f`>Nr`^BU?8Q16?A$LvVm$jQL9TU`37pgkwOI2KAz zLsIJRLY?`Dx_PME$e-e=q1h~FiZt#Sg>l)(=+gXaTan}G)pO$ShNEYh%8=hiA?R8^ zrH2W;UMq?g9`Q&+w}}9+={SWN0)wgVzzIDFvd6V4qYf0j-FrwaJs|{sS3`Xpj$~!+8M7 zTm9S*qpu?P`=Ql5F00Z-#pi5qfxf@J>8H=<;_lhMnZDde%F+!fh3u)T0qibNJBj<- zuEA1l#nqAau72eA%2^OX(Cw{=)sTnN5C$pMULkFS+XJb~f3+!h-n?4;T8*oZY0;M8s`ugzrl8}d=vUz^txJ0ggK}`9 zEZ5sl@9fsq8yVm|+FGN-=!$w$l#R+5 z72Xaqr7MmYi*#>c^&mV2O$rX;B^Vzt-usrHIe@sMSHGJ27BUwWf%tywabNambHcGz zi0olqjNVhE(S;pks z%rudn0j&e)oI+4N7~%*(3MG7a;rN|s$CJJih3Rb3Qt61D%}z=2tB=*sOcF}siU!hx z>zT28+taFZRk4FPiN%z}K5cF-(!>K<_?nt!?C6Znk z`#ZEO6C-3FtA+;A5&Z7$#;eMuQ5dIlhK#M`FP1- zi{!A(ldR`gGn^9ZeYF0J+SMrycVHq$_Wk!Y&!anje;B8d@ z8}nq>3{5hq0nMvB@w4*O;M!UOLhPcDgAGU=5pdD`Gk*Pv)Bz#%;U5&Igdhx#m3}V0 zn%99R(CTtFg3EWTz0ieAzFW{F4t7T5izZZ(hElo8i6V!Q|&~&b8_jvRE;BtR9O-%GM5GkmSHDcw= z2XMXBRzg(&V2IBXM^xtBOTc88rY6Qv0l<2yAr?M@|Egy9ylD6L%vegeJsq_)<=jwM z0w$*JFubfF2~&lcId?W`Da76)kQn%A2JC&k$;S_|r-wgl-Rtsx%uFwhib^q9DRc{u z@{n8cdzkjiFCbs5ez<1MA=QYr&%m?+Y$2g8hd>lb(|RJ zSA2ov$}5%pa8YsOMHyQu&zf&4!7-tjwwyAvApN#npwh0e`o&K|1MQnQ(M8SANECE& zx+3zg$aOz&&aVj->=XR#wRYWerI=_qK6~>hMB-F9c_8i!&(LHOEYA8zQ-2Y2V~haB zwbVp*9zNl&Bop?M!Yz1iKqPSvPs3XQCuDgO8XH;FTwASNEJP)CfQOHz=2)mc*sj$9 zMX)~tp$V}xAU(0nIYdCI*(Q7}Ch|Gbd@GnEyi{!#jvo8$9< zfe`c~HEOmz1l!A9M;}}DchBJV859H$XRrsN$-f)>f@hBB1NPu%2AqZwW8va|Hwf%Y z)Mut$TaW*Jm{RZvCP{+(&jI28zQF;W_pnW%M_l%g!UJt2=)@HTK!4m zAH7B+_ZVz7tIwu-w)B&2Ws3Gl_>vAI> zLgf!zLBU)GA_eZ<<@ffLlR?gQu^65KCVN(zv{{hl=u%>>xDPc`KrOmXpm{eO5}()r zQMUl%1uocJ;M{euAoTDAlRYRWjI+)>vhNFx?oU`s!j$p1-(YA>kwGg)Yc&v0dud(* z!5;Hzz=gHR9LC*{j0{Tf;6l*03QW>PmFSwWi)#dVi#8DQai04d9!gCKJkkkBa` zx!D(iF%7X$g4(|<6lgoSIx*&Jz#hW6+WDy71Ul+xlg;sC04cLuPP2-`pbj{h!kT z8uztB;!KRdtW6Ci88x01HY)^aYmj3mI0*`kL5w}{RrDz^M*5SuQ`(k7F#GH+M73Ey zKLzP-OB&(5rqw3QW4lzhJJ;Q;vCQ?YNJ7)>k`z-j}5d$ zd*HQWT2kFX&yf)!0QUzSX)zF!OJT@X$p5Fk>kMlu+oBRtDZz<=8XrYkK$;R{z=678b+f)1e7A8M9PQ|96BoE04h}=fPiKUodH3lL=Xre?eeT|C?G;}jGKhsdSzss|UQ>*_W&1mjPK$5Xcz=OW53D)H0L>%_ zQN$5o0=xoMKH*ZfeD3RnHD1rk+v5%87Xlj$GlFfVpqzFT;ur6@xqgl zEi1kIfIIBybvej#bW9!U8{27HQULUh;k9HR2$()JM-(cdBt?sh2t+`aly5;0B zHaLzsT2pwr;q7n#c5{s_1BIxvb6^}pom9F#zpwx$hVGn)OGb-gYO+Hp&?MMYz`Hdo8>y0asnmDHH^g6C ze^Soreq*W1sstiop(?q8J<-Hf9kgitVb1jh=Ha?e!YORXAZ0o##HF~ZH7-Je=vRx8 z4my#06k0Cq&y$HFx#r8DX4i1ybl1$+t6uX}3s# zZ&Io+>`p4fjG5b=R0+d&`08aE4)AcM0}~rMZ1BC1)Vc+SD}InE0P1*AgL?5!SkY+$ zIlbkIHJ#pDzlfLRS3Y4Lf~mDQXP<-gng-l3DeaV&lcJh6{pB_5gKstW-9BEnfmgoy zxFnK2b8JAH*gV<4q$|YyBjtN>Bdlo~?9Y$v!y$p;YjR1;*-JS^58qu^Nt*Z_l$Cdx zx#AplYR?{LWq6zliS)$SzNuc$p0JyJ4lb5G!)gj`1znUq#nkKVekU50thxz>qBM|y zdm$QkgaEEJIp^9YXtv>3AV+v*1y#1c1pJ^_>rtP!Y^W5~^Ge6~y?h3)UOXj2>xj4@qx{uwL8zx}qxa$J8f5qZH725IVqVfi7($px z@f#dxS}YJs(4bUbWpC5$ea?m-oJKpcKJI04f2&AP`~{TgiK>rj`N5t@4q=V0`rA&&HVeEp}*dp`sz=>GZ8yKQvb{SK`ZGIy94p&R!! zG7awsEKhgG_LZs!E6=;nT>$h-)7>3r|H_T42Er-rl`75rk%ZFa+0IJluxe$S3?r(G z;Rioi9~Z#kyP`NFgU|!DzLz7LD^DWG{q(%51|hMKr)MGnEAk4o9@Cvp58Ywo$z&j~ zdwU@zrf=b=n9pi<8VPR_K-aSVrQOfKirJdrqG3QYpURaZDAN= zs#OmaJE~}(fBK3b6Ns1h$FA0mH0OpwLf{EuGT>y!Kwk@-6mAh9;jNUJb0|!@NE%_u zV(;#PHhfDg??G*|Fjkk z?17qNtXj*1dEA$Yqk`L8>aE><%SZNmMu$E6Sys&l-sv+Sp%-MXgr3|`kedAzo}ZLu z7EzJ@5wdb&V2;=GdNi|@V{IP-2ITRQT&whlCxnqiur3edg07@Qc6az^_;%||saCb7 zC)>R@%GU~iABIlVNTA42P}8+JOz0t^yb&hYq~njhu*Rr|iYTTz6f11+8-1+C(>{n% z+8+LHtZ_>;Tx1@LCUPQ&KVaxfd zcfyef%^`drGJW6b)%pg=h6$rPgR+6M)7}nt(B6Uv$dRQ(dk&yxz zai?^Q@Ub_;XZ`6GM{KyuHWwo?((B|P8=>ecFkV3m2R)0(zw!lNpW zXt2?rOCcR-W|x#KIjivti#2J?3Pl0({>wY7GhhaM~Iw7?MOnk z=MXzlmgm5SBU+Bl4uksG=Y6Z`6RFg+NFCAcl03HNqQv;>IZ$ycbJbDDCh<=x{-#+& z2L2m0!)VouDM;;vmZg$MmDpnX*lS2Z>m-t^EB|0`vc_0fycAFtRbXye z)aDXFG9e_Y05Z5mK?JFH!3EvBkim1xzG)SrWP^e0LSWso@1*#D>m|RYH~*82L3||0 z>@AW)3!<4(^MG9#0?0#Uwb8e$Aa)%Ts>)wi z0?uOV&LFW@2$YV9RYbj{AW7MT%mZ2UZ`xJ@i2XW%WjyxHL}J(e(4tqp8)8c!ZMp`c zL@_hG>8pi=9S*c|!DH5!0~%E?q@4$B{P`VSkeqbuj6MQ1qKZh*r>|S!W=jDSTW)#a zHlQo826?8nLDgTT7PVk8uZ^Gj%a}_Y98jI73-aOGI^dtclLH8pJiq68Z9pK4R1LLJ!mMV!~0dGUwghNtoGPys2x1sG^0lrElf7`8(algieggZ*O z6X*XI6ynTAXJ=;_>+9 Date: Tue, 1 Aug 2023 16:20:25 -0700 Subject: [PATCH 10/12] Fix subtask docs --- src/genbench/tasks/multilingual_scan/cmn/doc.md | 8 ++++++-- src/genbench/tasks/multilingual_scan/doc.md | 10 +++++++--- src/genbench/tasks/multilingual_scan/fra/doc.md | 8 ++++++-- src/genbench/tasks/multilingual_scan/hin/doc.md | 8 ++++++-- src/genbench/tasks/multilingual_scan/rus/doc.md | 7 +++++-- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/cmn/doc.md b/src/genbench/tasks/multilingual_scan/cmn/doc.md index 56d3000..afdd16b 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/doc.md +++ b/src/genbench/tasks/multilingual_scan/cmn/doc.md @@ -1,10 +1,14 @@ # Multilingual SCAN (cmn) ## Abstract -*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (cmn).* + +This specific task consists of adapted SCAN benchmark into Mandarin Chinese (cmn). ## Examples -*Give some examples of the Multilingual SCAN (cmn).* + +```json +{"input": "xiang you fan fangxiang pao hou, xiang you fan fangxiang xingzou san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/multilingual_scan/doc.md b/src/genbench/tasks/multilingual_scan/doc.md index 95c1621..bb787be 100644 --- a/src/genbench/tasks/multilingual_scan/doc.md +++ b/src/genbench/tasks/multilingual_scan/doc.md @@ -7,19 +7,23 @@ The majority of research on compositional generalisation has focussed on English ## Examples We have five subtasks, corresponding each to five languages: French (fra), Hindi (hin), Russian (ru), Turkish (tur), and Mandarin Chinese (cmn). -Each subtasks consists of the adapted SCAN dataset into the aforementioned languages. +Each subtask consists of the adapted SCAN dataset into the aforementioned languages. Example: -ENG-SCAN +### SCAN example +``` IN: walk opposite right thrice after run opposite right OUT: I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK +``` +### FRA-SCAN example -FRA-SCAN +``` IN: marcher à l'envers par la droite trois fois après courir à l'envers par la droite OUT: I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK +``` ## Data Source To generate the data, native speakers of the five selected languages have been asked to manually translate the vocabulary of the original SCAN dataset as well as either the equivalent interpretation function or grammar in their own language. diff --git a/src/genbench/tasks/multilingual_scan/fra/doc.md b/src/genbench/tasks/multilingual_scan/fra/doc.md index f41c6f6..a2bc615 100644 --- a/src/genbench/tasks/multilingual_scan/fra/doc.md +++ b/src/genbench/tasks/multilingual_scan/fra/doc.md @@ -1,10 +1,14 @@ # Multilingual SCAN (fra) ## Abstract -*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (fra).* + +This specific task consists of adapted SCAN benchmark into French (fra). ## Examples -*Give some examples of the Multilingual SCAN (fra).* + +```json +{"input": "marcher en face à droite trois fois ensuite courir en face à droite", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/multilingual_scan/hin/doc.md b/src/genbench/tasks/multilingual_scan/hin/doc.md index 4780f3c..5c7756f 100644 --- a/src/genbench/tasks/multilingual_scan/hin/doc.md +++ b/src/genbench/tasks/multilingual_scan/hin/doc.md @@ -1,10 +1,14 @@ # Multilingual SCAN (hin) ## Abstract -*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (hin).* + +This specific task consists of adapted SCAN benchmark into Hindi (hin). ## Examples -*Give some examples of the Multilingual SCAN (hin).* + +```json +{"input": "daaye doosra bhaag ke baad theenbaar daaye doosra chal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/multilingual_scan/rus/doc.md b/src/genbench/tasks/multilingual_scan/rus/doc.md index b972d20..7bc91ee 100644 --- a/src/genbench/tasks/multilingual_scan/rus/doc.md +++ b/src/genbench/tasks/multilingual_scan/rus/doc.md @@ -1,11 +1,14 @@ # Multilingual SCAN (rus) ## Abstract -*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (rus).* + +This specific task consists of adapted SCAN benchmark into Russian (rus). ## Examples -*Give some examples of the Multilingual SCAN (rus).* +```json +{"input": "Идти напротив справа трижды после бега справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* From 0dcc6b09f53ebe87c621f9e8385a07a3f61f416c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Tue, 1 Aug 2023 16:27:58 -0700 Subject: [PATCH 11/12] Fix preparation_strategies --- src/genbench/tasks/multilingual_scan/cmn/config.jsonnet | 6 ++++++ src/genbench/tasks/multilingual_scan/config.jsonnet | 2 +- src/genbench/tasks/multilingual_scan/fra/config.jsonnet | 9 ++++++++- src/genbench/tasks/multilingual_scan/hin/config.jsonnet | 6 ++++++ src/genbench/tasks/multilingual_scan/rus/config.jsonnet | 6 ++++++ src/genbench/tasks/multilingual_scan/tur/config.jsonnet | 6 ++++++ 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet index f546239..f1e98f2 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet @@ -27,6 +27,12 @@ task_type: 'free_form', + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + }, + evaluation_metrics: [ { hf_id: 'exact_match', diff --git a/src/genbench/tasks/multilingual_scan/config.jsonnet b/src/genbench/tasks/multilingual_scan/config.jsonnet index 286c271..881dd87 100644 --- a/src/genbench/tasks/multilingual_scan/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/config.jsonnet @@ -29,7 +29,7 @@ finetuning: { objective: 'maximum_likelihood', }, - } + }, subtasks_order: [ 'fra', diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet index 100fbef..7842d93 100644 --- a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet @@ -7,7 +7,8 @@ keywords: [ 'cross-lingual', 'compositional', - 'french' + 'french', + ], authors: [ @@ -26,6 +27,12 @@ task_type: 'free_form', + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + }, + evaluation_metrics: [ { hf_id: 'exact_match', diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet index ecb5862..741775c 100644 --- a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet @@ -25,6 +25,12 @@ task_type: 'free_form', + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + }, + evaluation_metrics: [ { hf_id: 'exact_match', diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet index c4d9d71..04f144f 100644 --- a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet @@ -24,6 +24,12 @@ task_type: 'free_form', + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + }, + evaluation_metrics: [ { hf_id: 'exact_match', diff --git a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet index 419464c..21ca9d4 100644 --- a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet @@ -26,6 +26,12 @@ task_type: 'free_form', + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + }, + evaluation_metrics: [ { hf_id: 'exact_match', From 65becdc4bcc641ac95759722989a04a7eaa194b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20T=2E=20Reymond?= Date: Mon, 4 Sep 2023 22:37:15 -0700 Subject: [PATCH 12/12] Fix subtasks + add missing splits --- scripts/mscan/create-mscan-jsonnets.py | 90 +++++++++++++++++++ scripts/mscan/create-mscan.sh | 38 ++++++++ .../tasks/multilingual_scan/__init__.py | 2 +- .../multilingual_scan/cmn/config.jsonnet | 43 --------- .../tasks/multilingual_scan/cmn/doc.md | 23 ----- .../tasks/multilingual_scan/cmn/sample.jsonl | 5 -- .../{cmn => cmn_add_prim_jump}/__init__.py | 0 .../cmn_add_prim_jump/config.jsonnet | 55 ++++++++++++ .../cmn_add_prim_jump/doc.md | 19 ++++ .../{fra => cmn_add_prim_jump}/task.py | 2 +- .../__init__.py | 0 .../cmn_add_prim_turn_left/config.jsonnet | 55 ++++++++++++ .../cmn_add_prim_turn_left/doc.md | 19 ++++ .../cmn_add_prim_turn_left/task.py | 5 ++ .../{hin => cmn_length}/__init__.py | 0 .../cmn_length/config.jsonnet | 55 ++++++++++++ .../{tur => cmn_length}/doc.md | 10 +-- .../{hin => cmn_length}/task.py | 2 +- .../{rus => cmn_mcd1}/__init__.py | 0 .../multilingual_scan/cmn_mcd1/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/cmn_mcd1/doc.md | 19 ++++ .../{rus => cmn_mcd1}/task.py | 2 +- .../{tur => cmn_mcd2}/__init__.py | 0 .../multilingual_scan/cmn_mcd2/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/cmn_mcd2/doc.md | 19 ++++ .../{cmn => cmn_mcd2}/task.py | 2 +- .../multilingual_scan/cmn_mcd3/__init__.py | 0 .../multilingual_scan/cmn_mcd3/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/cmn_mcd3/doc.md | 19 ++++ .../tasks/multilingual_scan/cmn_mcd3/task.py | 5 ++ .../multilingual_scan/cmn_simple/__init__.py | 0 .../cmn_simple/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/cmn_simple/doc.md | 19 ++++ .../multilingual_scan/cmn_simple/task.py | 5 ++ .../tasks/multilingual_scan/config.jsonnet | 59 +++++++----- .../eng_add_prim_jump/__init__.py | 0 .../eng_add_prim_jump/config.jsonnet | 55 ++++++++++++ .../eng_add_prim_jump/doc.md | 19 ++++ .../eng_add_prim_jump/task.py | 5 ++ .../eng_add_prim_turn_left/__init__.py | 0 .../eng_add_prim_turn_left/config.jsonnet | 55 ++++++++++++ .../eng_add_prim_turn_left/doc.md | 19 ++++ .../eng_add_prim_turn_left/task.py | 5 ++ .../multilingual_scan/eng_length/__init__.py | 0 .../eng_length/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/eng_length/doc.md | 19 ++++ .../multilingual_scan/eng_length/task.py | 5 ++ .../multilingual_scan/eng_mcd1/__init__.py | 0 .../multilingual_scan/eng_mcd1/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/eng_mcd1/doc.md | 19 ++++ .../tasks/multilingual_scan/eng_mcd1/task.py | 5 ++ .../multilingual_scan/eng_mcd2/__init__.py | 0 .../multilingual_scan/eng_mcd2/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/eng_mcd2/doc.md | 19 ++++ .../tasks/multilingual_scan/eng_mcd2/task.py | 5 ++ .../multilingual_scan/eng_mcd3/__init__.py | 0 .../multilingual_scan/eng_mcd3/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/eng_mcd3/doc.md | 19 ++++ .../tasks/multilingual_scan/eng_mcd3/task.py | 5 ++ .../multilingual_scan/eng_simple/__init__.py | 0 .../eng_simple/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/eng_simple/doc.md | 19 ++++ .../multilingual_scan/eng_simple/task.py | 5 ++ .../multilingual_scan/fra/config.jsonnet | 43 --------- .../tasks/multilingual_scan/fra/doc.md | 23 ----- .../tasks/multilingual_scan/fra/sample.jsonl | 10 --- .../fra_add_prim_jump/__init__.py | 0 .../fra_add_prim_jump/config.jsonnet | 55 ++++++++++++ .../fra_add_prim_jump/doc.md | 19 ++++ .../fra_add_prim_jump/task.py | 5 ++ .../fra_add_prim_turn_left/__init__.py | 0 .../fra_add_prim_turn_left/config.jsonnet | 55 ++++++++++++ .../fra_add_prim_turn_left/doc.md | 19 ++++ .../fra_add_prim_turn_left/task.py | 5 ++ .../multilingual_scan/fra_length/__init__.py | 0 .../fra_length/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/fra_length/doc.md | 19 ++++ .../multilingual_scan/fra_length/task.py | 5 ++ .../multilingual_scan/fra_mcd1/__init__.py | 0 .../multilingual_scan/fra_mcd1/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/fra_mcd1/doc.md | 19 ++++ .../tasks/multilingual_scan/fra_mcd1/task.py | 5 ++ .../multilingual_scan/fra_mcd2/__init__.py | 0 .../multilingual_scan/fra_mcd2/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/fra_mcd2/doc.md | 19 ++++ .../tasks/multilingual_scan/fra_mcd2/task.py | 5 ++ .../multilingual_scan/fra_mcd3/__init__.py | 0 .../multilingual_scan/fra_mcd3/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/fra_mcd3/doc.md | 19 ++++ .../tasks/multilingual_scan/fra_mcd3/task.py | 5 ++ .../multilingual_scan/fra_simple/__init__.py | 0 .../fra_simple/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/fra_simple/doc.md | 19 ++++ .../multilingual_scan/fra_simple/task.py | 5 ++ .../multilingual_scan/hin/config.jsonnet | 41 --------- .../tasks/multilingual_scan/hin/doc.md | 23 ----- .../tasks/multilingual_scan/hin/sample.jsonl | 10 --- .../hin_add_prim_jump/__init__.py | 0 .../hin_add_prim_jump/config.jsonnet | 55 ++++++++++++ .../hin_add_prim_jump/doc.md | 19 ++++ .../hin_add_prim_jump/task.py | 5 ++ .../hin_add_prim_turn_left/__init__.py | 0 .../hin_add_prim_turn_left/config.jsonnet | 55 ++++++++++++ .../hin_add_prim_turn_left/doc.md | 19 ++++ .../hin_add_prim_turn_left/task.py | 5 ++ .../multilingual_scan/hin_length/__init__.py | 0 .../hin_length/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/hin_length/doc.md | 19 ++++ .../multilingual_scan/hin_length/task.py | 5 ++ .../multilingual_scan/hin_mcd1/__init__.py | 0 .../multilingual_scan/hin_mcd1/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/hin_mcd1/doc.md | 19 ++++ .../tasks/multilingual_scan/hin_mcd1/task.py | 5 ++ .../multilingual_scan/hin_mcd2/__init__.py | 0 .../multilingual_scan/hin_mcd2/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/hin_mcd2/doc.md | 19 ++++ .../tasks/multilingual_scan/hin_mcd2/task.py | 5 ++ .../multilingual_scan/hin_mcd3/__init__.py | 0 .../multilingual_scan/hin_mcd3/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/hin_mcd3/doc.md | 19 ++++ .../tasks/multilingual_scan/hin_mcd3/task.py | 5 ++ .../multilingual_scan/hin_simple/__init__.py | 0 .../hin_simple/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/hin_simple/doc.md | 19 ++++ .../multilingual_scan/hin_simple/task.py | 5 ++ .../multilingual_scan/rus/config.jsonnet | 40 --------- .../tasks/multilingual_scan/rus/doc.md | 22 ----- .../tasks/multilingual_scan/rus/sample.jsonl | 10 --- .../rus_add_prim_jump/__init__.py | 0 .../rus_add_prim_jump/config.jsonnet | 55 ++++++++++++ .../rus_add_prim_jump/doc.md | 19 ++++ .../rus_add_prim_jump/task.py | 5 ++ .../rus_add_prim_turn_left/__init__.py | 0 .../rus_add_prim_turn_left/config.jsonnet | 55 ++++++++++++ .../rus_add_prim_turn_left/doc.md | 19 ++++ .../rus_add_prim_turn_left/task.py | 5 ++ .../multilingual_scan/rus_length/__init__.py | 0 .../rus_length/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/rus_length/doc.md | 19 ++++ .../multilingual_scan/rus_length/task.py | 5 ++ .../multilingual_scan/rus_mcd1/__init__.py | 0 .../multilingual_scan/rus_mcd1/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/rus_mcd1/doc.md | 19 ++++ .../tasks/multilingual_scan/rus_mcd1/task.py | 5 ++ .../multilingual_scan/rus_mcd2/__init__.py | 0 .../multilingual_scan/rus_mcd2/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/rus_mcd2/doc.md | 19 ++++ .../tasks/multilingual_scan/rus_mcd2/task.py | 5 ++ .../multilingual_scan/rus_mcd3/__init__.py | 0 .../multilingual_scan/rus_mcd3/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/rus_mcd3/doc.md | 19 ++++ .../tasks/multilingual_scan/rus_mcd3/task.py | 5 ++ .../multilingual_scan/rus_simple/__init__.py | 0 .../rus_simple/config.jsonnet | 55 ++++++++++++ .../tasks/multilingual_scan/rus_simple/doc.md | 19 ++++ .../multilingual_scan/rus_simple/task.py | 5 ++ .../multilingual_scan/tur/config.jsonnet | 42 --------- .../tasks/multilingual_scan/tur/task.py | 5 -- 158 files changed, 2900 insertions(+), 373 deletions(-) create mode 100644 scripts/mscan/create-mscan-jsonnets.py create mode 100755 scripts/mscan/create-mscan.sh delete mode 100644 src/genbench/tasks/multilingual_scan/cmn/config.jsonnet delete mode 100644 src/genbench/tasks/multilingual_scan/cmn/doc.md delete mode 100644 src/genbench/tasks/multilingual_scan/cmn/sample.jsonl rename src/genbench/tasks/multilingual_scan/{cmn => cmn_add_prim_jump}/__init__.py (100%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/doc.md rename src/genbench/tasks/multilingual_scan/{fra => cmn_add_prim_jump}/task.py (52%) rename src/genbench/tasks/multilingual_scan/{fra => cmn_add_prim_turn_left}/__init__.py (100%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/task.py rename src/genbench/tasks/multilingual_scan/{hin => cmn_length}/__init__.py (100%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_length/config.jsonnet rename src/genbench/tasks/multilingual_scan/{tur => cmn_length}/doc.md (58%) rename src/genbench/tasks/multilingual_scan/{hin => cmn_length}/task.py (52%) rename src/genbench/tasks/multilingual_scan/{rus => cmn_mcd1}/__init__.py (100%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd1/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd1/doc.md rename src/genbench/tasks/multilingual_scan/{rus => cmn_mcd1}/task.py (52%) rename src/genbench/tasks/multilingual_scan/{tur => cmn_mcd2}/__init__.py (100%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd2/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd2/doc.md rename src/genbench/tasks/multilingual_scan/{cmn => cmn_mcd2}/task.py (52%) create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd3/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd3/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd3/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/cmn_mcd3/task.py create mode 100644 src/genbench/tasks/multilingual_scan/cmn_simple/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/cmn_simple/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/cmn_simple/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/cmn_simple/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_jump/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_jump/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_jump/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_jump/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_length/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_length/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_length/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_length/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd1/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd1/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd1/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd1/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd2/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd2/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd2/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd2/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd3/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd3/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd3/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_mcd3/task.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_simple/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/eng_simple/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/eng_simple/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/eng_simple/task.py delete mode 100644 src/genbench/tasks/multilingual_scan/fra/config.jsonnet delete mode 100644 src/genbench/tasks/multilingual_scan/fra/doc.md delete mode 100644 src/genbench/tasks/multilingual_scan/fra/sample.jsonl create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_jump/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_jump/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_jump/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_jump/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_length/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_length/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_length/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_length/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd1/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd1/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd1/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd1/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd2/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd2/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd2/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd2/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd3/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd3/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd3/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_mcd3/task.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_simple/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/fra_simple/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/fra_simple/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/fra_simple/task.py delete mode 100644 src/genbench/tasks/multilingual_scan/hin/config.jsonnet delete mode 100644 src/genbench/tasks/multilingual_scan/hin/doc.md delete mode 100644 src/genbench/tasks/multilingual_scan/hin/sample.jsonl create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_jump/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_jump/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_jump/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_jump/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_length/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_length/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_length/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_length/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd1/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd1/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd1/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd1/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd2/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd2/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd2/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd2/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd3/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd3/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd3/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_mcd3/task.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_simple/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/hin_simple/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/hin_simple/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/hin_simple/task.py delete mode 100644 src/genbench/tasks/multilingual_scan/rus/config.jsonnet delete mode 100644 src/genbench/tasks/multilingual_scan/rus/doc.md delete mode 100644 src/genbench/tasks/multilingual_scan/rus/sample.jsonl create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_jump/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_jump/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_jump/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_jump/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_length/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_length/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_length/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_length/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd1/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd1/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd1/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd1/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd2/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd2/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd2/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd2/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd3/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd3/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd3/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_mcd3/task.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_simple/__init__.py create mode 100644 src/genbench/tasks/multilingual_scan/rus_simple/config.jsonnet create mode 100644 src/genbench/tasks/multilingual_scan/rus_simple/doc.md create mode 100644 src/genbench/tasks/multilingual_scan/rus_simple/task.py delete mode 100644 src/genbench/tasks/multilingual_scan/tur/config.jsonnet delete mode 100644 src/genbench/tasks/multilingual_scan/tur/task.py diff --git a/scripts/mscan/create-mscan-jsonnets.py b/scripts/mscan/create-mscan-jsonnets.py new file mode 100644 index 0000000..fcfe433 --- /dev/null +++ b/scripts/mscan/create-mscan-jsonnets.py @@ -0,0 +1,90 @@ + +def get_jsonnet(language, split, has_dev): + return f"""{{ + name: 'mSCAN (language: {language}, split: {split})', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN ({language}) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: {{ + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/{language}/{split}/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/{language}/{split}/test.jsonl', + {f"validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/{language}/{split}/dev.jsonl'," if has_dev else ""} + }}, + + has_validation_set: {"true" if has_dev else "false"}, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: {{ + finetuning: {{ + objective: 'maximum_likelihood', + }}, + + prompt_based_testing: {{ + prompt_builder: {{ + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\\n\\n', + stop_string: '\\n\\n', + }} + }}, + + }}, + + evaluation_metrics: [ + {{ + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + }} + ] +}}""" + + +languages = ["cmn", "eng", "fra", "hin", "rus"] +splits = { + "add_prim_jump": { + "has_dev": False + }, + "add_prim_turn_left": { + "has_dev": False + }, + "length": { + "has_dev": False + }, + "mcd1": { + "has_dev": True + }, + "mcd2": { + "has_dev": True + }, + "mcd3": { + "has_dev": True + }, + "simple": { + "has_dev": False + }, +} + +for language in languages: + for split_name, split_args in splits.items(): + jsonnet = get_jsonnet(language, split_name, split_args["has_dev"]) + with open(f"src/genbench/tasks/mscan/{language}_{split_name}/config.jsonnet", "w") as f: + f.write(jsonnet) + diff --git a/scripts/mscan/create-mscan.sh b/scripts/mscan/create-mscan.sh new file mode 100755 index 0000000..b98a718 --- /dev/null +++ b/scripts/mscan/create-mscan.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +genbench-cli create-task --id "mscan" --name "mSCAN" \ + -s cmn_add_prim_jump \ + -s cmn_add_prim_turn_left \ + -s cmn_length \ + -s cmn_mcd1 \ + -s cmn_mcd2 \ + -s cmn_mcd3 \ + -s cmn_simple \ + -s eng_add_prim_jump \ + -s eng_add_prim_turn_left \ + -s eng_length \ + -s eng_mcd1 \ + -s eng_mcd2 \ + -s eng_mcd3 \ + -s eng_simple \ + -s fra_add_prim_jump \ + -s fra_add_prim_turn_left \ + -s fra_length \ + -s fra_mcd1 \ + -s fra_mcd2 \ + -s fra_mcd3 \ + -s fra_simple \ + -s hin_add_prim_jump \ + -s hin_add_prim_turn_left \ + -s hin_length \ + -s hin_mcd1 \ + -s hin_mcd2 \ + -s hin_mcd3 \ + -s hin_simple \ + -s rus_add_prim_jump \ + -s rus_add_prim_turn_left \ + -s rus_length \ + -s rus_mcd1 \ + -s rus_mcd2 \ + -s rus_mcd3 \ + -s rus_simple diff --git a/src/genbench/tasks/multilingual_scan/__init__.py b/src/genbench/tasks/multilingual_scan/__init__.py index e3f7abc..98caede 100644 --- a/src/genbench/tasks/multilingual_scan/__init__.py +++ b/src/genbench/tasks/multilingual_scan/__init__.py @@ -1,5 +1,5 @@ from genbench import TaskDict -class MultilingualScan(TaskDict): +class Mscan(TaskDict): pass diff --git a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet deleted file mode 100644 index f1e98f2..0000000 --- a/src/genbench/tasks/multilingual_scan/cmn/config.jsonnet +++ /dev/null @@ -1,43 +0,0 @@ -{ - name: 'Multilingual SCAN (cmn)', - - // @TODO: Add a description of the task - description: 'Multilingual SCAN (cmn) aims to measure compositional and cross-lingual generalization in multilingual LLMs', - - // @TODO: Add a list of keywords that describe the task - keywords: [ - 'cross-lingual', - 'compositional', - 'Mandarin Chinese' - ], - - authors: [ - 'Amélie Reymond', - 'Shane Steinert-Threlkeld', - - ], - - data_source: { - type: 'manual', - test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl', - }, - - has_validation_set: false, - has_train_set: false, - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - - evaluation_metrics: [ - { - hf_id: 'exact_match', - git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", - best_score: 1.0, - } - ] -} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn/doc.md b/src/genbench/tasks/multilingual_scan/cmn/doc.md deleted file mode 100644 index afdd16b..0000000 --- a/src/genbench/tasks/multilingual_scan/cmn/doc.md +++ /dev/null @@ -1,23 +0,0 @@ -# Multilingual SCAN (cmn) - -## Abstract - -This specific task consists of adapted SCAN benchmark into Mandarin Chinese (cmn). - -## Examples - -```json -{"input": "xiang you fan fangxiang pao hou, xiang you fan fangxiang xingzou san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -``` - -## Usage -*Describe how to load your task and what is required for evaluation, if anything.* - -## Data Source -*Describe the data source for this Multilingual SCAN (cmn).* - -## Limitations and Bias -*Note any known limitations or biases that the Multilingual SCAN (cmn) has, with links and references if possible.* - -## GenBench Eval card -*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl b/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl deleted file mode 100644 index dbb8a27..0000000 --- a/src/genbench/tasks/multilingual_scan/cmn/sample.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"input": "xiang you fan fangxiang pao hou, xiang you fan fangxiang xingzou san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -{"input": "xiang you fan fangxiang pao liang ci hou, xiang you zhuan san ci", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} -{"input": "xiang you zou liang quan hou, xiang zuo fan fangxiang kan liang ci", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "xiang zuo fan fangxiang pao, ranhou xiang zuo kan liang quan", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} -{"input": "xiang you kan san ci, ranhou xiang zuo fan fangxiang kan liang ci", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} diff --git a/src/genbench/tasks/multilingual_scan/cmn/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/__init__.py similarity index 100% rename from src/genbench/tasks/multilingual_scan/cmn/__init__.py rename to src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/__init__.py diff --git a/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/config.jsonnet new file mode 100644 index 0000000..60f1bb3 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: add_prim_jump)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/add_prim_jump/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/add_prim_jump/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/doc.md b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/doc.md new file mode 100644 index 0000000..e0e8dc5 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_add_prim_jump) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_add_prim_jump).* + +## Examples +*Give some examples of the mSCAN (cmn_add_prim_jump).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_add_prim_jump).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_add_prim_jump) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra/task.py b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/task.py similarity index 52% rename from src/genbench/tasks/multilingual_scan/fra/task.py rename to src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/task.py index 7ba7d80..6e6e74c 100644 --- a/src/genbench/tasks/multilingual_scan/fra/task.py +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_jump/task.py @@ -1,5 +1,5 @@ from genbench import Task -class MultilingualScanFra(Task): +class MscanCmnAddPrimJump(Task): pass diff --git a/src/genbench/tasks/multilingual_scan/fra/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/__init__.py similarity index 100% rename from src/genbench/tasks/multilingual_scan/fra/__init__.py rename to src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/__init__.py diff --git a/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/config.jsonnet new file mode 100644 index 0000000..ebd68c8 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: add_prim_turn_left)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/add_prim_turn_left/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/add_prim_turn_left/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/doc.md b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/doc.md new file mode 100644 index 0000000..fa4e687 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_add_prim_turn_left) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_add_prim_turn_left).* + +## Examples +*Give some examples of the mSCAN (cmn_add_prim_turn_left).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_add_prim_turn_left).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_add_prim_turn_left) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/task.py b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/task.py new file mode 100644 index 0000000..877634c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_add_prim_turn_left/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanCmnAddPrimTurnLeft(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_length/__init__.py similarity index 100% rename from src/genbench/tasks/multilingual_scan/hin/__init__.py rename to src/genbench/tasks/multilingual_scan/cmn_length/__init__.py diff --git a/src/genbench/tasks/multilingual_scan/cmn_length/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_length/config.jsonnet new file mode 100644 index 0000000..64f1e40 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_length/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: length)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/length/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/length/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/tur/doc.md b/src/genbench/tasks/multilingual_scan/cmn_length/doc.md similarity index 58% rename from src/genbench/tasks/multilingual_scan/tur/doc.md rename to src/genbench/tasks/multilingual_scan/cmn_length/doc.md index 3c00a3a..7f55b74 100644 --- a/src/genbench/tasks/multilingual_scan/tur/doc.md +++ b/src/genbench/tasks/multilingual_scan/cmn_length/doc.md @@ -1,19 +1,19 @@ -# Multilingual SCAN (tur) +# mSCAN (cmn_length) ## Abstract -*Copy the abstract of your accompanying paper for this task here Multilingual SCAN (tur).* +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_length).* ## Examples -*Give some examples of the Multilingual SCAN (tur).* +*Give some examples of the mSCAN (cmn_length).* ## Usage *Describe how to load your task and what is required for evaluation, if anything.* ## Data Source -*Describe the data source for this Multilingual SCAN (tur).* +*Describe the data source for this mSCAN (cmn_length).* ## Limitations and Bias -*Note any known limitations or biases that the Multilingual SCAN (tur) has, with links and references if possible.* +*Note any known limitations or biases that the mSCAN (cmn_length) has, with links and references if possible.* ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin/task.py b/src/genbench/tasks/multilingual_scan/cmn_length/task.py similarity index 52% rename from src/genbench/tasks/multilingual_scan/hin/task.py rename to src/genbench/tasks/multilingual_scan/cmn_length/task.py index 5029129..c35dd82 100644 --- a/src/genbench/tasks/multilingual_scan/hin/task.py +++ b/src/genbench/tasks/multilingual_scan/cmn_length/task.py @@ -1,5 +1,5 @@ from genbench import Task -class MultilingualScanHin(Task): +class MscanCmnLength(Task): pass diff --git a/src/genbench/tasks/multilingual_scan/rus/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_mcd1/__init__.py similarity index 100% rename from src/genbench/tasks/multilingual_scan/rus/__init__.py rename to src/genbench/tasks/multilingual_scan/cmn_mcd1/__init__.py diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd1/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_mcd1/config.jsonnet new file mode 100644 index 0000000..ea36061 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd1/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: mcd1)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd1/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd1/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd1/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd1/doc.md b/src/genbench/tasks/multilingual_scan/cmn_mcd1/doc.md new file mode 100644 index 0000000..938963f --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd1/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_mcd1) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_mcd1).* + +## Examples +*Give some examples of the mSCAN (cmn_mcd1).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_mcd1).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_mcd1) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus/task.py b/src/genbench/tasks/multilingual_scan/cmn_mcd1/task.py similarity index 52% rename from src/genbench/tasks/multilingual_scan/rus/task.py rename to src/genbench/tasks/multilingual_scan/cmn_mcd1/task.py index b9c0912..dbce13d 100644 --- a/src/genbench/tasks/multilingual_scan/rus/task.py +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd1/task.py @@ -1,5 +1,5 @@ from genbench import Task -class MultilingualScanRus(Task): +class MscanCmnMcd1(Task): pass diff --git a/src/genbench/tasks/multilingual_scan/tur/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_mcd2/__init__.py similarity index 100% rename from src/genbench/tasks/multilingual_scan/tur/__init__.py rename to src/genbench/tasks/multilingual_scan/cmn_mcd2/__init__.py diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd2/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_mcd2/config.jsonnet new file mode 100644 index 0000000..ba69cbe --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd2/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: mcd2)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd2/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd2/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd2/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd2/doc.md b/src/genbench/tasks/multilingual_scan/cmn_mcd2/doc.md new file mode 100644 index 0000000..ba34eed --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd2/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_mcd2) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_mcd2).* + +## Examples +*Give some examples of the mSCAN (cmn_mcd2).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_mcd2).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_mcd2) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn/task.py b/src/genbench/tasks/multilingual_scan/cmn_mcd2/task.py similarity index 52% rename from src/genbench/tasks/multilingual_scan/cmn/task.py rename to src/genbench/tasks/multilingual_scan/cmn_mcd2/task.py index 9a48092..e1208eb 100644 --- a/src/genbench/tasks/multilingual_scan/cmn/task.py +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd2/task.py @@ -1,5 +1,5 @@ from genbench import Task -class MultilingualScanCmn(Task): +class MscanCmnMcd2(Task): pass diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd3/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_mcd3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd3/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_mcd3/config.jsonnet new file mode 100644 index 0000000..3997db4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd3/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: mcd3)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd3/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd3/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/mcd3/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd3/doc.md b/src/genbench/tasks/multilingual_scan/cmn_mcd3/doc.md new file mode 100644 index 0000000..4563e0e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd3/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_mcd3) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_mcd3).* + +## Examples +*Give some examples of the mSCAN (cmn_mcd3).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_mcd3).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_mcd3) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn_mcd3/task.py b/src/genbench/tasks/multilingual_scan/cmn_mcd3/task.py new file mode 100644 index 0000000..01223b9 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_mcd3/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanCmnMcd3(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/cmn_simple/__init__.py b/src/genbench/tasks/multilingual_scan/cmn_simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/cmn_simple/config.jsonnet b/src/genbench/tasks/multilingual_scan/cmn_simple/config.jsonnet new file mode 100644 index 0000000..f475651 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_simple/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: cmn, split: simple)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (cmn) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/simple/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/cmn/simple/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/cmn_simple/doc.md b/src/genbench/tasks/multilingual_scan/cmn_simple/doc.md new file mode 100644 index 0000000..addcb76 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_simple/doc.md @@ -0,0 +1,19 @@ +# mSCAN (cmn_simple) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (cmn_simple).* + +## Examples +*Give some examples of the mSCAN (cmn_simple).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (cmn_simple).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (cmn_simple) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/cmn_simple/task.py b/src/genbench/tasks/multilingual_scan/cmn_simple/task.py new file mode 100644 index 0000000..5970d1d --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/cmn_simple/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanCmnSimple(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/config.jsonnet b/src/genbench/tasks/multilingual_scan/config.jsonnet index 881dd87..8563022 100644 --- a/src/genbench/tasks/multilingual_scan/config.jsonnet +++ b/src/genbench/tasks/multilingual_scan/config.jsonnet @@ -1,5 +1,5 @@ { - name: 'Multilingual SCAN', + name: 'mSCAN', description: 'Multilingual SCAN aims to measure compositional and cross-lingual generalization in multilingual LLMs.', @@ -14,29 +14,42 @@ 'Shane Steinert-Threlkeld', ], - data_source: { - type: 'manual', - test: 'https://somewebsite.com/path/to/test.jsonl', - }, - - # TODO: provide finetuning set - has_train_set: true, - - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - subtasks_order: [ - 'fra', - 'rus', - 'cmn', - 'tur', - 'hin', + 'cmn_add_prim_jump', + 'cmn_add_prim_turn_left', + 'cmn_length', + 'cmn_mcd1', + 'cmn_mcd2', + 'cmn_mcd3', + 'cmn_simple', + 'eng_add_prim_jump', + 'eng_add_prim_turn_left', + 'eng_length', + 'eng_mcd1', + 'eng_mcd2', + 'eng_mcd3', + 'eng_simple', + 'fra_add_prim_jump', + 'fra_add_prim_turn_left', + 'fra_length', + 'fra_mcd1', + 'fra_mcd2', + 'fra_mcd3', + 'fra_simple', + 'hin_add_prim_jump', + 'hin_add_prim_turn_left', + 'hin_length', + 'hin_mcd1', + 'hin_mcd2', + 'hin_mcd3', + 'hin_simple', + 'rus_add_prim_jump', + 'rus_add_prim_turn_left', + 'rus_length', + 'rus_mcd1', + 'rus_mcd2', + 'rus_mcd3', + 'rus_simple', ], } \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/__init__.py b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/config.jsonnet new file mode 100644 index 0000000..d7254af --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: add_prim_jump)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/add_prim_jump/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/add_prim_jump/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/doc.md b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/doc.md new file mode 100644 index 0000000..fa998b1 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_add_prim_jump) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_add_prim_jump).* + +## Examples +*Give some examples of the mSCAN (eng_add_prim_jump).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_add_prim_jump).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_add_prim_jump) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/task.py b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/task.py new file mode 100644 index 0000000..43b7245 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_jump/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngAddPrimJump(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/__init__.py b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/config.jsonnet new file mode 100644 index 0000000..83eddca --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: add_prim_turn_left)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/add_prim_turn_left/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/add_prim_turn_left/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/doc.md b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/doc.md new file mode 100644 index 0000000..62ff5da --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_add_prim_turn_left) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_add_prim_turn_left).* + +## Examples +*Give some examples of the mSCAN (eng_add_prim_turn_left).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_add_prim_turn_left).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_add_prim_turn_left) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/task.py b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/task.py new file mode 100644 index 0000000..a3e22f4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_add_prim_turn_left/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngAddPrimTurnLeft(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_length/__init__.py b/src/genbench/tasks/multilingual_scan/eng_length/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_length/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_length/config.jsonnet new file mode 100644 index 0000000..304c30e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_length/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: length)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/length/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/length/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_length/doc.md b/src/genbench/tasks/multilingual_scan/eng_length/doc.md new file mode 100644 index 0000000..e7216e7 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_length/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_length) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_length).* + +## Examples +*Give some examples of the mSCAN (eng_length).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_length).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_length) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_length/task.py b/src/genbench/tasks/multilingual_scan/eng_length/task.py new file mode 100644 index 0000000..b8d5b2a --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_length/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngLength(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd1/__init__.py b/src/genbench/tasks/multilingual_scan/eng_mcd1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd1/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_mcd1/config.jsonnet new file mode 100644 index 0000000..70ffd6e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd1/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: mcd1)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd1/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd1/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd1/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd1/doc.md b/src/genbench/tasks/multilingual_scan/eng_mcd1/doc.md new file mode 100644 index 0000000..d5858a9 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd1/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_mcd1) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_mcd1).* + +## Examples +*Give some examples of the mSCAN (eng_mcd1).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_mcd1).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_mcd1) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd1/task.py b/src/genbench/tasks/multilingual_scan/eng_mcd1/task.py new file mode 100644 index 0000000..d7f0b82 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd1/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngMcd1(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd2/__init__.py b/src/genbench/tasks/multilingual_scan/eng_mcd2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd2/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_mcd2/config.jsonnet new file mode 100644 index 0000000..967966f --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd2/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: mcd2)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd2/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd2/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd2/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd2/doc.md b/src/genbench/tasks/multilingual_scan/eng_mcd2/doc.md new file mode 100644 index 0000000..aa34290 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd2/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_mcd2) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_mcd2).* + +## Examples +*Give some examples of the mSCAN (eng_mcd2).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_mcd2).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_mcd2) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd2/task.py b/src/genbench/tasks/multilingual_scan/eng_mcd2/task.py new file mode 100644 index 0000000..bf38658 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd2/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngMcd2(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd3/__init__.py b/src/genbench/tasks/multilingual_scan/eng_mcd3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd3/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_mcd3/config.jsonnet new file mode 100644 index 0000000..7469dfe --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd3/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: mcd3)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd3/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd3/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/mcd3/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd3/doc.md b/src/genbench/tasks/multilingual_scan/eng_mcd3/doc.md new file mode 100644 index 0000000..84c2207 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd3/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_mcd3) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_mcd3).* + +## Examples +*Give some examples of the mSCAN (eng_mcd3).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_mcd3).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_mcd3) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_mcd3/task.py b/src/genbench/tasks/multilingual_scan/eng_mcd3/task.py new file mode 100644 index 0000000..7204d95 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_mcd3/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngMcd3(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/eng_simple/__init__.py b/src/genbench/tasks/multilingual_scan/eng_simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/eng_simple/config.jsonnet b/src/genbench/tasks/multilingual_scan/eng_simple/config.jsonnet new file mode 100644 index 0000000..261e6a4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_simple/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: eng, split: simple)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (eng) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/simple/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/eng/simple/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/eng_simple/doc.md b/src/genbench/tasks/multilingual_scan/eng_simple/doc.md new file mode 100644 index 0000000..3e26b46 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_simple/doc.md @@ -0,0 +1,19 @@ +# mSCAN (eng_simple) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (eng_simple).* + +## Examples +*Give some examples of the mSCAN (eng_simple).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (eng_simple).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (eng_simple) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/eng_simple/task.py b/src/genbench/tasks/multilingual_scan/eng_simple/task.py new file mode 100644 index 0000000..68d3507 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/eng_simple/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanEngSimple(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra/config.jsonnet deleted file mode 100644 index 7842d93..0000000 --- a/src/genbench/tasks/multilingual_scan/fra/config.jsonnet +++ /dev/null @@ -1,43 +0,0 @@ -{ - name: 'Multilingual SCAN (fra)', - - - description: 'Multilingual SCAN (fra) aims to measure compositional and cross-lingual generalization in multilingual LLMs', - - keywords: [ - 'cross-lingual', - 'compositional', - 'french', - - ], - - authors: [ - 'Amélie Reymond', - 'Shane Steinert-Threlkeld', - - ], - - data_source: { - type: 'manual', - test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/fra/sample.jsonl', - }, - - has_validation_set: false, - has_train_set: false, - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - - evaluation_metrics: [ - { - hf_id: 'exact_match', - git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", - best_score: 1.0, - } - ] -} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra/doc.md b/src/genbench/tasks/multilingual_scan/fra/doc.md deleted file mode 100644 index a2bc615..0000000 --- a/src/genbench/tasks/multilingual_scan/fra/doc.md +++ /dev/null @@ -1,23 +0,0 @@ -# Multilingual SCAN (fra) - -## Abstract - -This specific task consists of adapted SCAN benchmark into French (fra). - -## Examples - -```json -{"input": "marcher en face à droite trois fois ensuite courir en face à droite", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -``` - -## Usage -*Describe how to load your task and what is required for evaluation, if anything.* - -## Data Source -*Describe the data source for this Multilingual SCAN (fra).* - -## Limitations and Bias -*Note any known limitations or biases that the Multilingual SCAN (fra) has, with links and references if possible.* - -## GenBench Eval card -*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra/sample.jsonl b/src/genbench/tasks/multilingual_scan/fra/sample.jsonl deleted file mode 100644 index bab8227..0000000 --- a/src/genbench/tasks/multilingual_scan/fra/sample.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -{"input": "marcher en face à droite trois fois ensuite courir en face à droite", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -{"input": "tourner autour à droite trois fois ensuite courir en face à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} -{"input": "regarder en face à gauche deux fois ensuite marcher autour à droite deux fois", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "courir en face à gauche et regarder autour à gauche deux fois", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} -{"input": "regarder à droite trois fois et regarder en face à gauche deux fois", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "marcher en face à gauche deux fois et courir", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} -{"input": "regarder trois fois ensuite sauter en face à droite trois fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} -{"input": "tourner en face à gauche deux fois ensuite regarder en face à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} -{"input": "sauter à gauche deux fois ensuite tourner autour à droite deux fois", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} -{"input": "regarder autour à droite et courir en face à gauche trois fois", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/__init__.py b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/config.jsonnet new file mode 100644 index 0000000..c941e07 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: add_prim_jump)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/add_prim_jump/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/add_prim_jump/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/doc.md b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/doc.md new file mode 100644 index 0000000..ee0e7f0 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_add_prim_jump) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_add_prim_jump).* + +## Examples +*Give some examples of the mSCAN (fra_add_prim_jump).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_add_prim_jump).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_add_prim_jump) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/task.py b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/task.py new file mode 100644 index 0000000..758a4cd --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_jump/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraAddPrimJump(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/__init__.py b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/config.jsonnet new file mode 100644 index 0000000..be80b72 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: add_prim_turn_left)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/add_prim_turn_left/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/add_prim_turn_left/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/doc.md b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/doc.md new file mode 100644 index 0000000..12d0753 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_add_prim_turn_left) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_add_prim_turn_left).* + +## Examples +*Give some examples of the mSCAN (fra_add_prim_turn_left).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_add_prim_turn_left).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_add_prim_turn_left) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/task.py b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/task.py new file mode 100644 index 0000000..038c5a6 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_add_prim_turn_left/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraAddPrimTurnLeft(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_length/__init__.py b/src/genbench/tasks/multilingual_scan/fra_length/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_length/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_length/config.jsonnet new file mode 100644 index 0000000..fdc0563 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_length/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: length)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/length/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/length/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_length/doc.md b/src/genbench/tasks/multilingual_scan/fra_length/doc.md new file mode 100644 index 0000000..bf5cb16 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_length/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_length) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_length).* + +## Examples +*Give some examples of the mSCAN (fra_length).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_length).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_length) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_length/task.py b/src/genbench/tasks/multilingual_scan/fra_length/task.py new file mode 100644 index 0000000..448d083 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_length/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraLength(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd1/__init__.py b/src/genbench/tasks/multilingual_scan/fra_mcd1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd1/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_mcd1/config.jsonnet new file mode 100644 index 0000000..0b92ef5 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd1/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: mcd1)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd1/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd1/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd1/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd1/doc.md b/src/genbench/tasks/multilingual_scan/fra_mcd1/doc.md new file mode 100644 index 0000000..4a6dadb --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd1/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_mcd1) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_mcd1).* + +## Examples +*Give some examples of the mSCAN (fra_mcd1).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_mcd1).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_mcd1) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd1/task.py b/src/genbench/tasks/multilingual_scan/fra_mcd1/task.py new file mode 100644 index 0000000..afad2ed --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd1/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraMcd1(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd2/__init__.py b/src/genbench/tasks/multilingual_scan/fra_mcd2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd2/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_mcd2/config.jsonnet new file mode 100644 index 0000000..27bff0e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd2/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: mcd2)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd2/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd2/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd2/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd2/doc.md b/src/genbench/tasks/multilingual_scan/fra_mcd2/doc.md new file mode 100644 index 0000000..fadde5e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd2/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_mcd2) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_mcd2).* + +## Examples +*Give some examples of the mSCAN (fra_mcd2).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_mcd2).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_mcd2) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd2/task.py b/src/genbench/tasks/multilingual_scan/fra_mcd2/task.py new file mode 100644 index 0000000..6e0301e --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd2/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraMcd2(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd3/__init__.py b/src/genbench/tasks/multilingual_scan/fra_mcd3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd3/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_mcd3/config.jsonnet new file mode 100644 index 0000000..6084da4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd3/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: mcd3)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd3/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd3/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/mcd3/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd3/doc.md b/src/genbench/tasks/multilingual_scan/fra_mcd3/doc.md new file mode 100644 index 0000000..7e32d52 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd3/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_mcd3) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_mcd3).* + +## Examples +*Give some examples of the mSCAN (fra_mcd3).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_mcd3).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_mcd3) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_mcd3/task.py b/src/genbench/tasks/multilingual_scan/fra_mcd3/task.py new file mode 100644 index 0000000..0e548a4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_mcd3/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraMcd3(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/fra_simple/__init__.py b/src/genbench/tasks/multilingual_scan/fra_simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/fra_simple/config.jsonnet b/src/genbench/tasks/multilingual_scan/fra_simple/config.jsonnet new file mode 100644 index 0000000..a9baeeb --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_simple/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: fra, split: simple)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (fra) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/simple/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/fra/simple/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/fra_simple/doc.md b/src/genbench/tasks/multilingual_scan/fra_simple/doc.md new file mode 100644 index 0000000..2a78947 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_simple/doc.md @@ -0,0 +1,19 @@ +# mSCAN (fra_simple) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (fra_simple).* + +## Examples +*Give some examples of the mSCAN (fra_simple).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (fra_simple).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (fra_simple) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/fra_simple/task.py b/src/genbench/tasks/multilingual_scan/fra_simple/task.py new file mode 100644 index 0000000..af865b5 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/fra_simple/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanFraSimple(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin/config.jsonnet deleted file mode 100644 index 741775c..0000000 --- a/src/genbench/tasks/multilingual_scan/hin/config.jsonnet +++ /dev/null @@ -1,41 +0,0 @@ -{ - name: 'Multilingual SCAN (hin)', - - description: 'Multilingual SCAN (hin) aims to measure compositional and cross-lingual generalization in multilingual LLMs', - - keywords: [ - 'cross-lingual', - 'compositional', - 'hindi', - ], - - authors: [ - 'Amélie Reymond', - 'Shane Steinert-Threlkeld', - - ], - - data_source: { - type: 'manual', - test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/hin/sample.jsonl', - }, - - has_validation_set: false, - has_train_set: false, - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - - evaluation_metrics: [ - { - hf_id: 'exact_match', - git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", - best_score: 1.0, - } - ] -} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin/doc.md b/src/genbench/tasks/multilingual_scan/hin/doc.md deleted file mode 100644 index 5c7756f..0000000 --- a/src/genbench/tasks/multilingual_scan/hin/doc.md +++ /dev/null @@ -1,23 +0,0 @@ -# Multilingual SCAN (hin) - -## Abstract - -This specific task consists of adapted SCAN benchmark into Hindi (hin). - -## Examples - -```json -{"input": "daaye doosra bhaag ke baad theenbaar daaye doosra chal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -``` - -## Usage -*Describe how to load your task and what is required for evaluation, if anything.* - -## Data Source -*Describe the data source for this Multilingual SCAN (hin).* - -## Limitations and Bias -*Note any known limitations or biases that the Multilingual SCAN (hin) has, with links and references if possible.* - -## GenBench Eval card -*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin/sample.jsonl b/src/genbench/tasks/multilingual_scan/hin/sample.jsonl deleted file mode 100644 index fa931bd..0000000 --- a/src/genbench/tasks/multilingual_scan/hin/sample.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -{"input": "daaye doosra bhaag ke baad theenbaar daaye doosra chal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -{"input": "dobaar daaye doosra bhaag ke baad theenbaar pura muro daaye", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} -{"input": "dobaar pura chal daaye ke baad dobaar baaye doosra dekh", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "baaye doosra bhaag or dobaar pura dekh baaye", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} -{"input": "theenbaar daaye dekh or dobaar baaye doosra dekh", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "dobaar baaye doosra chal or bhaag", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} -{"input": "theenbaar daaye doosra uchal ke baad theenbaar dekh", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} -{"input": "dobaar daaye doosra dekh ke baad dobaar muro baaye doosra", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} -{"input": "dobaar pura muro daaye ke baad dobaar baaye uchal", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} -{"input": "pura dekh daaye or theenbaar baaye doosra bhaag", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/__init__.py b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/config.jsonnet new file mode 100644 index 0000000..34859fa --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: add_prim_jump)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/add_prim_jump/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/add_prim_jump/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/doc.md b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/doc.md new file mode 100644 index 0000000..d80ee89 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_add_prim_jump) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_add_prim_jump).* + +## Examples +*Give some examples of the mSCAN (hin_add_prim_jump).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_add_prim_jump).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_add_prim_jump) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/task.py b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/task.py new file mode 100644 index 0000000..843b5aa --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_jump/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinAddPrimJump(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/__init__.py b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/config.jsonnet new file mode 100644 index 0000000..2c0d571 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: add_prim_turn_left)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/add_prim_turn_left/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/add_prim_turn_left/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/doc.md b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/doc.md new file mode 100644 index 0000000..82d9539 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_add_prim_turn_left) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_add_prim_turn_left).* + +## Examples +*Give some examples of the mSCAN (hin_add_prim_turn_left).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_add_prim_turn_left).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_add_prim_turn_left) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/task.py b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/task.py new file mode 100644 index 0000000..3d3354a --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_add_prim_turn_left/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinAddPrimTurnLeft(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_length/__init__.py b/src/genbench/tasks/multilingual_scan/hin_length/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_length/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_length/config.jsonnet new file mode 100644 index 0000000..1dd5b89 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_length/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: length)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/length/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/length/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_length/doc.md b/src/genbench/tasks/multilingual_scan/hin_length/doc.md new file mode 100644 index 0000000..b98cb9d --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_length/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_length) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_length).* + +## Examples +*Give some examples of the mSCAN (hin_length).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_length).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_length) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_length/task.py b/src/genbench/tasks/multilingual_scan/hin_length/task.py new file mode 100644 index 0000000..86aa014 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_length/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinLength(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd1/__init__.py b/src/genbench/tasks/multilingual_scan/hin_mcd1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd1/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_mcd1/config.jsonnet new file mode 100644 index 0000000..57373cb --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd1/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: mcd1)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd1/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd1/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd1/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd1/doc.md b/src/genbench/tasks/multilingual_scan/hin_mcd1/doc.md new file mode 100644 index 0000000..c5c7420 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd1/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_mcd1) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_mcd1).* + +## Examples +*Give some examples of the mSCAN (hin_mcd1).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_mcd1).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_mcd1) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd1/task.py b/src/genbench/tasks/multilingual_scan/hin_mcd1/task.py new file mode 100644 index 0000000..74d9b29 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd1/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinMcd1(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd2/__init__.py b/src/genbench/tasks/multilingual_scan/hin_mcd2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd2/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_mcd2/config.jsonnet new file mode 100644 index 0000000..27ff3dd --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd2/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: mcd2)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd2/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd2/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd2/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd2/doc.md b/src/genbench/tasks/multilingual_scan/hin_mcd2/doc.md new file mode 100644 index 0000000..60065d1 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd2/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_mcd2) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_mcd2).* + +## Examples +*Give some examples of the mSCAN (hin_mcd2).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_mcd2).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_mcd2) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd2/task.py b/src/genbench/tasks/multilingual_scan/hin_mcd2/task.py new file mode 100644 index 0000000..e08a1cb --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd2/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinMcd2(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd3/__init__.py b/src/genbench/tasks/multilingual_scan/hin_mcd3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd3/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_mcd3/config.jsonnet new file mode 100644 index 0000000..7fc968d --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd3/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: mcd3)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd3/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd3/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/mcd3/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd3/doc.md b/src/genbench/tasks/multilingual_scan/hin_mcd3/doc.md new file mode 100644 index 0000000..ec9c069 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd3/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_mcd3) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_mcd3).* + +## Examples +*Give some examples of the mSCAN (hin_mcd3).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_mcd3).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_mcd3) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_mcd3/task.py b/src/genbench/tasks/multilingual_scan/hin_mcd3/task.py new file mode 100644 index 0000000..063a585 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_mcd3/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinMcd3(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/hin_simple/__init__.py b/src/genbench/tasks/multilingual_scan/hin_simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/hin_simple/config.jsonnet b/src/genbench/tasks/multilingual_scan/hin_simple/config.jsonnet new file mode 100644 index 0000000..1950780 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_simple/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: hin, split: simple)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (hin) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/simple/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/hin/simple/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/hin_simple/doc.md b/src/genbench/tasks/multilingual_scan/hin_simple/doc.md new file mode 100644 index 0000000..7d1529b --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_simple/doc.md @@ -0,0 +1,19 @@ +# mSCAN (hin_simple) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (hin_simple).* + +## Examples +*Give some examples of the mSCAN (hin_simple).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (hin_simple).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (hin_simple) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/hin_simple/task.py b/src/genbench/tasks/multilingual_scan/hin_simple/task.py new file mode 100644 index 0000000..1c57a11 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/hin_simple/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanHinSimple(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus/config.jsonnet deleted file mode 100644 index 04f144f..0000000 --- a/src/genbench/tasks/multilingual_scan/rus/config.jsonnet +++ /dev/null @@ -1,40 +0,0 @@ -{ - name: 'Multilingual SCAN (rus)', - - description: 'Multilingual SCAN (rus) aims to measure aims to measure compositional and cross-lingual generalization in multilingual LLMs', - - keywords: [ - 'cross-lingual', - 'compositional', - ], - - authors: [ - 'Amélie Reymond', - 'Shane Steinert-Threlkeld', - - ], - - data_source: { - type: 'manual', - test: 'https://raw.githubusercontent.com/amelietamreymond/genbench_cbt/multilingual_scan/src/genbench/tasks/multilingual_scan/rus/sample.jsonl', - }, - - has_validation_set: false, - has_train_set: false, - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - - evaluation_metrics: [ - { - hf_id: 'exact_match', - git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", - best_score: 1.0, - } - ] -} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus/doc.md b/src/genbench/tasks/multilingual_scan/rus/doc.md deleted file mode 100644 index 7bc91ee..0000000 --- a/src/genbench/tasks/multilingual_scan/rus/doc.md +++ /dev/null @@ -1,22 +0,0 @@ -# Multilingual SCAN (rus) - -## Abstract - -This specific task consists of adapted SCAN benchmark into Russian (rus). - -## Examples - -```json -{"input": "Идти напротив справа трижды после бега справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -``` -## Usage -*Describe how to load your task and what is required for evaluation, if anything.* - -## Data Source -*Describe the data source for this Multilingual SCAN (rus).* - -## Limitations and Bias -*Note any known limitations or biases that the Multilingual SCAN (rus) has, with links and references if possible.* - -## GenBench Eval card -*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus/sample.jsonl b/src/genbench/tasks/multilingual_scan/rus/sample.jsonl deleted file mode 100644 index c8f7117..0000000 --- a/src/genbench/tasks/multilingual_scan/rus/sample.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -{"input": "Идти напротив справа трижды после бега справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK I_TURN_RIGHT I_TURN_RIGHT I_WALK"} -{"input": "Повернуть кругом направо посла бега напротив справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_RUN I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT"} -{"input": "Смотреть напротив слева дважды после езды кругом справа дважды", "target": "I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_RIGHT I_WALK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "Бегать напротив слева и смотреть вокруг налево дважды", "target": "I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK I_TURN_LEFT I_LOOK"} -{"input": "Смотреть справа трижды и смотреть напротив слева дважды", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK I_TURN_LEFT I_TURN_LEFT I_LOOK"} -{"input": "Идти напротив слева дважды и бегать", "target": "I_TURN_LEFT I_TURN_LEFT I_WALK I_TURN_LEFT I_TURN_LEFT I_WALK I_RUN"} -{"input": "Смотреть трижды после прыжка напротив справа трижды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_TURN_RIGHT I_TURN_RIGHT I_JUMP I_LOOK I_LOOK I_LOOK"} -{"input": "Повернуть напротив слева дважды после осмотра напротив справа дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT I_TURN_LEFT"} -{"input": "Прыгать влево дважды после поворота кругом направо дважды", "target": "I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_RIGHT I_TURN_LEFT I_JUMP I_TURN_LEFT I_JUMP"} -{"input": "Смотреть вокруг направо и бегать напротив налево трижды", "target": "I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_RIGHT I_LOOK I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN I_TURN_LEFT I_TURN_LEFT I_RUN"} diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/__init__.py b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/config.jsonnet new file mode 100644 index 0000000..55cfc72 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: add_prim_jump)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/add_prim_jump/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/add_prim_jump/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/doc.md b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/doc.md new file mode 100644 index 0000000..2b781db --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_add_prim_jump) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_add_prim_jump).* + +## Examples +*Give some examples of the mSCAN (rus_add_prim_jump).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_add_prim_jump).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_add_prim_jump) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/task.py b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/task.py new file mode 100644 index 0000000..decfa49 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_jump/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusAddPrimJump(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/__init__.py b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/config.jsonnet new file mode 100644 index 0000000..400ba0f --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: add_prim_turn_left)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/add_prim_turn_left/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/add_prim_turn_left/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/doc.md b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/doc.md new file mode 100644 index 0000000..eec63b0 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_add_prim_turn_left) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_add_prim_turn_left).* + +## Examples +*Give some examples of the mSCAN (rus_add_prim_turn_left).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_add_prim_turn_left).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_add_prim_turn_left) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/task.py b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/task.py new file mode 100644 index 0000000..3441e5b --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_add_prim_turn_left/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusAddPrimTurnLeft(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_length/__init__.py b/src/genbench/tasks/multilingual_scan/rus_length/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_length/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_length/config.jsonnet new file mode 100644 index 0000000..2aad9aa --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_length/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: length)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/length/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/length/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_length/doc.md b/src/genbench/tasks/multilingual_scan/rus_length/doc.md new file mode 100644 index 0000000..68be06c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_length/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_length) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_length).* + +## Examples +*Give some examples of the mSCAN (rus_length).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_length).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_length) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_length/task.py b/src/genbench/tasks/multilingual_scan/rus_length/task.py new file mode 100644 index 0000000..b9d8b4c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_length/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusLength(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd1/__init__.py b/src/genbench/tasks/multilingual_scan/rus_mcd1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd1/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_mcd1/config.jsonnet new file mode 100644 index 0000000..b491ec5 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd1/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: mcd1)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd1/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd1/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd1/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd1/doc.md b/src/genbench/tasks/multilingual_scan/rus_mcd1/doc.md new file mode 100644 index 0000000..70750f3 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd1/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_mcd1) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_mcd1).* + +## Examples +*Give some examples of the mSCAN (rus_mcd1).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_mcd1).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_mcd1) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd1/task.py b/src/genbench/tasks/multilingual_scan/rus_mcd1/task.py new file mode 100644 index 0000000..b4562f4 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd1/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusMcd1(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd2/__init__.py b/src/genbench/tasks/multilingual_scan/rus_mcd2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd2/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_mcd2/config.jsonnet new file mode 100644 index 0000000..5cad3f2 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd2/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: mcd2)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd2/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd2/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd2/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd2/doc.md b/src/genbench/tasks/multilingual_scan/rus_mcd2/doc.md new file mode 100644 index 0000000..07a5546 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd2/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_mcd2) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_mcd2).* + +## Examples +*Give some examples of the mSCAN (rus_mcd2).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_mcd2).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_mcd2) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd2/task.py b/src/genbench/tasks/multilingual_scan/rus_mcd2/task.py new file mode 100644 index 0000000..40fb09a --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd2/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusMcd2(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd3/__init__.py b/src/genbench/tasks/multilingual_scan/rus_mcd3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd3/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_mcd3/config.jsonnet new file mode 100644 index 0000000..87595a6 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd3/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: mcd3)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd3/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd3/test.jsonl', + validation: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/mcd3/dev.jsonl', + }, + + has_validation_set: true, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd3/doc.md b/src/genbench/tasks/multilingual_scan/rus_mcd3/doc.md new file mode 100644 index 0000000..892aad5 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd3/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_mcd3) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_mcd3).* + +## Examples +*Give some examples of the mSCAN (rus_mcd3).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_mcd3).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_mcd3) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_mcd3/task.py b/src/genbench/tasks/multilingual_scan/rus_mcd3/task.py new file mode 100644 index 0000000..4c7d22c --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_mcd3/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusMcd3(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/rus_simple/__init__.py b/src/genbench/tasks/multilingual_scan/rus_simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/multilingual_scan/rus_simple/config.jsonnet b/src/genbench/tasks/multilingual_scan/rus_simple/config.jsonnet new file mode 100644 index 0000000..9c08282 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_simple/config.jsonnet @@ -0,0 +1,55 @@ +{ + name: 'mSCAN (language: rus, split: simple)', + + // @TODO: Add a description of the task + description: 'Multilingual SCAN (rus) aims to measure ...', + + // @TODO: Add a list of keywords that describe the task + keywords: [ + 'keyword1', + 'keyword2', + ], + + authors: [ + 'Amélie Reymond', + 'Shane Steinert-Threlkeld', + ], + + data_source: { + type: 'manual', + train: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/simple/train.jsonl', + test: 'https://huggingface.co/datasets/ameliettr/mSCAN/raw/main/rus/simple/test.jsonl', + + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + }, + + prompt_based_testing: { + prompt_builder: { + instruction_zero_shot: '', + input_prefix: 'IN: ', + output_prefix: 'OUT: ', + append_choices_to_input: false, + few_shot_example_separator: '\n\n', + stop_string: '\n\n', + } + }, + + }, + + evaluation_metrics: [ + { + hf_id: 'exact_match', + git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", + best_score: 1.0, + } + ] +} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/rus_simple/doc.md b/src/genbench/tasks/multilingual_scan/rus_simple/doc.md new file mode 100644 index 0000000..124640a --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_simple/doc.md @@ -0,0 +1,19 @@ +# mSCAN (rus_simple) + +## Abstract +*Copy the abstract of your accompanying paper for this task here mSCAN (rus_simple).* + +## Examples +*Give some examples of the mSCAN (rus_simple).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this mSCAN (rus_simple).* + +## Limitations and Bias +*Note any known limitations or biases that the mSCAN (rus_simple) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/multilingual_scan/rus_simple/task.py b/src/genbench/tasks/multilingual_scan/rus_simple/task.py new file mode 100644 index 0000000..d76e6c8 --- /dev/null +++ b/src/genbench/tasks/multilingual_scan/rus_simple/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class MscanRusSimple(Task): + pass diff --git a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet b/src/genbench/tasks/multilingual_scan/tur/config.jsonnet deleted file mode 100644 index 21ca9d4..0000000 --- a/src/genbench/tasks/multilingual_scan/tur/config.jsonnet +++ /dev/null @@ -1,42 +0,0 @@ -{ - name: 'Multilingual SCAN (tur)', - - // @TODO: Add a description of the task - description: 'Multilingual SCAN (tur) aims to measure ...', - - // @TODO: Add a list of keywords that describe the task - keywords: [ - 'keyword1', - 'keyword2', - ], - - authors: [ - 'Amélie Reymond', - ' Shane Steinert-Threlkeld', - - ], - - data_source: { - type: 'manual', - test: 'https://raw.githubusercontent.com/GenBench/genbench_cbt/main/src/genbench/dummy_data/LLM_test.jsonl', - }, - - has_validation_set: false, - has_train_set: false, - - task_type: 'free_form', - - preparation_strategies: { - finetuning: { - objective: 'maximum_likelihood', - }, - }, - - evaluation_metrics: [ - { - hf_id: 'exact_match', - git_commit_sha: "758135da6a37ce962b7bc38c6dd5eab672d2b742", - best_score: 1.0, - } - ] -} \ No newline at end of file diff --git a/src/genbench/tasks/multilingual_scan/tur/task.py b/src/genbench/tasks/multilingual_scan/tur/task.py deleted file mode 100644 index 8f08a88..0000000 --- a/src/genbench/tasks/multilingual_scan/tur/task.py +++ /dev/null @@ -1,5 +0,0 @@ -from genbench import Task - - -class MultilingualScanTur(Task): - pass