From 1c4d44ad9ab82fdcfda6032fc5a58a887749b2c2 Mon Sep 17 00:00:00 2001 From: Oleg Somov Date: Mon, 24 Jul 2023 10:11:26 +0300 Subject: [PATCH 01/15] Added shifted pauq in in iid and tl setting --- src/genbench/tasks/shifted_pauq/__init__.py | 5 + .../tasks/shifted_pauq/config.jsonnet | 25 +++ src/genbench/tasks/shifted_pauq/doc.md | 148 ++++++++++++++++++ .../en_pauq_iid_split/__init__.py | 0 .../en_pauq_iid_split/config.jsonnet | 46 ++++++ .../shifted_pauq/en_pauq_iid_split/doc.md | 19 +++ .../shifted_pauq/en_pauq_iid_split/task.py | 5 + .../en_pauq_target_length_split/__init__.py | 0 .../config.jsonnet | 47 ++++++ .../en_pauq_target_length_split/doc.md | 19 +++ .../en_pauq_target_length_split/task.py | 5 + .../ru_pauq_iid_split/__init__.py | 0 .../ru_pauq_iid_split/config.jsonnet | 47 ++++++ .../shifted_pauq/ru_pauq_iid_split/doc.md | 19 +++ .../shifted_pauq/ru_pauq_iid_split/task.py | 5 + .../ru_pauq_target_length_split/__init__.py | 0 .../config.jsonnet | 47 ++++++ .../ru_pauq_target_length_split/doc.md | 19 +++ .../ru_pauq_target_length_split/task.py | 5 + 19 files changed, 461 insertions(+) create mode 100644 src/genbench/tasks/shifted_pauq/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_iid_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py diff --git a/src/genbench/tasks/shifted_pauq/__init__.py b/src/genbench/tasks/shifted_pauq/__init__.py new file mode 100644 index 0000000..eacc664 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/__init__.py @@ -0,0 +1,5 @@ +from genbench import TaskDict + + +class ShiftedPauq(TaskDict): + pass diff --git a/src/genbench/tasks/shifted_pauq/config.jsonnet b/src/genbench/tasks/shifted_pauq/config.jsonnet new file mode 100644 index 0000000..c02bc47 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/config.jsonnet @@ -0,0 +1,25 @@ +{ + name: 'Shifted Pauq', + + description: 'Shifted Pauq aims to measure compositional generalization in task of text2sql on different label shift splits.', + + keywords: [ + 'compositional generalization', + 'text2sql', + 'distribution shift', + 'multilingual' + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + subtasks_order: [ + 'ru_pauq_target_length_split', + 'en_pauq_target_length_split', + 'ru_pauq_iid_split', + 'en_pauq_iid_split', + ], +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/doc.md b/src/genbench/tasks/shifted_pauq/doc.md new file mode 100644 index 0000000..6651667 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/doc.md @@ -0,0 +1,148 @@ +## Motivation +Shifted Pauq is an update of Spider dataset. We have fixed the original queries and database content. +We have also translated both questions and queries to Russian language and updated database content with Russian entities. +In this evaluation we want to explore the ability of finetuned language models to cope with distribution shift +in a field of semantic parsing on different generated splits - target query length, +target maximum compound divergence(will be submitted later) in contrast to i.i.d. generated split. + +## Examples +Each sample follows the original Spider structure. +**English example** +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Treasury' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Homeland Security';", + 'question': "List the states where both the secretary of 'Treasury' department and the secretary of 'Homeland Security' were born.", + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, 2, [0, [0, 2, False], None], '"Treasury"', None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + ...}, + 'question_toks': ['List', + 'the', + 'states', + 'where', + 'both', + 'the', + 'secretary', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` + +**Russian example** + +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Управление общего учёта' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Бюджетная служба Конгресса';", + 'question': 'Перечислите штаты, в которых родились и секретарь управления общего учёта, и секретарь бюджетной службы Конгресса.', + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Управление общего учёта"', + None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + 'limit': None, + 'intersect': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Бюджетная служба Конгресса"', + None]], + ...}, + 'question_toks': ['Перечислите', + 'штаты', + ',', + 'в', + 'которых', + 'родились', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + 'management', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` + +Each query is supported by a corresponding database for measuring execution match. + +## Data Source +Originaly Spider was annotated by 11 Yale students. In our work we have refined it and translated with experts help. + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq has, with links and references if possible.* + +## Citation +[https://aclanthology.org/2022.findings-emnlp.175.pdf](https://aclanthology.org/2022.findings-emnlp.175.pdf) +``` +@inproceedings{bakshandaeva-etal-2022-pauq, + title = "{PAUQ}: Text-to-{SQL} in {R}ussian", + author = "Bakshandaeva, Daria and + Somov, Oleg and + Dmitrieva, Ekaterina and + Davydova, Vera and + Tutubalina, Elena", + booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2022", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.findings-emnlp.175", + pages = "2355--2376", + abstract = "Semantic parsing is an important task that allows to democratize human-computer interaction. One of the most popular text-to-SQL datasets with complex and diverse natural language (NL) questions and SQL queries is Spider. We construct and complement a Spider dataset for Russian, thus creating the first publicly available text-to-SQL dataset for this language. While examining its components - NL questions, SQL queries and databases content - we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. We select thirty functional test sets with different features that can be used for the evaluation of neural models{'} abilities. To conduct the experiments, we adapt baseline architectures RAT-SQL and BRIDGE and provide in-depth query component analysis. On the target language, both models demonstrate strong results with monolingual training and improved accuracy in multilingual scenario. In this paper, we also study trade-offs between machine-translated and manually-created NL queries. At present, Russian text-to-SQL is lacking in datasets as well as trained models, and we view this work as an important step towards filling this gap.", +} +``` + +## Further References +*Add any useful further references.* \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/__init__.py b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet new file mode 100644 index 0000000..6b168b3 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet @@ -0,0 +1,46 @@ +{ + name: 'Shifted Pauq (en_pauq_iid_split)', + + description: 'Shifted Pauq (en_pauq_iid_split) aims to measure model ability to generate SQL queries from English input with splits generated in i.i.d. strategy.', + + keywords: [ + 'text2sql', + 'iid distribution', + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'en_pauq_iid'], + git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e', + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md new file mode 100644 index 0000000..8c20139 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md @@ -0,0 +1,19 @@ +# Shifted Pauq (en_pauq_iid_split) + +## Abstract +This is original split from Spider dataset in English. + +## Examples +*Give some examples of the Shifted Pauq (en_pauq_iid_split).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Shifted Pauq (en_pauq_iid_split).* + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (en_pauq_iid_split) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py new file mode 100644 index 0000000..f84f8a7 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqEnPauqIidSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/__init__.py b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet new file mode 100644 index 0000000..2851f89 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet @@ -0,0 +1,47 @@ +{ + name: 'Shifted Pauq (en_pauq_target_length_split)', + + description: 'Shifted Pauq (en_pauq_target_length_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is English. Short queries are in test, while long queries are in train. Split was made on 30 percentile of dataset token length. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + + keywords: [ + 'text2sql', + 'label shift', + 'compositional generalization', + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'en_pauq_tl'], + git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e' + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md new file mode 100644 index 0000000..f02fab1 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md @@ -0,0 +1,19 @@ +# Shifted Pauq (en_pauq_target_length_split) + +## Abstract +This is length based split based on the token length of target query in English. + +## Examples +*Give some examples of the Shifted Pauq (en_pauq_target_length_split).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Shifted Pauq (en_pauq_target_length_split).* + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (en_pauq_target_length_split) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py new file mode 100644 index 0000000..d0dbfd1 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqEnPauqTargetLengthSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/__init__.py b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet new file mode 100644 index 0000000..83b8e44 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet @@ -0,0 +1,47 @@ +{ + name: 'Shifted Pauq (ru_pauq_iid_split)', + + description: 'Shifted Pauq (en_pauq_iid_split) aims to measure model ability to generate SQL queries from Russian input with splits generated in i.i.d. strategy.', + + keywords: [ + 'text2sql', + 'iid distribution', + 'multilingual' + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'ru_pauq_iid'], + git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e' + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md new file mode 100644 index 0000000..f7b461c --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -0,0 +1,19 @@ +# Shifted Pauq (ru_pauq_iid_split) + +## Abstract +This is original expertly translated split from Spider dataset in Russian. Questions and query values were translated. + +## Examples +*Give some examples of the Shifted Pauq (ru_pauq_iid_split).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Shifted Pauq (ru_pauq_iid_split).* + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (ru_pauq_iid_split) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py new file mode 100644 index 0000000..06d903a --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqRuPauqIidSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/__init__.py b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet new file mode 100644 index 0000000..85f2cd3 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet @@ -0,0 +1,47 @@ +{ + name: 'Shifted Pauq (ru_pauq_target_length_split)', + + description: 'Shifted Pauq (en_pauq_target_length_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is Russian. Short queries are in test, while long queries are in train. Split was made on 30 percentile of dataset token length. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + + keywords: [ + 'text2sql', + 'label shift', + 'compositional generalization', + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'ru_pauq_tl'], + git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e' + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md new file mode 100644 index 0000000..c443a6e --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -0,0 +1,19 @@ +# Shifted Pauq (ru_pauq_target_length_split) + +## Abstract +This is length based split based on the token length of target query in Russian. + +## Examples +*Give some examples of the Shifted Pauq (ru_pauq_target_length_split).* + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +## Data Source +*Describe the data source for this Shifted Pauq (ru_pauq_target_length_split).* + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py new file mode 100644 index 0000000..3e23e7d --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqRuPauqTargetLengthSplit(Task): + pass From de28ae9c3eae601370ca68fa9af927e5d3d123d8 Mon Sep 17 00:00:00 2001 From: Oleg Somov Date: Mon, 24 Jul 2023 12:43:11 +0300 Subject: [PATCH 02/15] add examples and tl explanation --- .../shifted_pauq/en_pauq_iid_split/doc.md | 42 +++++++++++- .../en_pauq_target_length_split/doc.md | 45 ++++++++++++- .../shifted_pauq/ru_pauq_iid_split/doc.md | 63 +++++++++++++++++- .../ru_pauq_target_length_split/doc.md | 66 ++++++++++++++++++- 4 files changed, 210 insertions(+), 6 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md index 8c20139..fbbbdea 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md @@ -4,7 +4,47 @@ This is original split from Spider dataset in English. ## Examples -*Give some examples of the Shifted Pauq (en_pauq_iid_split).* +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Treasury' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Homeland Security';", + 'question': "List the states where both the secretary of 'Treasury' department and the secretary of 'Homeland Security' were born.", + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, 2, [0, [0, 2, False], None], '"Treasury"', None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + ...}, + 'question_toks': ['List', + 'the', + 'states', + 'where', + 'both', + 'the', + 'secretary', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md index f02fab1..27b06d4 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md @@ -1,10 +1,51 @@ # Shifted Pauq (en_pauq_target_length_split) ## Abstract -This is length based split based on the token length of target query in English. +This is length based split based on the token length of target query in English. Dataset is split between train and test based +on 30th percentile. In order to measure compositional generalization we have verified that all SQL test tokens are present in train. ## Examples -*Give some examples of the Shifted Pauq (en_pauq_target_length_split).* +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Treasury' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Homeland Security';", + 'question': "List the states where both the secretary of 'Treasury' department and the secretary of 'Homeland Security' were born.", + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, 2, [0, [0, 2, False], None], '"Treasury"', None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + ...}, + 'question_toks': ['List', + 'the', + 'states', + 'where', + 'both', + 'the', + 'secretary', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md index f7b461c..d3a1308 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -4,7 +4,68 @@ This is original expertly translated split from Spider dataset in Russian. Questions and query values were translated. ## Examples -*Give some examples of the Shifted Pauq (ru_pauq_iid_split).* +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Управление общего учёта' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Бюджетная служба Конгресса';", + 'question': 'Перечислите штаты, в которых родились и секретарь управления общего учёта, и секретарь бюджетной службы Конгресса.', + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Управление общего учёта"', + None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + 'limit': None, + 'intersect': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Бюджетная служба Конгресса"', + None]], + ...}, + 'question_toks': ['Перечислите', + 'штаты', + ',', + 'в', + 'которых', + 'родились', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + 'management', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md index c443a6e..d69ac1a 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -1,10 +1,72 @@ # Shifted Pauq (ru_pauq_target_length_split) ## Abstract -This is length based split based on the token length of target query in Russian. +This is length based split based on the token length of target query in Russian. Dataset is split between train and test based +on 30th percentile. In order to measure compositional generalization we have verified that all SQL test tokens are present in train. ## Examples -*Give some examples of the Shifted Pauq (ru_pauq_target_length_split).* +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Управление общего учёта' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Бюджетная служба Конгресса';", + 'question': 'Перечислите штаты, в которых родились и секретарь управления общего учёта, и секретарь бюджетной службы Конгресса.', + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Управление общего учёта"', + None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + 'limit': None, + 'intersect': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Бюджетная служба Конгресса"', + None]], + ...}, + 'question_toks': ['Перечислите', + 'штаты', + ',', + 'в', + 'которых', + 'родились', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + 'management', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` ## Usage *Describe how to load your task and what is required for evaluation, if anything.* From 2ee052ee0b3b424b5198c2f28731c3bb7f929ac7 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:09:36 +0300 Subject: [PATCH 03/15] Update doc.md --- .../ru_pauq_target_length_split/doc.md | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md index d69ac1a..e624dc1 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -1,8 +1,21 @@ -# Shifted Pauq (ru_pauq_target_length_split) +# Shifted PAUQ -- Text-to-SQL (ru_pauq_target_length_split) +*A brief explaination of the task, its motivation and a description of the submssion data. Minimally, this should include: what generalisation property(ies) the task is testing for (GenBench taxonomy values); what assumptions are made about the training data; the size of the dataset; baseline scores. If it already contains this information, feel free to copy the abstract of the accompanying paper.* ## Abstract -This is length based split based on the token length of target query in Russian. Dataset is split between train and test based -on 30th percentile. In order to measure compositional generalization we have verified that all SQL test tokens are present in train. +In this work, we propose a custom split of the Russian _Text-to-SQL_ dataset, PAUQ [1], that assesses compositional generalization in a _text-to-query_ models. +The proposed split, ru_pauq_target_length_split, is length-based split based on the token length, i.e. datasets' items are separated by length such that the test set contains examples of different lengths than those in the train set. In this setup, we measure generalization to more complex queries. +Dataset is split between train and test based on 30th percentile. +In order to measure compositional generalization, we have verified that all SQL test tokens are present in train. + +baseline scores of a sequence-to-sequence model: TODO + +the size of the dataset: please refer to [1] for details + +the size of the split: TODO + + + + ## Examples ``` @@ -71,11 +84,45 @@ on 30th percentile. In order to measure compositional generalization we have ver ## Usage *Describe how to load your task and what is required for evaluation, if anything.* +For evaluation, we will use a sequence-to-sequence architecture. Please refer to config.jsonnet for details for now. + + ## Data Source *Describe the data source for this Shifted Pauq (ru_pauq_target_length_split).* +The PAUQ statistics can be found in [1] + ## Limitations and Bias *Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* +Our research explores distribution shift, i.e. spurious correlation using ru_pauq_target_length_split, and investigates how they affect the model. Specifically, we examine the scenario where test tokens are present in the training set and do not explore the more challenging case of modeling unseen tokens. + +Big language models such as Codex, a 175B GPT model further fine-tuned on code, are out of the scope of this study. + ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. + +**GenBench taxonomy values for the experiments:** + +Motivation: Practical + +Generalisation type: Compositional, Structural + +Shift type: Covariate + +Shift source: Naturally occuring + +Shift locus: train--test, Finetune train--test + + +## References +[1] PAUQ: Text-to-SQL in Russian +``` +@inproceedings{pauq, + title={PAUQ: Text-to-SQL in Russian}, + author={Bakshandaeva, Daria and Somov, Oleg and Dmitrieva, Ekaterina and Davydova, Vera and Tutubalina, Elena}, + booktitle={Findings of the Association for Computational Linguistics: EMNLP 2022}, + pages={2355--2376}, + year={2022} +} +``` From 5b8c14876efaf38cb0afd87510b97a024e5384c1 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:13:27 +0300 Subject: [PATCH 04/15] Update doc.md --- .../tasks/shifted_pauq/ru_pauq_target_length_split/doc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md index e624dc1..c9fb400 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -92,6 +92,8 @@ For evaluation, we will use a sequence-to-sequence architecture. Please refer to The PAUQ statistics can be found in [1] +Train/test data for the ru_pauq_target_length_split split is available at https://huggingface.co/datasets/composite/pauq/viewer/ru_pauq_tl + ## Limitations and Bias *Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* From 411698b00659af968b5ef883bc3b88d163f98310 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:07:59 +0300 Subject: [PATCH 05/15] Update doc.md --- .../shifted_pauq/ru_pauq_iid_split/doc.md | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md index d3a1308..f1a81b3 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -1,7 +1,14 @@ -# Shifted Pauq (ru_pauq_iid_split) +# Shifted PAUQ -- Text-to-SQL (ru_pauq_iid_split) +*A brief explaination of the task, its motivation and a description of the submssion data. Minimally, this should include: what generalisation property(ies) the task is testing for (GenBench taxonomy values); what assumptions are made about the training data; the size of the dataset; baseline scores. If it already contains this information, feel free to copy the abstract of the accompanying paper.* ## Abstract -This is original expertly translated split from Spider dataset in Russian. Questions and query values were translated. +This is an original split of the Russian _Text-to-SQL_ dataset, PAUQ [1], which is under the assumption of independency and identically (i.i.d) distributed train and test data. We provide this split in addition to the **ru_pauq_target_length_split** split. + +From PAUQ abstract [1]: We construct and complement a Spider dataset for the Russian language, thus creating the first publicly available text-to-SQL dataset in Russian. While examining dataset components—NL questions, SQL queries, and database content—we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. To conduct the experiment, we adapt baseline models RAT-SQL and BRIDGE and provide in-depth query component analysis. Both models demonstrate strong single language results and improved accuracy with multilingual training on the target language. + +baseline scores of a sequence-to-sequence model such as BRIDGE: please refer to [1] for details + +the size of the dataset: please refer to [1] for details ## Examples ``` @@ -70,11 +77,34 @@ This is original expertly translated split from Spider dataset in Russian. Quest ## Usage *Describe how to load your task and what is required for evaluation, if anything.* +For evaluation, we will use a sequence-to-sequence architecture. Evaluation metric is exact match. Please refer to config.jsonnet for details for now. + + ## Data Source -*Describe the data source for this Shifted Pauq (ru_pauq_iid_split).* +*Describe the data source for this Shifted Pauq (ru_pauq_target_length_split).* + +The PAUQ statistics can be found in [1] + +Train/test data for the ru_pauq_iid split is available at https://huggingface.co/datasets/composite/pauq/viewer/ru_pauq_iid ## Limitations and Bias -*Note any known limitations or biases that the Shifted Pauq (ru_pauq_iid_split) has, with links and references if possible.* +*Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* + +from [1]: PAUQ is an adaptation of the Spider dataset to Russian language, it indeed inherits most of Spider's limitations. First of all, the data is still `artificial' which means that it was created by a limited number of people specifically for training and evaluating text-to-SQL models, thus it lacks the diversity and complexity of natural data formed by questions that people formulate in order to get the desired information from the database. For instance, the real-world data contain NL queries that require common sense knowledge which can't be extracted directly from the database; ambiguous questions allowing various ways of interpretation that are quite frequent and queries with window functions that make the process easier and more convenient, -- all of these aren't included in the Spider dataset, as well as in PAUQ. Another limitation concerns evaluation metric -- exact match, which is the most commonly used to evaluate text-to-SQL models performance. However, the metric is too strict and prone to false negative results. + ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. + +**GenBench taxonomy values for the experiments:** + +Motivation: Practical + +Generalisation type: Compositional, Structural + +Shift type: Covariate + +Shift source: Naturally occuring + +Shift locus: train--test, Finetune train--test + From 962b6bb92903e287a2bb7a9fd9763a7ecefe9696 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:09:48 +0300 Subject: [PATCH 06/15] Update doc.md --- .../tasks/shifted_pauq/ru_pauq_target_length_split/doc.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md index c9fb400..9e21ea3 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -84,7 +84,7 @@ the size of the split: TODO ## Usage *Describe how to load your task and what is required for evaluation, if anything.* -For evaluation, we will use a sequence-to-sequence architecture. Please refer to config.jsonnet for details for now. +For evaluation, we will use a sequence-to-sequence architecture. Evaluation metric is exact match. Please refer to config.jsonnet for details for now. ## Data Source @@ -99,6 +99,8 @@ Train/test data for the ru_pauq_target_length_split split is available at https: Our research explores distribution shift, i.e. spurious correlation using ru_pauq_target_length_split, and investigates how they affect the model. Specifically, we examine the scenario where test tokens are present in the training set and do not explore the more challenging case of modeling unseen tokens. +Another limitation concerns evaluation metric -- exact match, which is the most commonly used to evaluate text-to-SQL models performance. However, this metric is too strict and prone to false negative results [1]. + Big language models such as Codex, a 175B GPT model further fine-tuned on code, are out of the scope of this study. ## GenBench Eval card From 98c76181e13f9842d37c8eb2e219b9b25898b9f8 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:21:37 +0300 Subject: [PATCH 07/15] Update doc.md --- src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md index f1a81b3..16e2592 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -1,9 +1,10 @@ # Shifted PAUQ -- Text-to-SQL (ru_pauq_iid_split) -*A brief explaination of the task, its motivation and a description of the submssion data. Minimally, this should include: what generalisation property(ies) the task is testing for (GenBench taxonomy values); what assumptions are made about the training data; the size of the dataset; baseline scores. If it already contains this information, feel free to copy the abstract of the accompanying paper.* ## Abstract This is an original split of the Russian _Text-to-SQL_ dataset, PAUQ [1], which is under the assumption of independency and identically (i.i.d) distributed train and test data. We provide this split in addition to the **ru_pauq_target_length_split** split. +PAUQ is the Russian version of the Spider, where all three components have been modified and localized: the natural language (NL) questions, the SQL queries, and the content of the databases. + From PAUQ abstract [1]: We construct and complement a Spider dataset for the Russian language, thus creating the first publicly available text-to-SQL dataset in Russian. While examining dataset components—NL questions, SQL queries, and database content—we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. To conduct the experiment, we adapt baseline models RAT-SQL and BRIDGE and provide in-depth query component analysis. Both models demonstrate strong single language results and improved accuracy with multilingual training on the target language. baseline scores of a sequence-to-sequence model such as BRIDGE: please refer to [1] for details From 86bb215afdd0b606ef9bc14b5c044dac95e2cb8d Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:22:37 +0300 Subject: [PATCH 08/15] Update doc.md --- src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md index 16e2592..b57bcaf 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -3,10 +3,6 @@ ## Abstract This is an original split of the Russian _Text-to-SQL_ dataset, PAUQ [1], which is under the assumption of independency and identically (i.i.d) distributed train and test data. We provide this split in addition to the **ru_pauq_target_length_split** split. -PAUQ is the Russian version of the Spider, where all three components have been modified and localized: the natural language (NL) questions, the SQL queries, and the content of the databases. - -From PAUQ abstract [1]: We construct and complement a Spider dataset for the Russian language, thus creating the first publicly available text-to-SQL dataset in Russian. While examining dataset components—NL questions, SQL queries, and database content—we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. To conduct the experiment, we adapt baseline models RAT-SQL and BRIDGE and provide in-depth query component analysis. Both models demonstrate strong single language results and improved accuracy with multilingual training on the target language. - baseline scores of a sequence-to-sequence model such as BRIDGE: please refer to [1] for details the size of the dataset: please refer to [1] for details From 6ac5a62da296862ca1ddfd55b2045992b24f3f46 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:32:20 +0300 Subject: [PATCH 09/15] Update doc.md --- src/genbench/tasks/shifted_pauq/doc.md | 39 +++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/doc.md b/src/genbench/tasks/shifted_pauq/doc.md index 6651667..1422f24 100644 --- a/src/genbench/tasks/shifted_pauq/doc.md +++ b/src/genbench/tasks/shifted_pauq/doc.md @@ -1,9 +1,19 @@ +In this work, we propose a custom split of a _Text-to-SQL_ dataset in Russian (PAUQ [1]) and English (Spider [2]) that assesses compositional generalization in _text-to-query_ models. + ## Motivation -Shifted Pauq is an update of Spider dataset. We have fixed the original queries and database content. +PAUQ [1] is the Russian version of the Spider [2]. We have fixed the original queries and database content. We have also translated both questions and queries to Russian language and updated database content with Russian entities. In this evaluation we want to explore the ability of finetuned language models to cope with distribution shift -in a field of semantic parsing on different generated splits - target query length, -target maximum compound divergence(will be submitted later) in contrast to i.i.d. generated split. +in a field of semantic parsing on different generated splits - target query length (see **ru_pauq_target_length_split**, **en_pauq_target_length_split**), +target maximum compound divergence (will be submitted later) in contrast to i.i.d. generated split (see **ru_pauq_iid_split**, **en_pauq_iid_split**). + +Additional details from PAUQ abstract [1]: We construct and complement a Spider dataset for the Russian language, thus creating the first publicly available text-to-SQL dataset in Russian. While examining dataset components—NL questions, SQL queries, and database content—we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. To conduct the experiment, we adapt baseline models RAT-SQL and BRIDGE and provide in-depth query component analysis. Both models demonstrate strong single language results and improved accuracy with multilingual training on the target language. + +baseline scores of a sequence-to-sequence model on our splits: TODO + +The size of the datasets: please refer to [1,2] for details + + ## Examples Each sample follows the original Spider structure. @@ -118,15 +128,18 @@ Each sample follows the original Spider structure. Each query is supported by a corresponding database for measuring execution match. ## Data Source -Originaly Spider was annotated by 11 Yale students. In our work we have refined it and translated with experts help. +Spider was annotated by 11 Yale students. In PAUQ, we have refined it and translated with experts help. Please refer to [1,2] ## Limitations and Bias *Note any known limitations or biases that the Shifted Pauq has, with links and references if possible.* +from [1]: PAUQ is an adaptation of the Spider dataset to Russian language, it indeed inherits most of Spider's limitations. First of all, the data is still `artificial' which means that it was created by a limited number of people specifically for training and evaluating text-to-SQL models, thus it lacks the diversity and complexity of natural data formed by questions that people formulate in order to get the desired information from the database. For instance, the real-world data contain NL queries that require common sense knowledge which can't be extracted directly from the database; ambiguous questions allowing various ways of interpretation that are quite frequent and queries with window functions that make the process easier and more convenient, -- all of these aren't included in the Spider dataset, as well as in PAUQ. Another limitation concerns evaluation metric -- exact match, which is the most commonly used to evaluate text-to-SQL models performance. However, the metric is too strict and prone to false negative results. + + ## Citation -[https://aclanthology.org/2022.findings-emnlp.175.pdf](https://aclanthology.org/2022.findings-emnlp.175.pdf) +[1] [https://aclanthology.org/2022.findings-emnlp.175.pdf](https://aclanthology.org/2022.findings-emnlp.175.pdf) ``` -@inproceedings{bakshandaeva-etal-2022-pauq, +@inproceedings{pauq, title = "{PAUQ}: Text-to-{SQL} in {R}ussian", author = "Bakshandaeva, Daria and Somov, Oleg and @@ -143,6 +156,18 @@ Originaly Spider was annotated by 11 Yale students. In our work we have refined abstract = "Semantic parsing is an important task that allows to democratize human-computer interaction. One of the most popular text-to-SQL datasets with complex and diverse natural language (NL) questions and SQL queries is Spider. We construct and complement a Spider dataset for Russian, thus creating the first publicly available text-to-SQL dataset for this language. While examining its components - NL questions, SQL queries and databases content - we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. We select thirty functional test sets with different features that can be used for the evaluation of neural models{'} abilities. To conduct the experiments, we adapt baseline architectures RAT-SQL and BRIDGE and provide in-depth query component analysis. On the target language, both models demonstrate strong results with monolingual training and improved accuracy in multilingual scenario. In this paper, we also study trade-offs between machine-translated and manually-created NL queries. At present, Russian text-to-SQL is lacking in datasets as well as trained models, and we view this work as an important step towards filling this gap.", } ``` +[2] [https://aclanthology.org/D18-1425/](https://aclanthology.org/D18-1425/) +``` +@inproceedings{spider, + title={Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task}, + author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others}, + booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing}, + pages={3911--3921}, + year={2018} +} +``` + + ## Further References -*Add any useful further references.* \ No newline at end of file +*Add any useful further references.* From 785a4c66be3a4228b6ff823b8a4abd02336c23ad Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:38:48 +0300 Subject: [PATCH 10/15] Update doc.md --- .../shifted_pauq/en_pauq_iid_split/doc.md | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md index fbbbdea..5882be1 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md @@ -1,7 +1,7 @@ -# Shifted Pauq (en_pauq_iid_split) +# Shifted PAUQ -- Text-to-SQL (en_pauq_iid_split) ## Abstract -This is original split from Spider dataset in English. +This is original split from Spider dataset in English [1]. We will use this split in addition to the **ru_pauq_iid_split** split to measure generalisation across languages. ## Examples ``` @@ -49,11 +49,35 @@ This is original split from Spider dataset in English. ## Usage *Describe how to load your task and what is required for evaluation, if anything.* +Please refer to the **ru_pauq_iid_split** card. + ## Data Source -*Describe the data source for this Shifted Pauq (en_pauq_iid_split).* +*Describe the data source for this Shifted PAUQ (en_pauq_iid_split).* + +Please refer to the **ru_pauq_iid_split** card. ## Limitations and Bias -*Note any known limitations or biases that the Shifted Pauq (en_pauq_iid_split) has, with links and references if possible.* + +Please refer to the **ru_pauq_iid_split** card. ## GenBench Eval card -*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for +your task*. + +**GenBench taxonomy values for the experiments:** + +Same as in the **ru_pauq_iid_split** card with addition: + +Generalisation type: generalisation across languages (from English to Russian) + +## References +[1] [https://aclanthology.org/D18-1425/](https://aclanthology.org/D18-1425/) +``` +@inproceedings{spider, + title={Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task}, + author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others}, + booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing}, + pages={3911--3921}, + year={2018} +} +``` From 669634c42614d722908488e78a1d1afb8addd21e Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:44:16 +0300 Subject: [PATCH 11/15] Update doc.md --- .../en_pauq_target_length_split/doc.md | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md index 27b06d4..2d94f16 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md @@ -1,7 +1,11 @@ -# Shifted Pauq (en_pauq_target_length_split) +# # Shifted PAUQ -- Text-to-SQL (en_pauq_target_length_split) ## Abstract -This is length based split based on the token length of target query in English. Dataset is split between train and test based +This is length based split based on the token length of target query in English. + +We will use this split in addition to the **ru_pauq_target_length_split** split to measure generalisation across languages. + +Dataset is split between train and test based on 30th percentile. In order to measure compositional generalization we have verified that all SQL test tokens are present in train. ## Examples @@ -50,11 +54,35 @@ on 30th percentile. In order to measure compositional generalization we have ver ## Usage *Describe how to load your task and what is required for evaluation, if anything.* +Please refer to the **ru_pauq_target_length_split** card. + ## Data Source *Describe the data source for this Shifted Pauq (en_pauq_target_length_split).* +Please refer to the **ru_pauq_target_length_split** card. + ## Limitations and Bias *Note any known limitations or biases that the Shifted Pauq (en_pauq_target_length_split) has, with links and references if possible.* +Please refer to the **ru_pauq_target_length_split** card. + ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. + +**GenBench taxonomy values for the experiments:** + +Same as in the **ru_pauq_target_length_split** card with addition: + +Generalisation type: generalisation across languages (from English to Russian) + +## References +[1] [https://aclanthology.org/D18-1425/](https://aclanthology.org/D18-1425/) +``` +@inproceedings{spider, + title={Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task}, + author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others}, + booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing}, + pages={3911--3921}, + year={2018} +} +``` From c2486f2b69b7c3dd5880e5fff521541890f369c2 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:45:42 +0300 Subject: [PATCH 12/15] Update doc.md --- .../shifted_pauq/ru_pauq_iid_split/doc.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md index b57bcaf..627aa73 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md @@ -105,3 +105,24 @@ Shift source: Naturally occuring Shift locus: train--test, Finetune train--test +## Citation +[1] [https://aclanthology.org/2022.findings-emnlp.175.pdf](https://aclanthology.org/2022.findings-emnlp.175.pdf) +``` +@inproceedings{pauq, + title = "{PAUQ}: Text-to-{SQL} in {R}ussian", + author = "Bakshandaeva, Daria and + Somov, Oleg and + Dmitrieva, Ekaterina and + Davydova, Vera and + Tutubalina, Elena", + booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2022", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.findings-emnlp.175", + pages = "2355--2376", + abstract = "Semantic parsing is an important task that allows to democratize human-computer interaction. One of the most popular text-to-SQL datasets with complex and diverse natural language (NL) questions and SQL queries is Spider. We construct and complement a Spider dataset for Russian, thus creating the first publicly available text-to-SQL dataset for this language. While examining its components - NL questions, SQL queries and databases content - we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. We select thirty functional test sets with different features that can be used for the evaluation of neural models{'} abilities. To conduct the experiments, we adapt baseline architectures RAT-SQL and BRIDGE and provide in-depth query component analysis. On the target language, both models demonstrate strong results with monolingual training and improved accuracy in multilingual scenario. In this paper, we also study trade-offs between machine-translated and manually-created NL queries. At present, Russian text-to-SQL is lacking in datasets as well as trained models, and we view this work as an important step towards filling this gap.", +} +``` + From 39ae7cef5516d7620d33aa8c38a1a797712e9261 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:48:23 +0300 Subject: [PATCH 13/15] Update doc.md --- src/genbench/tasks/shifted_pauq/doc.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/genbench/tasks/shifted_pauq/doc.md b/src/genbench/tasks/shifted_pauq/doc.md index 1422f24..e951bbb 100644 --- a/src/genbench/tasks/shifted_pauq/doc.md +++ b/src/genbench/tasks/shifted_pauq/doc.md @@ -7,6 +7,20 @@ In this evaluation we want to explore the ability of finetuned language models t in a field of semantic parsing on different generated splits - target query length (see **ru_pauq_target_length_split**, **en_pauq_target_length_split**), target maximum compound divergence (will be submitted later) in contrast to i.i.d. generated split (see **ru_pauq_iid_split**, **en_pauq_iid_split**). + +**GenBench taxonomy values for the experiments:** + +Motivation: Practical + +Generalisation type: Compositional, Structural, Across language + +Shift type: Covariate + +Shift source: Naturally occuring + +Shift locus: train--test, Finetune train--test + + Additional details from PAUQ abstract [1]: We construct and complement a Spider dataset for the Russian language, thus creating the first publicly available text-to-SQL dataset in Russian. While examining dataset components—NL questions, SQL queries, and database content—we identify limitations of the existing database structure, fill out missing values for tables and add new requests for underrepresented categories. To conduct the experiment, we adapt baseline models RAT-SQL and BRIDGE and provide in-depth query component analysis. Both models demonstrate strong single language results and improved accuracy with multilingual training on the target language. baseline scores of a sequence-to-sequence model on our splits: TODO From 0b2f82a00e7d6ef307dd930fad448fe7dee8c731 Mon Sep 17 00:00:00 2001 From: ai-spiderweb <108121422+ai-spiderweb@users.noreply.github.com> Date: Sat, 29 Jul 2023 22:07:25 +0700 Subject: [PATCH 14/15] REFACTOR: Update doc.md --- .../ru_pauq_target_length_split/doc.md | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md index 9e21ea3..3f27f72 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md @@ -7,15 +7,9 @@ The proposed split, ru_pauq_target_length_split, is length-based split based on Dataset is split between train and test based on 30th percentile. In order to measure compositional generalization, we have verified that all SQL test tokens are present in train. -baseline scores of a sequence-to-sequence model: TODO - -the size of the dataset: please refer to [1] for details - -the size of the split: TODO - - - - +* Baseline scores of a sequence-to-sequence model: TODO +* The size of the dataset: please refer to [1] for details +* The size of the split: TODO ## Examples ``` @@ -106,17 +100,13 @@ Big language models such as Codex, a 175B GPT model further fine-tuned on code, ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. -**GenBench taxonomy values for the experiments:** - -Motivation: Practical - -Generalisation type: Compositional, Structural - -Shift type: Covariate - -Shift source: Naturally occuring +### GenBench taxonomy values for the experiments: -Shift locus: train--test, Finetune train--test +* **Motivation:** Practical +* **Generalisation type:** Compositional, Structural +* **Shift type:** Covariate +* **Shift source:** Naturally occuring +* **Shift locus:** train--test, Finetune train--test ## References From 07b121acb53a276a7e254c429257ef9831ba14e7 Mon Sep 17 00:00:00 2001 From: Oleg Somov Date: Sat, 28 Oct 2023 12:41:23 +0300 Subject: [PATCH 15/15] update with paper splits --- .../tasks/shifted_pauq/config.jsonnet | 12 +- src/genbench/tasks/shifted_pauq/doc.md | 6 +- .../__init__.py | 0 .../config.jsonnet | 8 +- .../{en_pauq_iid_split => en_os_split}/doc.md | 10 +- .../tasks/shifted_pauq/en_os_split/task.py | 5 + .../shifted_pauq/en_pauq_iid_split/task.py | 5 - .../en_pauq_target_length_split/task.py | 5 - .../__init__.py | 0 .../config.jsonnet | 8 +- .../doc.md | 21 ++-- .../tasks/shifted_pauq/en_trl_split/task.py | 5 + .../__init__.py | 0 .../config.jsonnet | 8 +- .../tasks/shifted_pauq/en_tsl_split/doc.md | 87 +++++++++++++ .../tasks/shifted_pauq/en_tsl_split/task.py | 5 + .../__init__.py | 0 .../config.jsonnet | 8 +- .../{ru_pauq_iid_split => ru_os_split}/doc.md | 6 +- .../tasks/shifted_pauq/ru_os_split/task.py | 5 + .../shifted_pauq/ru_pauq_iid_split/task.py | 5 - .../ru_pauq_target_length_split/task.py | 5 - .../shifted_pauq/ru_trl_split/__init__.py | 0 .../shifted_pauq/ru_trl_split/config.jsonnet | 47 +++++++ .../doc.md | 15 +-- .../tasks/shifted_pauq/ru_trl_split/task.py | 5 + .../shifted_pauq/ru_tsl_split/__init__.py | 0 .../shifted_pauq/ru_tsl_split/config.jsonnet | 47 +++++++ .../tasks/shifted_pauq/ru_tsl_split/doc.md | 119 ++++++++++++++++++ .../tasks/shifted_pauq/ru_tsl_split/task.py | 5 + 30 files changed, 380 insertions(+), 72 deletions(-) rename src/genbench/tasks/shifted_pauq/{en_pauq_iid_split => en_os_split}/__init__.py (100%) rename src/genbench/tasks/shifted_pauq/{en_pauq_iid_split => en_os_split}/config.jsonnet (68%) rename src/genbench/tasks/shifted_pauq/{en_pauq_iid_split => en_os_split}/doc.md (90%) create mode 100644 src/genbench/tasks/shifted_pauq/en_os_split/task.py delete mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py delete mode 100644 src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py rename src/genbench/tasks/shifted_pauq/{en_pauq_target_length_split => en_trl_split}/__init__.py (100%) rename src/genbench/tasks/shifted_pauq/{ru_pauq_target_length_split => en_trl_split}/config.jsonnet (55%) rename src/genbench/tasks/shifted_pauq/{en_pauq_target_length_split => en_trl_split}/doc.md (78%) create mode 100644 src/genbench/tasks/shifted_pauq/en_trl_split/task.py rename src/genbench/tasks/shifted_pauq/{ru_pauq_iid_split => en_tsl_split}/__init__.py (100%) rename src/genbench/tasks/shifted_pauq/{en_pauq_target_length_split => en_tsl_split}/config.jsonnet (55%) create mode 100644 src/genbench/tasks/shifted_pauq/en_tsl_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/en_tsl_split/task.py rename src/genbench/tasks/shifted_pauq/{ru_pauq_target_length_split => ru_os_split}/__init__.py (100%) rename src/genbench/tasks/shifted_pauq/{ru_pauq_iid_split => ru_os_split}/config.jsonnet (68%) rename src/genbench/tasks/shifted_pauq/{ru_pauq_iid_split => ru_os_split}/doc.md (96%) create mode 100644 src/genbench/tasks/shifted_pauq/ru_os_split/task.py delete mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py delete mode 100644 src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_trl_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_trl_split/config.jsonnet rename src/genbench/tasks/shifted_pauq/{ru_pauq_target_length_split => ru_trl_split}/doc.md (78%) create mode 100644 src/genbench/tasks/shifted_pauq/ru_trl_split/task.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_tsl_split/__init__.py create mode 100644 src/genbench/tasks/shifted_pauq/ru_tsl_split/config.jsonnet create mode 100644 src/genbench/tasks/shifted_pauq/ru_tsl_split/doc.md create mode 100644 src/genbench/tasks/shifted_pauq/ru_tsl_split/task.py diff --git a/src/genbench/tasks/shifted_pauq/config.jsonnet b/src/genbench/tasks/shifted_pauq/config.jsonnet index c02bc47..bb19030 100644 --- a/src/genbench/tasks/shifted_pauq/config.jsonnet +++ b/src/genbench/tasks/shifted_pauq/config.jsonnet @@ -1,7 +1,7 @@ { name: 'Shifted Pauq', - description: 'Shifted Pauq aims to measure compositional generalization in task of text2sql on different label shift splits.', + description: 'Shifted Pauq aims to measure compositional generalization in task of text2sql on different full shift splits.', keywords: [ 'compositional generalization', @@ -17,9 +17,11 @@ ], subtasks_order: [ - 'ru_pauq_target_length_split', - 'en_pauq_target_length_split', - 'ru_pauq_iid_split', - 'en_pauq_iid_split', + 'ru_os_split', + 'en_os_split', + 'ru_trl_split', + 'en_trl_split', + 'ru_tsl_split', + 'en_tsl_split' ], } \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/doc.md b/src/genbench/tasks/shifted_pauq/doc.md index e951bbb..6fc66e9 100644 --- a/src/genbench/tasks/shifted_pauq/doc.md +++ b/src/genbench/tasks/shifted_pauq/doc.md @@ -4,8 +4,8 @@ In this work, we propose a custom split of a _Text-to-SQL_ dataset in Russian (P PAUQ [1] is the Russian version of the Spider [2]. We have fixed the original queries and database content. We have also translated both questions and queries to Russian language and updated database content with Russian entities. In this evaluation we want to explore the ability of finetuned language models to cope with distribution shift -in a field of semantic parsing on different generated splits - target query length (see **ru_pauq_target_length_split**, **en_pauq_target_length_split**), -target maximum compound divergence (will be submitted later) in contrast to i.i.d. generated split (see **ru_pauq_iid_split**, **en_pauq_iid_split**). +in a field of semantic parsing on different generated splits - target query length (see **ru_trl_split**, **en_trl_split**, **ru_tsl_split**, **en_tsl_split**), +target maximum compound divergence (will be submitted later) in contrast to i.i.d. generated split (see **ru_os_split**, **en_os_split**). **GenBench taxonomy values for the experiments:** @@ -14,7 +14,7 @@ Motivation: Practical Generalisation type: Compositional, Structural, Across language -Shift type: Covariate +Shift type: Full Shift source: Naturally occuring diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/__init__.py b/src/genbench/tasks/shifted_pauq/en_os_split/__init__.py similarity index 100% rename from src/genbench/tasks/shifted_pauq/en_pauq_iid_split/__init__.py rename to src/genbench/tasks/shifted_pauq/en_os_split/__init__.py diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/en_os_split/config.jsonnet similarity index 68% rename from src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet rename to src/genbench/tasks/shifted_pauq/en_os_split/config.jsonnet index 6b168b3..62e3260 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/config.jsonnet +++ b/src/genbench/tasks/shifted_pauq/en_os_split/config.jsonnet @@ -1,7 +1,7 @@ { - name: 'Shifted Pauq (en_pauq_iid_split)', + name: 'Shifted Pauq (en_os_split)', - description: 'Shifted Pauq (en_pauq_iid_split) aims to measure model ability to generate SQL queries from English input with splits generated in i.i.d. strategy.', + description: 'Shifted Pauq in English (en_os_split) aims to measure model ability to generate SQL queries from English input with splits generated in i.i.d. strategy.', keywords: [ 'text2sql', @@ -16,8 +16,8 @@ data_source: { type: 'hf', - hf_id: ['composite/pauq', 'en_pauq_iid'], - git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + hf_id: ['composite/pauq', 'en_os'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', }, has_validation_set: false, diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/en_os_split/doc.md similarity index 90% rename from src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md rename to src/genbench/tasks/shifted_pauq/en_os_split/doc.md index 5882be1..fd5a54b 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_os_split/doc.md @@ -1,7 +1,7 @@ # Shifted PAUQ -- Text-to-SQL (en_pauq_iid_split) ## Abstract -This is original split from Spider dataset in English [1]. We will use this split in addition to the **ru_pauq_iid_split** split to measure generalisation across languages. +This is original split from Spider dataset in English [1]. We will use this split in addition to the **ru_os_split** split to measure generalisation across languages. ## Examples ``` @@ -49,16 +49,16 @@ This is original split from Spider dataset in English [1]. We will use this spli ## Usage *Describe how to load your task and what is required for evaluation, if anything.* -Please refer to the **ru_pauq_iid_split** card. +Please refer to the **ru_os_split** card. ## Data Source *Describe the data source for this Shifted PAUQ (en_pauq_iid_split).* -Please refer to the **ru_pauq_iid_split** card. +Please refer to the **ru_os_split** card. ## Limitations and Bias -Please refer to the **ru_pauq_iid_split** card. +Please refer to the **ru_os_split** card. ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for @@ -66,7 +66,7 @@ your task*. **GenBench taxonomy values for the experiments:** -Same as in the **ru_pauq_iid_split** card with addition: +Same as in the **ru_os_split** card with addition: Generalisation type: generalisation across languages (from English to Russian) diff --git a/src/genbench/tasks/shifted_pauq/en_os_split/task.py b/src/genbench/tasks/shifted_pauq/en_os_split/task.py new file mode 100644 index 0000000..d612424 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_os_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqEnOsSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py b/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py deleted file mode 100644 index f84f8a7..0000000 --- a/src/genbench/tasks/shifted_pauq/en_pauq_iid_split/task.py +++ /dev/null @@ -1,5 +0,0 @@ -from genbench import Task - - -class ShiftedPauqEnPauqIidSplit(Task): - pass diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py b/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py deleted file mode 100644 index d0dbfd1..0000000 --- a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/task.py +++ /dev/null @@ -1,5 +0,0 @@ -from genbench import Task - - -class ShiftedPauqEnPauqTargetLengthSplit(Task): - pass diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/__init__.py b/src/genbench/tasks/shifted_pauq/en_trl_split/__init__.py similarity index 100% rename from src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/__init__.py rename to src/genbench/tasks/shifted_pauq/en_trl_split/__init__.py diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/en_trl_split/config.jsonnet similarity index 55% rename from src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet rename to src/genbench/tasks/shifted_pauq/en_trl_split/config.jsonnet index 85f2cd3..d40a2fc 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/config.jsonnet +++ b/src/genbench/tasks/shifted_pauq/en_trl_split/config.jsonnet @@ -1,7 +1,7 @@ { - name: 'Shifted Pauq (ru_pauq_target_length_split)', + name: 'Shifted Pauq (en_trl_split)', - description: 'Shifted Pauq (en_pauq_target_length_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is Russian. Short queries are in test, while long queries are in train. Split was made on 30 percentile of dataset token length. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + description: 'Shifted Pauq (en_trl_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is English. Short query templates are in test, while long query templates are in train. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', keywords: [ 'text2sql', @@ -17,8 +17,8 @@ data_source: { type: 'hf', - hf_id: ['composite/pauq', 'ru_pauq_tl'], - git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + hf_id: ['composite/pauq', 'en_trl'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', }, has_validation_set: false, diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/en_trl_split/doc.md similarity index 78% rename from src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md rename to src/genbench/tasks/shifted_pauq/en_trl_split/doc.md index 2d94f16..5f227c4 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/en_trl_split/doc.md @@ -1,12 +1,11 @@ -# # Shifted PAUQ -- Text-to-SQL (en_pauq_target_length_split) +# # Shifted PAUQ -- Text-to-SQL (en_trl_split) ## Abstract -This is length based split based on the token length of target query in English. +This is length based split based on the token length of target query template in English. -We will use this split in addition to the **ru_pauq_target_length_split** split to measure generalisation across languages. +We will use this split in addition to the **ru_trl_split** split to measure generalisation across languages. -Dataset is split between train and test based -on 30th percentile. In order to measure compositional generalization we have verified that all SQL test tokens are present in train. + In order to measure compositional generalization we have verified that all SQL test tokens are present in train. ## Examples ``` @@ -54,24 +53,24 @@ on 30th percentile. In order to measure compositional generalization we have ver ## Usage *Describe how to load your task and what is required for evaluation, if anything.* -Please refer to the **ru_pauq_target_length_split** card. +Please refer to the **ru_trl_split** card. ## Data Source -*Describe the data source for this Shifted Pauq (en_pauq_target_length_split).* +*Describe the data source for this Shifted Pauq (en_trl_split).* -Please refer to the **ru_pauq_target_length_split** card. +Please refer to the **ru_trl_split** card. ## Limitations and Bias -*Note any known limitations or biases that the Shifted Pauq (en_pauq_target_length_split) has, with links and references if possible.* +*Note any known limitations or biases that the Shifted Pauq (en_trl_split) has, with links and references if possible.* -Please refer to the **ru_pauq_target_length_split** card. +Please refer to the **ru_trl_split** card. ## GenBench Eval card *Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. **GenBench taxonomy values for the experiments:** -Same as in the **ru_pauq_target_length_split** card with addition: +Same as in the **ru_trl_split** card with addition: Generalisation type: generalisation across languages (from English to Russian) diff --git a/src/genbench/tasks/shifted_pauq/en_trl_split/task.py b/src/genbench/tasks/shifted_pauq/en_trl_split/task.py new file mode 100644 index 0000000..67d2876 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_trl_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqEnTRLSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/__init__.py b/src/genbench/tasks/shifted_pauq/en_tsl_split/__init__.py similarity index 100% rename from src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/__init__.py rename to src/genbench/tasks/shifted_pauq/en_tsl_split/__init__.py diff --git a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/en_tsl_split/config.jsonnet similarity index 55% rename from src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet rename to src/genbench/tasks/shifted_pauq/en_tsl_split/config.jsonnet index 2851f89..8520db3 100644 --- a/src/genbench/tasks/shifted_pauq/en_pauq_target_length_split/config.jsonnet +++ b/src/genbench/tasks/shifted_pauq/en_tsl_split/config.jsonnet @@ -1,7 +1,7 @@ { - name: 'Shifted Pauq (en_pauq_target_length_split)', + name: 'Shifted Pauq (en_tsl_split)', - description: 'Shifted Pauq (en_pauq_target_length_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is English. Short queries are in test, while long queries are in train. Split was made on 30 percentile of dataset token length. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + description: 'Shifted Pauq (en_tsl_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is English. Long query templates are in test, while short query templates are in train. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', keywords: [ 'text2sql', @@ -17,8 +17,8 @@ data_source: { type: 'hf', - hf_id: ['composite/pauq', 'en_pauq_tl'], - git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + hf_id: ['composite/pauq', 'en_tsl'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', }, has_validation_set: false, diff --git a/src/genbench/tasks/shifted_pauq/en_tsl_split/doc.md b/src/genbench/tasks/shifted_pauq/en_tsl_split/doc.md new file mode 100644 index 0000000..9274a80 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_tsl_split/doc.md @@ -0,0 +1,87 @@ +# # Shifted PAUQ -- Text-to-SQL (en_tsl_split) + +## Abstract +This is length based split based on the token length of target query template in English. + +We will use this split in addition to the **ru_tsl_split** split to measure generalisation across languages. + + In order to measure compositional generalization we have verified that all SQL test tokens are present in train. + +## Examples +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Treasury' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Homeland Security';", + 'question': "List the states where both the secretary of 'Treasury' department and the secretary of 'Homeland Security' were born.", + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, 2, [0, [0, 2, False], None], '"Treasury"', None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + ...}, + 'question_toks': ['List', + 'the', + 'states', + 'where', + 'both', + 'the', + 'secretary', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +Please refer to the **ru_tsl_split** card. + +## Data Source +*Describe the data source for this Shifted Pauq (en_tsl_split).* + +Please refer to the **ru_tsl_split** card. + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (en_tsl_split) has, with links and references if possible.* + +Please refer to the **ru_tsl_split** card. + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. + +**GenBench taxonomy values for the experiments:** + +Same as in the **ru_tsl_split** card with addition: + +Generalisation type: generalisation across languages (from English to Russian) + +## References +[1] [https://aclanthology.org/D18-1425/](https://aclanthology.org/D18-1425/) +``` +@inproceedings{spider, + title={Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task}, + author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others}, + booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing}, + pages={3911--3921}, + year={2018} +} +``` diff --git a/src/genbench/tasks/shifted_pauq/en_tsl_split/task.py b/src/genbench/tasks/shifted_pauq/en_tsl_split/task.py new file mode 100644 index 0000000..9976385 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/en_tsl_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqEnTSLSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/__init__.py b/src/genbench/tasks/shifted_pauq/ru_os_split/__init__.py similarity index 100% rename from src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/__init__.py rename to src/genbench/tasks/shifted_pauq/ru_os_split/__init__.py diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/ru_os_split/config.jsonnet similarity index 68% rename from src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet rename to src/genbench/tasks/shifted_pauq/ru_os_split/config.jsonnet index 83b8e44..44d8447 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/config.jsonnet +++ b/src/genbench/tasks/shifted_pauq/ru_os_split/config.jsonnet @@ -1,7 +1,7 @@ { - name: 'Shifted Pauq (ru_pauq_iid_split)', + name: 'Shifted Pauq (ru_os_split)', - description: 'Shifted Pauq (en_pauq_iid_split) aims to measure model ability to generate SQL queries from Russian input with splits generated in i.i.d. strategy.', + description: 'Shifted Pauq in Russian (ru_os_split) aims to measure model ability to generate SQL queries from Russian input with splits generated in i.i.d. strategy.', keywords: [ 'text2sql', @@ -17,8 +17,8 @@ data_source: { type: 'hf', - hf_id: ['composite/pauq', 'ru_pauq_iid'], - git_commit_sha: 'ddb9c0830ad37b084304e712eaacf02145c656e5', + hf_id: ['composite/pauq', 'ru_os'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', }, has_validation_set: false, diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_os_split/doc.md similarity index 96% rename from src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md rename to src/genbench/tasks/shifted_pauq/ru_os_split/doc.md index 627aa73..8c39f87 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_os_split/doc.md @@ -1,7 +1,7 @@ # Shifted PAUQ -- Text-to-SQL (ru_pauq_iid_split) ## Abstract -This is an original split of the Russian _Text-to-SQL_ dataset, PAUQ [1], which is under the assumption of independency and identically (i.i.d) distributed train and test data. We provide this split in addition to the **ru_pauq_target_length_split** split. +This is an original split of the Russian _Text-to-SQL_ dataset, PAUQ [1], which is under the assumption of independency and identically (i.i.d) distributed train and test data. We provide this split in addition to the **ru_trl_split** and **ru_tsl_split** splits. baseline scores of a sequence-to-sequence model such as BRIDGE: please refer to [1] for details @@ -78,11 +78,11 @@ For evaluation, we will use a sequence-to-sequence architecture. Evaluation metr ## Data Source -*Describe the data source for this Shifted Pauq (ru_pauq_target_length_split).* +*Describe the data source for this Shifted Pauq (ru_os_split).* The PAUQ statistics can be found in [1] -Train/test data for the ru_pauq_iid split is available at https://huggingface.co/datasets/composite/pauq/viewer/ru_pauq_iid +Train/test data for the ru_os split is available at https://huggingface.co/datasets/composite/pauq ## Limitations and Bias *Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* diff --git a/src/genbench/tasks/shifted_pauq/ru_os_split/task.py b/src/genbench/tasks/shifted_pauq/ru_os_split/task.py new file mode 100644 index 0000000..5c70a67 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_os_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqRuOsSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py b/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py deleted file mode 100644 index 06d903a..0000000 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_iid_split/task.py +++ /dev/null @@ -1,5 +0,0 @@ -from genbench import Task - - -class ShiftedPauqRuPauqIidSplit(Task): - pass diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py b/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py deleted file mode 100644 index 3e23e7d..0000000 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/task.py +++ /dev/null @@ -1,5 +0,0 @@ -from genbench import Task - - -class ShiftedPauqRuPauqTargetLengthSplit(Task): - pass diff --git a/src/genbench/tasks/shifted_pauq/ru_trl_split/__init__.py b/src/genbench/tasks/shifted_pauq/ru_trl_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/ru_trl_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/ru_trl_split/config.jsonnet new file mode 100644 index 0000000..0d58f70 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_trl_split/config.jsonnet @@ -0,0 +1,47 @@ +{ + name: 'Shifted Pauq (ru_trl_split)', + + description: 'Shifted Pauq (en_trl_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is Russian. Short query templates are in test, while long query templates are in train. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + + keywords: [ + 'text2sql', + 'label shift', + 'compositional generalization', + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'ru_trl'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e' + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_trl_split/doc.md similarity index 78% rename from src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md rename to src/genbench/tasks/shifted_pauq/ru_trl_split/doc.md index 3f27f72..a41fbe7 100644 --- a/src/genbench/tasks/shifted_pauq/ru_pauq_target_length_split/doc.md +++ b/src/genbench/tasks/shifted_pauq/ru_trl_split/doc.md @@ -1,10 +1,9 @@ -# Shifted PAUQ -- Text-to-SQL (ru_pauq_target_length_split) +# Shifted PAUQ -- Text-to-SQL (ru_trl_split) *A brief explaination of the task, its motivation and a description of the submssion data. Minimally, this should include: what generalisation property(ies) the task is testing for (GenBench taxonomy values); what assumptions are made about the training data; the size of the dataset; baseline scores. If it already contains this information, feel free to copy the abstract of the accompanying paper.* ## Abstract In this work, we propose a custom split of the Russian _Text-to-SQL_ dataset, PAUQ [1], that assesses compositional generalization in a _text-to-query_ models. -The proposed split, ru_pauq_target_length_split, is length-based split based on the token length, i.e. datasets' items are separated by length such that the test set contains examples of different lengths than those in the train set. In this setup, we measure generalization to more complex queries. -Dataset is split between train and test based on 30th percentile. +The proposed split, ru_trl_split, is length-based split based on the token length, i.e. datasets' items are separated by length such that the test set contains examples of different lengths than those in the train set. In this setup, we measure generalization to simpler queries. In order to measure compositional generalization, we have verified that all SQL test tokens are present in train. * Baseline scores of a sequence-to-sequence model: TODO @@ -82,18 +81,16 @@ For evaluation, we will use a sequence-to-sequence architecture. Evaluation metr ## Data Source -*Describe the data source for this Shifted Pauq (ru_pauq_target_length_split).* +*Describe the data source for this Shifted Pauq (ru_trl_split).* The PAUQ statistics can be found in [1] -Train/test data for the ru_pauq_target_length_split split is available at https://huggingface.co/datasets/composite/pauq/viewer/ru_pauq_tl +Train/test data for the ru_trl_split split is available at https://huggingface.co/datasets/composite/pauq ## Limitations and Bias -*Note any known limitations or biases that the Shifted Pauq (ru_pauq_target_length_split) has, with links and references if possible.* +*Note any known limitations or biases that the Shifted Pauq (ru_trl_split) has, with links and references if possible.* -Our research explores distribution shift, i.e. spurious correlation using ru_pauq_target_length_split, and investigates how they affect the model. Specifically, we examine the scenario where test tokens are present in the training set and do not explore the more challenging case of modeling unseen tokens. - -Another limitation concerns evaluation metric -- exact match, which is the most commonly used to evaluate text-to-SQL models performance. However, this metric is too strict and prone to false negative results [1]. +Our research explores distribution shift, i.e. data drift using ru_trl_split, and investigates how they affect the model. Specifically, we examine the scenario where test tokens are present in the training set and do not explore the more challenging case of modeling unseen tokens. Big language models such as Codex, a 175B GPT model further fine-tuned on code, are out of the scope of this study. diff --git a/src/genbench/tasks/shifted_pauq/ru_trl_split/task.py b/src/genbench/tasks/shifted_pauq/ru_trl_split/task.py new file mode 100644 index 0000000..d50e880 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_trl_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqRuTRLSplit(Task): + pass diff --git a/src/genbench/tasks/shifted_pauq/ru_tsl_split/__init__.py b/src/genbench/tasks/shifted_pauq/ru_tsl_split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/genbench/tasks/shifted_pauq/ru_tsl_split/config.jsonnet b/src/genbench/tasks/shifted_pauq/ru_tsl_split/config.jsonnet new file mode 100644 index 0000000..90ef79b --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_tsl_split/config.jsonnet @@ -0,0 +1,47 @@ +{ + name: 'Shifted Pauq (ru_tsl_split)', + + description: 'Shifted Pauq (en_tsl_split) aims to measure compositional generalization of on text2sql dataset with splits based on target length. Input language is Russian. Long query templates are in test, while short query templates are in train. To measure compostional distribution, it was made certain, that all of test tokens of target queries are present in train set.', + + keywords: [ + 'text2sql', + 'label shift', + 'compositional generalization', + ], + + authors: [ + 'Somov Oleg', + 'Dmietrieva Ekaterina', + 'Tutubalina Elena', + ], + + data_source: { + type: 'hf', + hf_id: ['composite/pauq', 'ru_tsl'], + git_commit_sha: '63e3e9329f785d097f4746618737d69530d1cdb4', + }, + + has_validation_set: false, + has_train_set: true, + + task_type: 'free_form', + + field_mapping: { + input: 'question', + target: 'query' + }, + + evaluation_metrics: [ + { + hf_id: 'evaluate-metric/exact_match', + best_score: 1.0, + git_commit_sha: '8e612716f2b1b08d23b0b2d7aa667d2f38eb989e' + } + ], + + preparation_strategies: { + finetuning: { + objective: 'maximum_likelihood', + } + } +} \ No newline at end of file diff --git a/src/genbench/tasks/shifted_pauq/ru_tsl_split/doc.md b/src/genbench/tasks/shifted_pauq/ru_tsl_split/doc.md new file mode 100644 index 0000000..45c4a33 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_tsl_split/doc.md @@ -0,0 +1,119 @@ +# Shifted PAUQ -- Text-to-SQL (ru_tsl_split) +*A brief explaination of the task, its motivation and a description of the submssion data. Minimally, this should include: what generalisation property(ies) the task is testing for (GenBench taxonomy values); what assumptions are made about the training data; the size of the dataset; baseline scores. If it already contains this information, feel free to copy the abstract of the accompanying paper.* + +## Abstract +In this work, we propose a custom split of the Russian _Text-to-SQL_ dataset, PAUQ [1], that assesses compositional generalization in a _text-to-query_ models. +The proposed split, ru_tsl_split, is length-based split based on the token length, i.e. datasets' items are separated by length such that the test set contains examples of different lengths than those in the train set. In this setup, we measure generalization to more complex queries. +In order to measure compositional generalization, we have verified that all SQL test tokens are present in train. + +* Baseline scores of a sequence-to-sequence model: TODO +* The size of the dataset: please refer to [1] for details +* The size of the split: TODO + +## Examples +``` +{'id': 'TS_0014', + 'db_id': 'department_management', + 'source': 'spider-train', + 'type': 'train', + 'query': "SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Управление общего учёта' INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = 'Бюджетная служба Конгресса';", + 'question': 'Перечислите штаты, в которых родились и секретарь управления общего учёта, и секретарь бюджетной службы Конгресса.', + 'sql': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Управление общего учёта"', + None]], + 'groupBy': [], + 'having': [], + 'orderBy': [], + 'limit': None, + 'intersect': {'from': {'table_units': [['table_unit', 0], + ['table_unit', 2], + ['table_unit', 1]], + 'conds': [[False, 2, [0, [0, 1, False], None], [0, 11, False], None], + 'and', + [False, 2, [0, [0, 12, False], None], [0, 7, False], None]]}, + 'select': [False, [[0, [0, [0, 9, False], None]]]], + 'where': [[False, + 2, + [0, [0, 2, False], None], + '"Бюджетная служба Конгресса"', + None]], + ...}, + 'question_toks': ['Перечислите', + 'штаты', + ',', + 'в', + 'которых', + 'родились', + ...], + 'query_toks': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + 'management', + ...], + 'query_toks_no_values': ['select', + 't3.born_state', + 'from', + 'department', + 'as', + 't1', + 'join', + ...], + 'masked_query': 'SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_1 INTERSECT SELECT T3.born_state FROM department AS T1 JOIN management AS T2 ON T1.department_id = T2.department_id JOIN head AS T3 ON T2.head_id = T3.head_id WHERE T1.name = TEXT_VALUE_2'} +``` + +## Usage +*Describe how to load your task and what is required for evaluation, if anything.* + +For evaluation, we will use a sequence-to-sequence architecture. Evaluation metric is exact match. Please refer to config.jsonnet for details for now. + + +## Data Source +*Describe the data source for this Shifted Pauq (ru_tsl_split).* + +The PAUQ statistics can be found in [1] + +Train/test data for the ru_tsl_split split is available at https://huggingface.co/datasets/composite/pauq + +## Limitations and Bias +*Note any known limitations or biases that the Shifted Pauq (ru_tsl_split) has, with links and references if possible.* + +Our research explores distribution shift, i.e. data drift using ru_tsl_split, and investigates how they affect the model. Specifically, we examine the scenario where test tokens are present in the training set and do not explore the more challenging case of modeling unseen tokens. + +Big language models such as Codex, a 175B GPT model further fine-tuned on code, are out of the scope of this study. + +## GenBench Eval card +*Describe what kind of generalisation your task is evaluating, and include a [genbench eval card](https://genbench.org/eval_cards/) for your task*. + +### GenBench taxonomy values for the experiments: + +* **Motivation:** Practical +* **Generalisation type:** Compositional, Structural +* **Shift type:** Covariate +* **Shift source:** Naturally occuring +* **Shift locus:** train--test, Finetune train--test + + +## References +[1] PAUQ: Text-to-SQL in Russian +``` +@inproceedings{pauq, + title={PAUQ: Text-to-SQL in Russian}, + author={Bakshandaeva, Daria and Somov, Oleg and Dmitrieva, Ekaterina and Davydova, Vera and Tutubalina, Elena}, + booktitle={Findings of the Association for Computational Linguistics: EMNLP 2022}, + pages={2355--2376}, + year={2022} +} +``` diff --git a/src/genbench/tasks/shifted_pauq/ru_tsl_split/task.py b/src/genbench/tasks/shifted_pauq/ru_tsl_split/task.py new file mode 100644 index 0000000..9ad79c3 --- /dev/null +++ b/src/genbench/tasks/shifted_pauq/ru_tsl_split/task.py @@ -0,0 +1,5 @@ +from genbench import Task + + +class ShiftedPauqRuTSLSplit(Task): + pass