From 6424f4cab19d437abcf5951a1591b4e4f4414eb9 Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 11:41:41 -0400 Subject: [PATCH 1/6] make pip-installable --- .gitignore | 3 +- abstentionbench/__init__.py | 7 ++ abstentionbench/analysis/__init__.py | 10 +++ .../analysis}/abstention_performance.csv | 0 .../analysis}/load_results.py | 4 +- .../analysis}/tables.py | 0 abstentionbench/configs/__init__.py | 1 + .../contains_abstention_keyword.yaml | 0 .../abstention_detector/llm_judge_gemini.yaml | 0 .../abstention_detector/llm_judge_gpt4o.yaml | 0 .../llm_judge_llama_3_1_8B_instruct.yaml | 0 .../llm_judge_llama_3_3_70B_instruct.yaml | 0 ..._llama_3_1_8B_instruct_with_reasoning.yaml | 0 .../abstention_method/direct_abstention.yaml | 0 .../llm_correctness_judge_gemini.yaml | 0 .../llm_correctness_judge_gpt4o.yaml | 0 ...rrectness_judge_llama_3_1_8B_instruct.yaml | 0 ...rectness_judge_llama_3_3_70B_instruct.yaml | 0 .../configs}/dataset/alcuna.yaml | 0 .../configs}/dataset/bbq.yaml | 0 .../dataset/big_bench_disambiguate.yaml | 0 .../dataset/big_bench_known_unknowns.yaml | 0 .../configs}/dataset/coconot.yaml | 0 .../configs}/dataset/dummy.yaml | 0 .../configs}/dataset/falseqa.yaml | 0 .../configs}/dataset/freshqa.yaml | 0 .../configs}/dataset/gpqa.yaml | 0 .../configs}/dataset/gsm8k.yaml | 0 .../configs}/dataset/kuq.yaml | 0 .../configs}/dataset/mediq.yaml | 0 .../configs}/dataset/mmlu_history.yaml | 0 .../configs}/dataset/mmlu_math.yaml | 0 .../configs}/dataset/moralchoice.yaml | 0 .../configs}/dataset/musique.yaml | 0 .../configs}/dataset/qaqa.yaml | 0 .../configs}/dataset/qasper.yaml | 0 .../configs}/dataset/self_aware.yaml | 0 .../configs}/dataset/situated_qa.yaml | 0 .../configs}/dataset/squad2.yaml | 0 .../configs}/dataset/umwp.yaml | 0 .../configs}/dataset/worldsense.yaml | 0 .../configs}/default_pipeline.yaml | 0 .../configs}/mode/cluster.yaml | 0 .../configs}/mode/local.yaml | 0 .../model/deepseek_r1_distill_llama_70B.yaml | 0 .../configs}/model/dummy.yaml | 0 .../configs}/model/gemini_15_pro.yaml | 0 .../configs}/model/gpt4o.yaml | 0 .../model/llama_3_1_405B_instruct.yaml | 0 .../configs}/model/llama_3_1_70B_base.yaml | 0 .../model/llama_3_1_70B_instruct.yaml | 0 .../model/llama_3_1_70B_tulu_3_dpo.yaml | 0 .../model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml | 0 .../model/llama_3_1_70B_tulu_3_sft.yaml | 0 .../configs}/model/llama_3_1_8B_base.yaml | 0 .../configs}/model/llama_3_1_8B_instruct.yaml | 0 .../model/llama_3_1_8B_tulu_3_dpo.yaml | 0 .../model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml | 0 .../model/llama_3_1_8B_tulu_3_sft.yaml | 0 .../model/llama_3_3_70B_instruct.yaml | 0 .../model/mistral_7B_instruct_v0_3.yaml | 0 .../configs}/model/o1.yaml | 0 .../configs}/model/olmo_7B_0724_instruct.yaml | 0 .../configs}/model/qwen2_5_32B_instruct.yaml | 0 .../configs}/model/s1_1_32B.yaml | 0 .../configs}/model/tiny_llama_chat.yaml | 0 .../data}/UMWP_indices_answerable.json | 0 abstentionbench/data/__init__.py | 35 +++++++++ .../data}/fast-subset-indices.json | 0 .../data}/kuq/new-category-mapping.csv | 0 .../data}/subsampling-indices.json | 0 .../recipe}/__init__.py | 0 .../recipe}/abstention.py | 2 +- .../abstract_abstention_dataset.py | 0 .../recipe}/abstention_datasets/alcuna.py | 7 +- .../recipe}/abstention_datasets/bbq.py | 10 ++- .../recipe}/abstention_datasets/big_bench.py | 9 ++- .../recipe}/abstention_datasets/coconot.py | 2 +- .../recipe}/abstention_datasets/false_qa.py | 12 +-- .../recipe}/abstention_datasets/freshqa.py | 2 +- .../recipe}/abstention_datasets/gpqa.py | 2 +- .../recipe}/abstention_datasets/gsm8k.py | 2 +- .../recipe}/abstention_datasets/kuq.py | 2 +- .../recipe}/abstention_datasets/mediq.py | 17 +++-- .../recipe}/abstention_datasets/mmlu.py | 2 +- .../abstention_datasets/moralchoice.py | 2 +- .../recipe}/abstention_datasets/musique.py | 3 +- .../recipe}/abstention_datasets/nq_dataset.py | 7 +- .../recipe}/abstention_datasets/qaqa.py | 9 ++- .../recipe}/abstention_datasets/qasper.py | 8 +- .../recipe}/abstention_datasets/self_aware.py | 10 ++- .../abstention_datasets/situated_qa.py | 4 +- .../recipe}/abstention_datasets/squad.py | 4 +- .../recipe}/abstention_datasets/umwp.py | 7 +- .../abstention_datasets/world_sense.py | 7 +- .../recipe}/abstention_keywords.py | 0 .../recipe}/evaluation.py | 8 +- .../recipe}/evaluation_judge_prompts.py | 0 .../recipe}/inference.py | 6 +- .../recipe}/job_manager.py | 0 {recipe => abstentionbench/recipe}/models.py | 2 +- .../recipe}/prompts_archive.py | 0 .../recipe}/system_prompt.py | 0 {recipe => abstentionbench/recipe}/utils.py | 0 main.py | 8 +- pyproject.toml | 76 +++++++++++++++++++ tests/run_llama3_instruct_inference.py | 2 +- tests/test_abstention_detector.py | 8 +- tests/test_correctness_evaluator.py | 8 +- tests/test_datasets.py | 42 +++++----- tests/test_inference.py | 6 +- tests/test_job_manager.py | 2 +- tests/test_models.py | 2 +- 113 files changed, 254 insertions(+), 106 deletions(-) create mode 100644 abstentionbench/__init__.py create mode 100644 abstentionbench/analysis/__init__.py rename {analysis => abstentionbench/analysis}/abstention_performance.csv (100%) rename {analysis => abstentionbench/analysis}/load_results.py (99%) rename {analysis => abstentionbench/analysis}/tables.py (100%) create mode 100644 abstentionbench/configs/__init__.py rename {configs => abstentionbench/configs}/abstention_detector/contains_abstention_keyword.yaml (100%) rename {configs => abstentionbench/configs}/abstention_detector/llm_judge_gemini.yaml (100%) rename {configs => abstentionbench/configs}/abstention_detector/llm_judge_gpt4o.yaml (100%) rename {configs => abstentionbench/configs}/abstention_detector/llm_judge_llama_3_1_8B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/abstention_detector/llm_judge_llama_3_3_70B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/abstention_detector_with_reasoning/llm_judge_llama_3_1_8B_instruct_with_reasoning.yaml (100%) rename {configs => abstentionbench/configs}/abstention_method/direct_abstention.yaml (100%) rename {configs => abstentionbench/configs}/correctness_evaluator/llm_correctness_judge_gemini.yaml (100%) rename {configs => abstentionbench/configs}/correctness_evaluator/llm_correctness_judge_gpt4o.yaml (100%) rename {configs => abstentionbench/configs}/correctness_evaluator/llm_correctness_judge_llama_3_1_8B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/correctness_evaluator/llm_correctness_judge_llama_3_3_70B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/dataset/alcuna.yaml (100%) rename {configs => abstentionbench/configs}/dataset/bbq.yaml (100%) rename {configs => abstentionbench/configs}/dataset/big_bench_disambiguate.yaml (100%) rename {configs => abstentionbench/configs}/dataset/big_bench_known_unknowns.yaml (100%) rename {configs => abstentionbench/configs}/dataset/coconot.yaml (100%) rename {configs => abstentionbench/configs}/dataset/dummy.yaml (100%) rename {configs => abstentionbench/configs}/dataset/falseqa.yaml (100%) rename {configs => abstentionbench/configs}/dataset/freshqa.yaml (100%) rename {configs => abstentionbench/configs}/dataset/gpqa.yaml (100%) rename {configs => abstentionbench/configs}/dataset/gsm8k.yaml (100%) rename {configs => abstentionbench/configs}/dataset/kuq.yaml (100%) rename {configs => abstentionbench/configs}/dataset/mediq.yaml (100%) rename {configs => abstentionbench/configs}/dataset/mmlu_history.yaml (100%) rename {configs => abstentionbench/configs}/dataset/mmlu_math.yaml (100%) rename {configs => abstentionbench/configs}/dataset/moralchoice.yaml (100%) rename {configs => abstentionbench/configs}/dataset/musique.yaml (100%) rename {configs => abstentionbench/configs}/dataset/qaqa.yaml (100%) rename {configs => abstentionbench/configs}/dataset/qasper.yaml (100%) rename {configs => abstentionbench/configs}/dataset/self_aware.yaml (100%) rename {configs => abstentionbench/configs}/dataset/situated_qa.yaml (100%) rename {configs => abstentionbench/configs}/dataset/squad2.yaml (100%) rename {configs => abstentionbench/configs}/dataset/umwp.yaml (100%) rename {configs => abstentionbench/configs}/dataset/worldsense.yaml (100%) rename {configs => abstentionbench/configs}/default_pipeline.yaml (100%) rename {configs => abstentionbench/configs}/mode/cluster.yaml (100%) rename {configs => abstentionbench/configs}/mode/local.yaml (100%) rename {configs => abstentionbench/configs}/model/deepseek_r1_distill_llama_70B.yaml (100%) rename {configs => abstentionbench/configs}/model/dummy.yaml (100%) rename {configs => abstentionbench/configs}/model/gemini_15_pro.yaml (100%) rename {configs => abstentionbench/configs}/model/gpt4o.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_405B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_70B_base.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_70B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_70B_tulu_3_dpo.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_70B_tulu_3_sft.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_8B_base.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_8B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_8B_tulu_3_dpo.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_1_8B_tulu_3_sft.yaml (100%) rename {configs => abstentionbench/configs}/model/llama_3_3_70B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/mistral_7B_instruct_v0_3.yaml (100%) rename {configs => abstentionbench/configs}/model/o1.yaml (100%) rename {configs => abstentionbench/configs}/model/olmo_7B_0724_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/qwen2_5_32B_instruct.yaml (100%) rename {configs => abstentionbench/configs}/model/s1_1_32B.yaml (100%) rename {configs => abstentionbench/configs}/model/tiny_llama_chat.yaml (100%) rename {data => abstentionbench/data}/UMWP_indices_answerable.json (100%) create mode 100644 abstentionbench/data/__init__.py rename {data => abstentionbench/data}/fast-subset-indices.json (100%) rename {data => abstentionbench/data}/kuq/new-category-mapping.csv (100%) rename {data => abstentionbench/data}/subsampling-indices.json (100%) rename {recipe => abstentionbench/recipe}/__init__.py (100%) rename {recipe => abstentionbench/recipe}/abstention.py (97%) rename {recipe => abstentionbench/recipe}/abstention_datasets/abstract_abstention_dataset.py (100%) rename {recipe => abstentionbench/recipe}/abstention_datasets/alcuna.py (97%) rename {recipe => abstentionbench/recipe}/abstention_datasets/bbq.py (90%) rename {recipe => abstentionbench/recipe}/abstention_datasets/big_bench.py (94%) rename {recipe => abstentionbench/recipe}/abstention_datasets/coconot.py (98%) rename {recipe => abstentionbench/recipe}/abstention_datasets/false_qa.py (84%) rename {recipe => abstentionbench/recipe}/abstention_datasets/freshqa.py (97%) rename {recipe => abstentionbench/recipe}/abstention_datasets/gpqa.py (98%) rename {recipe => abstentionbench/recipe}/abstention_datasets/gsm8k.py (98%) rename {recipe => abstentionbench/recipe}/abstention_datasets/kuq.py (97%) rename {recipe => abstentionbench/recipe}/abstention_datasets/mediq.py (88%) rename {recipe => abstentionbench/recipe}/abstention_datasets/mmlu.py (98%) rename {recipe => abstentionbench/recipe}/abstention_datasets/moralchoice.py (96%) rename {recipe => abstentionbench/recipe}/abstention_datasets/musique.py (92%) rename {recipe => abstentionbench/recipe}/abstention_datasets/nq_dataset.py (95%) rename {recipe => abstentionbench/recipe}/abstention_datasets/qaqa.py (89%) rename {recipe => abstentionbench/recipe}/abstention_datasets/qasper.py (94%) rename {recipe => abstentionbench/recipe}/abstention_datasets/self_aware.py (83%) rename {recipe => abstentionbench/recipe}/abstention_datasets/situated_qa.py (92%) rename {recipe => abstentionbench/recipe}/abstention_datasets/squad.py (93%) rename {recipe => abstentionbench/recipe}/abstention_datasets/umwp.py (95%) rename {recipe => abstentionbench/recipe}/abstention_datasets/world_sense.py (94%) rename {recipe => abstentionbench/recipe}/abstention_keywords.py (100%) rename {recipe => abstentionbench/recipe}/evaluation.py (97%) rename {recipe => abstentionbench/recipe}/evaluation_judge_prompts.py (100%) rename {recipe => abstentionbench/recipe}/inference.py (95%) rename {recipe => abstentionbench/recipe}/job_manager.py (100%) rename {recipe => abstentionbench/recipe}/models.py (99%) rename {recipe => abstentionbench/recipe}/prompts_archive.py (100%) rename {recipe => abstentionbench/recipe}/system_prompt.py (100%) rename {recipe => abstentionbench/recipe}/utils.py (100%) create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 5281781..a9521c7 100644 --- a/.gitignore +++ b/.gitignore @@ -204,4 +204,5 @@ __marimo__/ playground*.ipynb -data/* \ No newline at end of file +data/* +uv.lock \ No newline at end of file diff --git a/abstentionbench/__init__.py b/abstentionbench/__init__.py new file mode 100644 index 0000000..2347e19 --- /dev/null +++ b/abstentionbench/__init__.py @@ -0,0 +1,7 @@ +""" +AbstentionBench: A Holistic Benchmark for LLM Abstention + +This package provides tools and datasets for evaluating LLM abstention capabilities. +""" + +__version__ = "1.0.0" \ No newline at end of file diff --git a/abstentionbench/analysis/__init__.py b/abstentionbench/analysis/__init__.py new file mode 100644 index 0000000..364c5ad --- /dev/null +++ b/abstentionbench/analysis/__init__.py @@ -0,0 +1,10 @@ +""" +Analysis package for AbstentionBench. + +Provides utilities for loading results and generating tables. +""" + +from .load_results import Results +from .tables import AbstentionF1ScoreTable + +__all__ = ["Results", "AbstentionF1ScoreTable"] \ No newline at end of file diff --git a/analysis/abstention_performance.csv b/abstentionbench/analysis/abstention_performance.csv similarity index 100% rename from analysis/abstention_performance.csv rename to abstentionbench/analysis/abstention_performance.csv diff --git a/analysis/load_results.py b/abstentionbench/analysis/load_results.py similarity index 99% rename from analysis/load_results.py rename to abstentionbench/analysis/load_results.py index 76f0cdc..2d927eb 100644 --- a/analysis/load_results.py +++ b/abstentionbench/analysis/load_results.py @@ -11,8 +11,8 @@ import pandas as pd -from recipe.abstention import Responses -from recipe.job_manager import JobManager +from ..recipe.abstention import Responses +from ..recipe.job_manager import JobManager logger = logging.getLogger(__name__) diff --git a/analysis/tables.py b/abstentionbench/analysis/tables.py similarity index 100% rename from analysis/tables.py rename to abstentionbench/analysis/tables.py diff --git a/abstentionbench/configs/__init__.py b/abstentionbench/configs/__init__.py new file mode 100644 index 0000000..efd15c4 --- /dev/null +++ b/abstentionbench/configs/__init__.py @@ -0,0 +1 @@ +# Configuration files for AbstentionBench \ No newline at end of file diff --git a/configs/abstention_detector/contains_abstention_keyword.yaml b/abstentionbench/configs/abstention_detector/contains_abstention_keyword.yaml similarity index 100% rename from configs/abstention_detector/contains_abstention_keyword.yaml rename to abstentionbench/configs/abstention_detector/contains_abstention_keyword.yaml diff --git a/configs/abstention_detector/llm_judge_gemini.yaml b/abstentionbench/configs/abstention_detector/llm_judge_gemini.yaml similarity index 100% rename from configs/abstention_detector/llm_judge_gemini.yaml rename to abstentionbench/configs/abstention_detector/llm_judge_gemini.yaml diff --git a/configs/abstention_detector/llm_judge_gpt4o.yaml b/abstentionbench/configs/abstention_detector/llm_judge_gpt4o.yaml similarity index 100% rename from configs/abstention_detector/llm_judge_gpt4o.yaml rename to abstentionbench/configs/abstention_detector/llm_judge_gpt4o.yaml diff --git a/configs/abstention_detector/llm_judge_llama_3_1_8B_instruct.yaml b/abstentionbench/configs/abstention_detector/llm_judge_llama_3_1_8B_instruct.yaml similarity index 100% rename from configs/abstention_detector/llm_judge_llama_3_1_8B_instruct.yaml rename to abstentionbench/configs/abstention_detector/llm_judge_llama_3_1_8B_instruct.yaml diff --git a/configs/abstention_detector/llm_judge_llama_3_3_70B_instruct.yaml b/abstentionbench/configs/abstention_detector/llm_judge_llama_3_3_70B_instruct.yaml similarity index 100% rename from configs/abstention_detector/llm_judge_llama_3_3_70B_instruct.yaml rename to abstentionbench/configs/abstention_detector/llm_judge_llama_3_3_70B_instruct.yaml diff --git a/configs/abstention_detector_with_reasoning/llm_judge_llama_3_1_8B_instruct_with_reasoning.yaml b/abstentionbench/configs/abstention_detector_with_reasoning/llm_judge_llama_3_1_8B_instruct_with_reasoning.yaml similarity index 100% rename from configs/abstention_detector_with_reasoning/llm_judge_llama_3_1_8B_instruct_with_reasoning.yaml rename to abstentionbench/configs/abstention_detector_with_reasoning/llm_judge_llama_3_1_8B_instruct_with_reasoning.yaml diff --git a/configs/abstention_method/direct_abstention.yaml b/abstentionbench/configs/abstention_method/direct_abstention.yaml similarity index 100% rename from configs/abstention_method/direct_abstention.yaml rename to abstentionbench/configs/abstention_method/direct_abstention.yaml diff --git a/configs/correctness_evaluator/llm_correctness_judge_gemini.yaml b/abstentionbench/configs/correctness_evaluator/llm_correctness_judge_gemini.yaml similarity index 100% rename from configs/correctness_evaluator/llm_correctness_judge_gemini.yaml rename to abstentionbench/configs/correctness_evaluator/llm_correctness_judge_gemini.yaml diff --git a/configs/correctness_evaluator/llm_correctness_judge_gpt4o.yaml b/abstentionbench/configs/correctness_evaluator/llm_correctness_judge_gpt4o.yaml similarity index 100% rename from configs/correctness_evaluator/llm_correctness_judge_gpt4o.yaml rename to abstentionbench/configs/correctness_evaluator/llm_correctness_judge_gpt4o.yaml diff --git a/configs/correctness_evaluator/llm_correctness_judge_llama_3_1_8B_instruct.yaml b/abstentionbench/configs/correctness_evaluator/llm_correctness_judge_llama_3_1_8B_instruct.yaml similarity index 100% rename from configs/correctness_evaluator/llm_correctness_judge_llama_3_1_8B_instruct.yaml rename to abstentionbench/configs/correctness_evaluator/llm_correctness_judge_llama_3_1_8B_instruct.yaml diff --git a/configs/correctness_evaluator/llm_correctness_judge_llama_3_3_70B_instruct.yaml b/abstentionbench/configs/correctness_evaluator/llm_correctness_judge_llama_3_3_70B_instruct.yaml similarity index 100% rename from configs/correctness_evaluator/llm_correctness_judge_llama_3_3_70B_instruct.yaml rename to abstentionbench/configs/correctness_evaluator/llm_correctness_judge_llama_3_3_70B_instruct.yaml diff --git a/configs/dataset/alcuna.yaml b/abstentionbench/configs/dataset/alcuna.yaml similarity index 100% rename from configs/dataset/alcuna.yaml rename to abstentionbench/configs/dataset/alcuna.yaml diff --git a/configs/dataset/bbq.yaml b/abstentionbench/configs/dataset/bbq.yaml similarity index 100% rename from configs/dataset/bbq.yaml rename to abstentionbench/configs/dataset/bbq.yaml diff --git a/configs/dataset/big_bench_disambiguate.yaml b/abstentionbench/configs/dataset/big_bench_disambiguate.yaml similarity index 100% rename from configs/dataset/big_bench_disambiguate.yaml rename to abstentionbench/configs/dataset/big_bench_disambiguate.yaml diff --git a/configs/dataset/big_bench_known_unknowns.yaml b/abstentionbench/configs/dataset/big_bench_known_unknowns.yaml similarity index 100% rename from configs/dataset/big_bench_known_unknowns.yaml rename to abstentionbench/configs/dataset/big_bench_known_unknowns.yaml diff --git a/configs/dataset/coconot.yaml b/abstentionbench/configs/dataset/coconot.yaml similarity index 100% rename from configs/dataset/coconot.yaml rename to abstentionbench/configs/dataset/coconot.yaml diff --git a/configs/dataset/dummy.yaml b/abstentionbench/configs/dataset/dummy.yaml similarity index 100% rename from configs/dataset/dummy.yaml rename to abstentionbench/configs/dataset/dummy.yaml diff --git a/configs/dataset/falseqa.yaml b/abstentionbench/configs/dataset/falseqa.yaml similarity index 100% rename from configs/dataset/falseqa.yaml rename to abstentionbench/configs/dataset/falseqa.yaml diff --git a/configs/dataset/freshqa.yaml b/abstentionbench/configs/dataset/freshqa.yaml similarity index 100% rename from configs/dataset/freshqa.yaml rename to abstentionbench/configs/dataset/freshqa.yaml diff --git a/configs/dataset/gpqa.yaml b/abstentionbench/configs/dataset/gpqa.yaml similarity index 100% rename from configs/dataset/gpqa.yaml rename to abstentionbench/configs/dataset/gpqa.yaml diff --git a/configs/dataset/gsm8k.yaml b/abstentionbench/configs/dataset/gsm8k.yaml similarity index 100% rename from configs/dataset/gsm8k.yaml rename to abstentionbench/configs/dataset/gsm8k.yaml diff --git a/configs/dataset/kuq.yaml b/abstentionbench/configs/dataset/kuq.yaml similarity index 100% rename from configs/dataset/kuq.yaml rename to abstentionbench/configs/dataset/kuq.yaml diff --git a/configs/dataset/mediq.yaml b/abstentionbench/configs/dataset/mediq.yaml similarity index 100% rename from configs/dataset/mediq.yaml rename to abstentionbench/configs/dataset/mediq.yaml diff --git a/configs/dataset/mmlu_history.yaml b/abstentionbench/configs/dataset/mmlu_history.yaml similarity index 100% rename from configs/dataset/mmlu_history.yaml rename to abstentionbench/configs/dataset/mmlu_history.yaml diff --git a/configs/dataset/mmlu_math.yaml b/abstentionbench/configs/dataset/mmlu_math.yaml similarity index 100% rename from configs/dataset/mmlu_math.yaml rename to abstentionbench/configs/dataset/mmlu_math.yaml diff --git a/configs/dataset/moralchoice.yaml b/abstentionbench/configs/dataset/moralchoice.yaml similarity index 100% rename from configs/dataset/moralchoice.yaml rename to abstentionbench/configs/dataset/moralchoice.yaml diff --git a/configs/dataset/musique.yaml b/abstentionbench/configs/dataset/musique.yaml similarity index 100% rename from configs/dataset/musique.yaml rename to abstentionbench/configs/dataset/musique.yaml diff --git a/configs/dataset/qaqa.yaml b/abstentionbench/configs/dataset/qaqa.yaml similarity index 100% rename from configs/dataset/qaqa.yaml rename to abstentionbench/configs/dataset/qaqa.yaml diff --git a/configs/dataset/qasper.yaml b/abstentionbench/configs/dataset/qasper.yaml similarity index 100% rename from configs/dataset/qasper.yaml rename to abstentionbench/configs/dataset/qasper.yaml diff --git a/configs/dataset/self_aware.yaml b/abstentionbench/configs/dataset/self_aware.yaml similarity index 100% rename from configs/dataset/self_aware.yaml rename to abstentionbench/configs/dataset/self_aware.yaml diff --git a/configs/dataset/situated_qa.yaml b/abstentionbench/configs/dataset/situated_qa.yaml similarity index 100% rename from configs/dataset/situated_qa.yaml rename to abstentionbench/configs/dataset/situated_qa.yaml diff --git a/configs/dataset/squad2.yaml b/abstentionbench/configs/dataset/squad2.yaml similarity index 100% rename from configs/dataset/squad2.yaml rename to abstentionbench/configs/dataset/squad2.yaml diff --git a/configs/dataset/umwp.yaml b/abstentionbench/configs/dataset/umwp.yaml similarity index 100% rename from configs/dataset/umwp.yaml rename to abstentionbench/configs/dataset/umwp.yaml diff --git a/configs/dataset/worldsense.yaml b/abstentionbench/configs/dataset/worldsense.yaml similarity index 100% rename from configs/dataset/worldsense.yaml rename to abstentionbench/configs/dataset/worldsense.yaml diff --git a/configs/default_pipeline.yaml b/abstentionbench/configs/default_pipeline.yaml similarity index 100% rename from configs/default_pipeline.yaml rename to abstentionbench/configs/default_pipeline.yaml diff --git a/configs/mode/cluster.yaml b/abstentionbench/configs/mode/cluster.yaml similarity index 100% rename from configs/mode/cluster.yaml rename to abstentionbench/configs/mode/cluster.yaml diff --git a/configs/mode/local.yaml b/abstentionbench/configs/mode/local.yaml similarity index 100% rename from configs/mode/local.yaml rename to abstentionbench/configs/mode/local.yaml diff --git a/configs/model/deepseek_r1_distill_llama_70B.yaml b/abstentionbench/configs/model/deepseek_r1_distill_llama_70B.yaml similarity index 100% rename from configs/model/deepseek_r1_distill_llama_70B.yaml rename to abstentionbench/configs/model/deepseek_r1_distill_llama_70B.yaml diff --git a/configs/model/dummy.yaml b/abstentionbench/configs/model/dummy.yaml similarity index 100% rename from configs/model/dummy.yaml rename to abstentionbench/configs/model/dummy.yaml diff --git a/configs/model/gemini_15_pro.yaml b/abstentionbench/configs/model/gemini_15_pro.yaml similarity index 100% rename from configs/model/gemini_15_pro.yaml rename to abstentionbench/configs/model/gemini_15_pro.yaml diff --git a/configs/model/gpt4o.yaml b/abstentionbench/configs/model/gpt4o.yaml similarity index 100% rename from configs/model/gpt4o.yaml rename to abstentionbench/configs/model/gpt4o.yaml diff --git a/configs/model/llama_3_1_405B_instruct.yaml b/abstentionbench/configs/model/llama_3_1_405B_instruct.yaml similarity index 100% rename from configs/model/llama_3_1_405B_instruct.yaml rename to abstentionbench/configs/model/llama_3_1_405B_instruct.yaml diff --git a/configs/model/llama_3_1_70B_base.yaml b/abstentionbench/configs/model/llama_3_1_70B_base.yaml similarity index 100% rename from configs/model/llama_3_1_70B_base.yaml rename to abstentionbench/configs/model/llama_3_1_70B_base.yaml diff --git a/configs/model/llama_3_1_70B_instruct.yaml b/abstentionbench/configs/model/llama_3_1_70B_instruct.yaml similarity index 100% rename from configs/model/llama_3_1_70B_instruct.yaml rename to abstentionbench/configs/model/llama_3_1_70B_instruct.yaml diff --git a/configs/model/llama_3_1_70B_tulu_3_dpo.yaml b/abstentionbench/configs/model/llama_3_1_70B_tulu_3_dpo.yaml similarity index 100% rename from configs/model/llama_3_1_70B_tulu_3_dpo.yaml rename to abstentionbench/configs/model/llama_3_1_70B_tulu_3_dpo.yaml diff --git a/configs/model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml b/abstentionbench/configs/model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml similarity index 100% rename from configs/model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml rename to abstentionbench/configs/model/llama_3_1_70B_tulu_3_ppo_rlvf.yaml diff --git a/configs/model/llama_3_1_70B_tulu_3_sft.yaml b/abstentionbench/configs/model/llama_3_1_70B_tulu_3_sft.yaml similarity index 100% rename from configs/model/llama_3_1_70B_tulu_3_sft.yaml rename to abstentionbench/configs/model/llama_3_1_70B_tulu_3_sft.yaml diff --git a/configs/model/llama_3_1_8B_base.yaml b/abstentionbench/configs/model/llama_3_1_8B_base.yaml similarity index 100% rename from configs/model/llama_3_1_8B_base.yaml rename to abstentionbench/configs/model/llama_3_1_8B_base.yaml diff --git a/configs/model/llama_3_1_8B_instruct.yaml b/abstentionbench/configs/model/llama_3_1_8B_instruct.yaml similarity index 100% rename from configs/model/llama_3_1_8B_instruct.yaml rename to abstentionbench/configs/model/llama_3_1_8B_instruct.yaml diff --git a/configs/model/llama_3_1_8B_tulu_3_dpo.yaml b/abstentionbench/configs/model/llama_3_1_8B_tulu_3_dpo.yaml similarity index 100% rename from configs/model/llama_3_1_8B_tulu_3_dpo.yaml rename to abstentionbench/configs/model/llama_3_1_8B_tulu_3_dpo.yaml diff --git a/configs/model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml b/abstentionbench/configs/model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml similarity index 100% rename from configs/model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml rename to abstentionbench/configs/model/llama_3_1_8B_tulu_3_ppo_rlvf.yaml diff --git a/configs/model/llama_3_1_8B_tulu_3_sft.yaml b/abstentionbench/configs/model/llama_3_1_8B_tulu_3_sft.yaml similarity index 100% rename from configs/model/llama_3_1_8B_tulu_3_sft.yaml rename to abstentionbench/configs/model/llama_3_1_8B_tulu_3_sft.yaml diff --git a/configs/model/llama_3_3_70B_instruct.yaml b/abstentionbench/configs/model/llama_3_3_70B_instruct.yaml similarity index 100% rename from configs/model/llama_3_3_70B_instruct.yaml rename to abstentionbench/configs/model/llama_3_3_70B_instruct.yaml diff --git a/configs/model/mistral_7B_instruct_v0_3.yaml b/abstentionbench/configs/model/mistral_7B_instruct_v0_3.yaml similarity index 100% rename from configs/model/mistral_7B_instruct_v0_3.yaml rename to abstentionbench/configs/model/mistral_7B_instruct_v0_3.yaml diff --git a/configs/model/o1.yaml b/abstentionbench/configs/model/o1.yaml similarity index 100% rename from configs/model/o1.yaml rename to abstentionbench/configs/model/o1.yaml diff --git a/configs/model/olmo_7B_0724_instruct.yaml b/abstentionbench/configs/model/olmo_7B_0724_instruct.yaml similarity index 100% rename from configs/model/olmo_7B_0724_instruct.yaml rename to abstentionbench/configs/model/olmo_7B_0724_instruct.yaml diff --git a/configs/model/qwen2_5_32B_instruct.yaml b/abstentionbench/configs/model/qwen2_5_32B_instruct.yaml similarity index 100% rename from configs/model/qwen2_5_32B_instruct.yaml rename to abstentionbench/configs/model/qwen2_5_32B_instruct.yaml diff --git a/configs/model/s1_1_32B.yaml b/abstentionbench/configs/model/s1_1_32B.yaml similarity index 100% rename from configs/model/s1_1_32B.yaml rename to abstentionbench/configs/model/s1_1_32B.yaml diff --git a/configs/model/tiny_llama_chat.yaml b/abstentionbench/configs/model/tiny_llama_chat.yaml similarity index 100% rename from configs/model/tiny_llama_chat.yaml rename to abstentionbench/configs/model/tiny_llama_chat.yaml diff --git a/data/UMWP_indices_answerable.json b/abstentionbench/data/UMWP_indices_answerable.json similarity index 100% rename from data/UMWP_indices_answerable.json rename to abstentionbench/data/UMWP_indices_answerable.json diff --git a/abstentionbench/data/__init__.py b/abstentionbench/data/__init__.py new file mode 100644 index 0000000..1611796 --- /dev/null +++ b/abstentionbench/data/__init__.py @@ -0,0 +1,35 @@ +""" +Data files and utilities for AbstentionBench datasets. +""" +import os +from pathlib import Path +try: + from importlib import resources +except ImportError: + # Python < 3.9 fallback + import importlib_resources as resources + + +def get_data_dir(subdir=None): + """Get the path to the data directory, creating it if it doesn't exist. + + Args: + subdir: Optional subdirectory within the data directory + + Returns: + Path to the data directory or subdirectory + """ + try: + # When installed as a package, use the installed data directory + with resources.path(__name__, '.') as data_path: + base_dir = Path(data_path) + except (ImportError, AttributeError, FileNotFoundError): + # Fallback for development or when resources don't work + base_dir = Path(__file__).parent + + if subdir: + full_path = base_dir / subdir + full_path.mkdir(parents=True, exist_ok=True) + return str(full_path) + else: + return str(base_dir) \ No newline at end of file diff --git a/data/fast-subset-indices.json b/abstentionbench/data/fast-subset-indices.json similarity index 100% rename from data/fast-subset-indices.json rename to abstentionbench/data/fast-subset-indices.json diff --git a/data/kuq/new-category-mapping.csv b/abstentionbench/data/kuq/new-category-mapping.csv similarity index 100% rename from data/kuq/new-category-mapping.csv rename to abstentionbench/data/kuq/new-category-mapping.csv diff --git a/data/subsampling-indices.json b/abstentionbench/data/subsampling-indices.json similarity index 100% rename from data/subsampling-indices.json rename to abstentionbench/data/subsampling-indices.json diff --git a/recipe/__init__.py b/abstentionbench/recipe/__init__.py similarity index 100% rename from recipe/__init__.py rename to abstentionbench/recipe/__init__.py diff --git a/recipe/abstention.py b/abstentionbench/recipe/abstention.py similarity index 97% rename from recipe/abstention.py rename to abstentionbench/recipe/abstention.py index 825d614..6f5d125 100644 --- a/recipe/abstention.py +++ b/abstentionbench/recipe/abstention.py @@ -14,7 +14,7 @@ from pydantic import BaseModel -from recipe.inference import LoadSaveDataMixin, RawResponse, RawResponses +from .inference import LoadSaveDataMixin, RawResponse, RawResponses class Response(RawResponse): diff --git a/recipe/abstention_datasets/abstract_abstention_dataset.py b/abstentionbench/recipe/abstention_datasets/abstract_abstention_dataset.py similarity index 100% rename from recipe/abstention_datasets/abstract_abstention_dataset.py rename to abstentionbench/recipe/abstention_datasets/abstract_abstention_dataset.py diff --git a/recipe/abstention_datasets/alcuna.py b/abstentionbench/recipe/abstention_datasets/alcuna.py similarity index 97% rename from recipe/abstention_datasets/alcuna.py rename to abstentionbench/recipe/abstention_datasets/alcuna.py index 8900104..2cc238f 100644 --- a/recipe/abstention_datasets/alcuna.py +++ b/abstentionbench/recipe/abstention_datasets/alcuna.py @@ -17,10 +17,11 @@ logger = logging.getLogger(__name__) -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir class ALCUNADataset(AbstentionDataset): @@ -37,12 +38,12 @@ class ALCUNADataset(AbstentionDataset): def __init__( self, - data_dir="data/alcuna", + data_dir=None, max_num_samples=None, ): super().__init__() - self.data_dir = data_dir + self.data_dir = data_dir or get_data_dir("alcuna") self.max_num_samples = max_num_samples os.makedirs(self.data_dir, exist_ok=True) diff --git a/recipe/abstention_datasets/bbq.py b/abstentionbench/recipe/abstention_datasets/bbq.py similarity index 90% rename from recipe/abstention_datasets/bbq.py rename to abstentionbench/recipe/abstention_datasets/bbq.py index 27eabbd..2479bee 100644 --- a/recipe/abstention_datasets/bbq.py +++ b/abstentionbench/recipe/abstention_datasets/bbq.py @@ -6,8 +6,9 @@ """ import requests import datasets -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt +from ...data import get_data_dir from loguru import logger import jsonlines @@ -30,16 +31,17 @@ class BBQDataset(AbstentionDataset): def __init__( self, - data_dir="data/bbq", + data_dir=None, data_url_base="https://raw.githubusercontent.com/nyu-mll/BBQ/refs/heads/main/data/", max_num_samples=None, ): super().__init__() + self.data_dir = data_dir or get_data_dir("bbq") self.max_num_samples = max_num_samples try: - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: # If we haven't already saved the dataset, fetch and parse the dataset files one-by-one all_subdatasets = [] diff --git a/recipe/abstention_datasets/big_bench.py b/abstentionbench/recipe/abstention_datasets/big_bench.py similarity index 94% rename from recipe/abstention_datasets/big_bench.py rename to abstentionbench/recipe/abstention_datasets/big_bench.py index c695368..ec49d47 100644 --- a/recipe/abstention_datasets/big_bench.py +++ b/abstentionbench/recipe/abstention_datasets/big_bench.py @@ -14,10 +14,11 @@ import datasets import wget -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir logger = logging.getLogger(__name__) @@ -70,14 +71,14 @@ class BigBenchDisambiguateDataset(AbstentionDataset): def __init__( self, - data_dir="data/big_bench_disambiguate", + data_dir=None, data_url="https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/refs/heads/main/bbh/disambiguation_qa.json", max_num_samples=None, ): super().__init__() - self.data_dir = data_dir - self.data_path = Path(data_dir) / "disambiguation_qa.json" + self.data_dir = data_dir or get_data_dir("big_bench_disambiguate") + self.data_path = Path(self.data_dir) / "disambiguation_qa.json" self.data_url = data_url self.max_num_samples = max_num_samples self.dataset = self.load_dataset() diff --git a/recipe/abstention_datasets/coconot.py b/abstentionbench/recipe/abstention_datasets/coconot.py similarity index 98% rename from recipe/abstention_datasets/coconot.py rename to abstentionbench/recipe/abstention_datasets/coconot.py index 85220db..c01ccf5 100644 --- a/recipe/abstention_datasets/coconot.py +++ b/abstentionbench/recipe/abstention_datasets/coconot.py @@ -8,7 +8,7 @@ import datasets -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) diff --git a/recipe/abstention_datasets/false_qa.py b/abstentionbench/recipe/abstention_datasets/false_qa.py similarity index 84% rename from recipe/abstention_datasets/false_qa.py rename to abstentionbench/recipe/abstention_datasets/false_qa.py index 1939512..7248fe2 100644 --- a/recipe/abstention_datasets/false_qa.py +++ b/abstentionbench/recipe/abstention_datasets/false_qa.py @@ -14,25 +14,27 @@ import pandas as pd from loguru import logger -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt +from ...data import get_data_dir class FalseQADataset(AbstentionDataset): def __init__( self, - data_dir=Path("data/falseqa"), + data_dir=None, data_url="https://raw.githubusercontent.com/thunlp/FalseQA/refs/heads/main/dataset/test.csv", max_num_samples=None, ): super().__init__() + self.data_dir = data_dir or get_data_dir("falseqa") self.max_num_samples = max_num_samples try: # Load the formatted dataset from disk - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: logger.info(f"Downloading {data_url}") @@ -47,7 +49,7 @@ def __init__( self.dataset = datasets.Dataset.from_pandas(data_df) # Save to disk - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) def __len__(self) -> int: return self.max_num_samples or len(self.dataset) diff --git a/recipe/abstention_datasets/freshqa.py b/abstentionbench/recipe/abstention_datasets/freshqa.py similarity index 97% rename from recipe/abstention_datasets/freshqa.py rename to abstentionbench/recipe/abstention_datasets/freshqa.py index e10b587..04c6efc 100644 --- a/recipe/abstention_datasets/freshqa.py +++ b/abstentionbench/recipe/abstention_datasets/freshqa.py @@ -6,7 +6,7 @@ """ import pandas as pd -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset, Prompt +from .abstract_abstention_dataset import AbstentionDataset, Prompt class FreshQADataset(AbstentionDataset): diff --git a/recipe/abstention_datasets/gpqa.py b/abstentionbench/recipe/abstention_datasets/gpqa.py similarity index 98% rename from recipe/abstention_datasets/gpqa.py rename to abstentionbench/recipe/abstention_datasets/gpqa.py index e923159..445fb45 100644 --- a/recipe/abstention_datasets/gpqa.py +++ b/abstentionbench/recipe/abstention_datasets/gpqa.py @@ -10,7 +10,7 @@ from datasets import Dataset, concatenate_datasets, load_dataset -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) diff --git a/recipe/abstention_datasets/gsm8k.py b/abstentionbench/recipe/abstention_datasets/gsm8k.py similarity index 98% rename from recipe/abstention_datasets/gsm8k.py rename to abstentionbench/recipe/abstention_datasets/gsm8k.py index 3071d63..3810981 100644 --- a/recipe/abstention_datasets/gsm8k.py +++ b/abstentionbench/recipe/abstention_datasets/gsm8k.py @@ -8,7 +8,7 @@ from datasets import Dataset, concatenate_datasets, load_dataset -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) diff --git a/recipe/abstention_datasets/kuq.py b/abstentionbench/recipe/abstention_datasets/kuq.py similarity index 97% rename from recipe/abstention_datasets/kuq.py rename to abstentionbench/recipe/abstention_datasets/kuq.py index 7510021..c4550ed 100644 --- a/recipe/abstention_datasets/kuq.py +++ b/abstentionbench/recipe/abstention_datasets/kuq.py @@ -10,7 +10,7 @@ import pandas as pd from hydra.utils import to_absolute_path -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) diff --git a/recipe/abstention_datasets/mediq.py b/abstentionbench/recipe/abstention_datasets/mediq.py similarity index 88% rename from recipe/abstention_datasets/mediq.py rename to abstentionbench/recipe/abstention_datasets/mediq.py index 992bb31..4cf6652 100644 --- a/recipe/abstention_datasets/mediq.py +++ b/abstentionbench/recipe/abstention_datasets/mediq.py @@ -11,8 +11,9 @@ import jsonlines from loguru import logger -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt +from ...data import get_data_dir class _MediQSubDataset( @@ -22,14 +23,15 @@ class _MediQSubDataset( def __init__( self, - data_dir="data/mediq/icraftmd", + data_dir=None, data_url="https://raw.githubusercontent.com/stellalisy/mediQ/refs/heads/main/data/all_craft_md.jsonl", exclude_sample_ids=None, ): super().__init__() + self.data_dir = data_dir or get_data_dir("mediq/icraftmd") try: - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: # If we haven't already saved the dataset, fetch and parse the dataset files one-by-one @@ -105,20 +107,21 @@ class MediQDataset(AbstentionDataset): def __init__( self, - data_dir="data/mediq", + data_dir=None, icraftmd_url="https://raw.githubusercontent.com/stellalisy/mediQ/refs/heads/main/data/all_craft_md.jsonl", imedqa_url="https://raw.githubusercontent.com/stellalisy/mediQ/refs/heads/main/data/all_dev_good.jsonl", max_num_samples=None, ): super().__init__() + self.data_dir = data_dir or get_data_dir("mediq") self.max_num_samples = max_num_samples - self.icraftmd = _MediQSubDataset(Path(data_dir) / "icrafmd", icraftmd_url) + self.icraftmd = _MediQSubDataset(Path(self.data_dir) / "icrafmd", icraftmd_url) # Exclude 3 iMedQA samples which don't have a context self.imedqa = _MediQSubDataset( - Path(data_dir) / "imedqa", imedqa_url, exclude_sample_ids={224, 298, 779} + Path(self.data_dir) / "imedqa", imedqa_url, exclude_sample_ids={224, 298, 779} ) def __len__(self): diff --git a/recipe/abstention_datasets/mmlu.py b/abstentionbench/recipe/abstention_datasets/mmlu.py similarity index 98% rename from recipe/abstention_datasets/mmlu.py rename to abstentionbench/recipe/abstention_datasets/mmlu.py index 675d2f4..9e9fa56 100644 --- a/recipe/abstention_datasets/mmlu.py +++ b/abstentionbench/recipe/abstention_datasets/mmlu.py @@ -8,7 +8,7 @@ from datasets import Dataset, concatenate_datasets, load_dataset -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) diff --git a/recipe/abstention_datasets/moralchoice.py b/abstentionbench/recipe/abstention_datasets/moralchoice.py similarity index 96% rename from recipe/abstention_datasets/moralchoice.py rename to abstentionbench/recipe/abstention_datasets/moralchoice.py index 0765193..79ca4d3 100644 --- a/recipe/abstention_datasets/moralchoice.py +++ b/abstentionbench/recipe/abstention_datasets/moralchoice.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) from typing import Literal -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset, Prompt +from .abstract_abstention_dataset import AbstentionDataset, Prompt class MoralChoiceDataset(AbstentionDataset): diff --git a/recipe/abstention_datasets/musique.py b/abstentionbench/recipe/abstention_datasets/musique.py similarity index 92% rename from recipe/abstention_datasets/musique.py rename to abstentionbench/recipe/abstention_datasets/musique.py index f6b7341..20eecbb 100644 --- a/recipe/abstention_datasets/musique.py +++ b/abstentionbench/recipe/abstention_datasets/musique.py @@ -4,7 +4,8 @@ This source code is licensed under the license found in the LICENSE file in the root directory of this source tree. """ -from recipe.abstention_datasets.nq_dataset import NQDataset +from .nq_dataset import NQDataset +from ...data import get_data_dir class MusiqueDataset(NQDataset): diff --git a/recipe/abstention_datasets/nq_dataset.py b/abstentionbench/recipe/abstention_datasets/nq_dataset.py similarity index 95% rename from recipe/abstention_datasets/nq_dataset.py rename to abstentionbench/recipe/abstention_datasets/nq_dataset.py index 2efd394..3d1693c 100644 --- a/recipe/abstention_datasets/nq_dataset.py +++ b/abstentionbench/recipe/abstention_datasets/nq_dataset.py @@ -15,10 +15,11 @@ logger = logging.getLogger(__name__) -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir class NQDataset(AbstentionDataset): @@ -32,13 +33,13 @@ class NQDataset(AbstentionDataset): def __init__( self, - data_dir='data/NQ', + data_dir=None, file_name="NQ/test.json", max_num_samples=None, ): super().__init__() - self.data_dir = data_dir + self.data_dir = data_dir or get_data_dir("NQ") self.file_name = file_name self.max_num_samples = max_num_samples diff --git a/recipe/abstention_datasets/qaqa.py b/abstentionbench/recipe/abstention_datasets/qaqa.py similarity index 89% rename from recipe/abstention_datasets/qaqa.py rename to abstentionbench/recipe/abstention_datasets/qaqa.py index 3211bdc..fb41285 100644 --- a/recipe/abstention_datasets/qaqa.py +++ b/abstentionbench/recipe/abstention_datasets/qaqa.py @@ -5,7 +5,8 @@ LICENSE file in the root directory of this source tree. """ import pandas as pd -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset, Prompt +from .abstract_abstention_dataset import AbstentionDataset, Prompt +from ...data import get_data_dir import gdown import tarfile import os @@ -20,13 +21,13 @@ class QAQADataset(AbstentionDataset): def __init__( self, - data_dir="data/qaqa", + data_dir=None, max_num_samples=None, ): super().__init__() - self.data_dir = data_dir + self.data_dir = data_dir or get_data_dir("qaqa") os.makedirs(self.data_dir, exist_ok=True) - self.test_file_path = Path(data_dir) / "QAQA_evaluation_set_Dec2022.csv" + self.test_file_path = Path(self.data_dir) / "QAQA_evaluation_set_Dec2022.csv" if not os.path.exists(self.test_file_path): self._download_data() diff --git a/recipe/abstention_datasets/qasper.py b/abstentionbench/recipe/abstention_datasets/qasper.py similarity index 94% rename from recipe/abstention_datasets/qasper.py rename to abstentionbench/recipe/abstention_datasets/qasper.py index 8f67fc7..e273b74 100644 --- a/recipe/abstention_datasets/qasper.py +++ b/abstentionbench/recipe/abstention_datasets/qasper.py @@ -11,10 +11,11 @@ logger = logging.getLogger(__name__) -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir class QASPERDataset(AbstentionDataset): @@ -26,14 +27,15 @@ class QASPERDataset(AbstentionDataset): _PREPROMPT = "Respond to the question about the following scientific paper." _TEMPLATE = "{preprompt}\n\nPaper title: {title}\n\nPaper text:\n{full_text}\n\nQuestion: {question}" - def __init__(self, data_dir="data/qasper", max_num_samples=None): + def __init__(self, data_dir=None, max_num_samples=None): super().__init__() + self.data_dir = data_dir or get_data_dir("qasper") self.max_num_samples = max_num_samples try: # Load the formatted dataset from disk - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: logger.info("Fetching and processing allenai/qasper") dataset = datasets.load_dataset("allenai/qasper")["test"] diff --git a/recipe/abstention_datasets/self_aware.py b/abstentionbench/recipe/abstention_datasets/self_aware.py similarity index 83% rename from recipe/abstention_datasets/self_aware.py rename to abstentionbench/recipe/abstention_datasets/self_aware.py index 47272f4..07f2819 100644 --- a/recipe/abstention_datasets/self_aware.py +++ b/abstentionbench/recipe/abstention_datasets/self_aware.py @@ -6,24 +6,26 @@ """ import requests import datasets -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt +from ...data import get_data_dir class SelfAwareDataset(AbstentionDataset): def __init__( self, - data_dir="data/selfaware", + data_dir=None, data_url="https://raw.githubusercontent.com/yinzhangyue/SelfAware/refs/heads/main/data/SelfAware.json", max_num_samples=None, ): super().__init__() + self.data_dir = data_dir or get_data_dir("selfaware") self.max_num_samples = max_num_samples try: - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: # If we haven't already saved the dataset, fetch and parse the original JSON response = requests.get(data_url) diff --git a/recipe/abstention_datasets/situated_qa.py b/abstentionbench/recipe/abstention_datasets/situated_qa.py similarity index 92% rename from recipe/abstention_datasets/situated_qa.py rename to abstentionbench/recipe/abstention_datasets/situated_qa.py index cef4644..d972974 100644 --- a/recipe/abstention_datasets/situated_qa.py +++ b/abstentionbench/recipe/abstention_datasets/situated_qa.py @@ -6,8 +6,8 @@ """ import requests import datasets -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt class SituatedQAGeoDataset(AbstentionDataset): diff --git a/recipe/abstention_datasets/squad.py b/abstentionbench/recipe/abstention_datasets/squad.py similarity index 93% rename from recipe/abstention_datasets/squad.py rename to abstentionbench/recipe/abstention_datasets/squad.py index aedfa5d..4b215b9 100644 --- a/recipe/abstention_datasets/squad.py +++ b/abstentionbench/recipe/abstention_datasets/squad.py @@ -5,8 +5,8 @@ LICENSE file in the root directory of this source tree. """ import datasets -from recipe.abstention_datasets.abstract_abstention_dataset import AbstentionDataset -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from .abstract_abstention_dataset import AbstentionDataset +from .abstract_abstention_dataset import Prompt from pathlib import Path import pandas as pd import urllib.request diff --git a/recipe/abstention_datasets/umwp.py b/abstentionbench/recipe/abstention_datasets/umwp.py similarity index 95% rename from recipe/abstention_datasets/umwp.py rename to abstentionbench/recipe/abstention_datasets/umwp.py index 8a06b65..6d3e814 100644 --- a/recipe/abstention_datasets/umwp.py +++ b/abstentionbench/recipe/abstention_datasets/umwp.py @@ -12,10 +12,11 @@ import requests from hydra.utils import to_absolute_path -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir class UMWP(AbstentionDataset): @@ -29,9 +30,9 @@ class UMWP(AbstentionDataset): 5: "Question missing", } - def __init__(self, data_dir="data/umwp", max_num_samples=None): + def __init__(self, data_dir=None, max_num_samples=None): super().__init__() - self.data_dir = data_dir + self.data_dir = data_dir or get_data_dir("umwp") self.data_file = "UMWP.jsonl" if not os.path.exists(Path(self.data_dir) / self.data_file): diff --git a/recipe/abstention_datasets/world_sense.py b/abstentionbench/recipe/abstention_datasets/world_sense.py similarity index 94% rename from recipe/abstention_datasets/world_sense.py rename to abstentionbench/recipe/abstention_datasets/world_sense.py index d334dc1..1dfd285 100644 --- a/recipe/abstention_datasets/world_sense.py +++ b/abstentionbench/recipe/abstention_datasets/world_sense.py @@ -12,10 +12,11 @@ import pandas as pd import wget -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) +from ...data import get_data_dir logger = logging.getLogger(__name__) @@ -28,12 +29,12 @@ class WorldSenseDataset(AbstentionDataset): def __init__( self, - data_dir='data/world_sense', + data_dir=None, raw_data_url="https://github.com/facebookresearch/worldsense/raw/refs/heads/main/data/worldsense/training_set/trials_10k.jsonl.bz2", max_num_samples=None, ): super().__init__() - self.data_dir = data_dir + self.data_dir = data_dir or get_data_dir("world_sense") self.raw_data_url = raw_data_url self.max_num_samples = max_num_samples diff --git a/recipe/abstention_keywords.py b/abstentionbench/recipe/abstention_keywords.py similarity index 100% rename from recipe/abstention_keywords.py rename to abstentionbench/recipe/abstention_keywords.py diff --git a/recipe/evaluation.py b/abstentionbench/recipe/evaluation.py similarity index 97% rename from recipe/evaluation.py rename to abstentionbench/recipe/evaluation.py index a0af3e8..4d0f3f5 100644 --- a/recipe/evaluation.py +++ b/abstentionbench/recipe/evaluation.py @@ -15,14 +15,14 @@ logger = logging.getLogger(__name__) -from recipe.abstention import Response, Responses -from recipe.abstention_keywords import ABSTENTION_KEYWORDS -from recipe.evaluation_judge_prompts import ( +from .abstention import Response, Responses +from .abstention_keywords import ABSTENTION_KEYWORDS +from .evaluation_judge_prompts import ( LLM_JUDGE_ABSTENTION_COCONOT_STYLE_PROMPT, LLM_JUDGE_CORRECTNESS_PROMPT, LLM_JUDGE_MATH_CORRECTNESS_PROMPT, ) -from recipe.models import InferenceModel +from .models import InferenceModel class AbstentionEvaluator(ABC): diff --git a/recipe/evaluation_judge_prompts.py b/abstentionbench/recipe/evaluation_judge_prompts.py similarity index 100% rename from recipe/evaluation_judge_prompts.py rename to abstentionbench/recipe/evaluation_judge_prompts.py diff --git a/recipe/inference.py b/abstentionbench/recipe/inference.py similarity index 95% rename from recipe/inference.py rename to abstentionbench/recipe/inference.py index 1aaffcb..6a274fb 100644 --- a/recipe/inference.py +++ b/abstentionbench/recipe/inference.py @@ -16,12 +16,12 @@ from pydantic import BaseModel from torch.utils.data import DataLoader, SubsetRandomSampler -from recipe.abstention_datasets.abstract_abstention_dataset import ( +from .abstention_datasets.abstract_abstention_dataset import ( AbstentionDataset, Prompt, ) -from recipe.models import InferenceModel, VLLMReasoningChatModelBase -from recipe.utils import permissive_makedirs +from .models import InferenceModel, VLLMReasoningChatModelBase +from .utils import permissive_makedirs logger = logging.getLogger(__name__) diff --git a/recipe/job_manager.py b/abstentionbench/recipe/job_manager.py similarity index 100% rename from recipe/job_manager.py rename to abstentionbench/recipe/job_manager.py diff --git a/recipe/models.py b/abstentionbench/recipe/models.py similarity index 99% rename from recipe/models.py rename to abstentionbench/recipe/models.py index 05fe28c..efa7bc6 100644 --- a/recipe/models.py +++ b/abstentionbench/recipe/models.py @@ -23,7 +23,7 @@ from transformers import AutoTokenizer, pipeline from vllm import LLM, SamplingParams -from recipe.system_prompt import SYSTEM_PROMPT +from .system_prompt import SYSTEM_PROMPT logger = logging.getLogger(__name__) diff --git a/recipe/prompts_archive.py b/abstentionbench/recipe/prompts_archive.py similarity index 100% rename from recipe/prompts_archive.py rename to abstentionbench/recipe/prompts_archive.py diff --git a/recipe/system_prompt.py b/abstentionbench/recipe/system_prompt.py similarity index 100% rename from recipe/system_prompt.py rename to abstentionbench/recipe/system_prompt.py diff --git a/recipe/utils.py b/abstentionbench/recipe/utils.py similarity index 100% rename from recipe/utils.py rename to abstentionbench/recipe/utils.py diff --git a/main.py b/main.py index 2e1b62e..7b56b9f 100644 --- a/main.py +++ b/main.py @@ -24,9 +24,9 @@ from omegaconf import DictConfig, OmegaConf from submitit.helpers import clean_env -from recipe.abstention import Responses -from recipe.inference import InferencePipeline, RawResponses -from recipe.models import InferenceModel +from abstentionbench.recipe.abstention import Responses +from abstentionbench.recipe.inference import InferencePipeline, RawResponses +from abstentionbench.recipe.models import InferenceModel logger = logging.getLogger(__name__) @@ -400,7 +400,7 @@ def _get_indices_subset(self, dataset_name: str) -> Optional[List[int]]: @hydra.main( version_base="1.2", - config_path="configs", + config_path="pkg://abstentionbench.configs", config_name="default_pipeline.yaml", ) def main(config: DictConfig) -> None: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2a4f1b2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "abstentionbench" +version = "1.0.0" +description = "A Holistic Benchmark for LLM Abstention" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "CC-BY-NC"} +authors = [ + {name = "Meta Platforms, Inc."} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: Other/Proprietary License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +dependencies = [ + "accelerate", + "altair", + "datasets", + "fire", + "GitPython", + "gdown", + "google-generativeai", + "hydra-core", + "hydra-submitit-launcher", + "huggingface-hub", + "ipykernel", + "ipywidgets", + "jsonlines", + "kaleido", + "loguru", + "openai", + "plotly", + "pydantic", + "PyPDF2", + "pytest", + "requests", + "scikit-learn", + "submitit", + "tabulate", + "torch", + "tqdm", + "transformers", + "wget", +] + +[project.optional-dependencies] +dev = [ + "pytest", +] +gpu = [ + "vllm>=0.6.4.post1", +] + +[project.scripts] +abstention-bench = "main:main" + +[project.urls] +Homepage = "https://github.com/facebookresearch/AbstentionBench" +Repository = "https://github.com/facebookresearch/AbstentionBench" +"Bug Reports" = "https://github.com/facebookresearch/AbstentionBench/issues" + +[tool.setuptools.packages.find] +include = ["abstentionbench*"] +exclude = ["tests*"] + +[tool.setuptools.package-data] +"*" = ["*.yaml", "*.yml", "*.json", "*.csv"] \ No newline at end of file diff --git a/tests/run_llama3_instruct_inference.py b/tests/run_llama3_instruct_inference.py index 9e653c6..b8e39d8 100644 --- a/tests/run_llama3_instruct_inference.py +++ b/tests/run_llama3_instruct_inference.py @@ -6,7 +6,7 @@ """ import torch -from recipe.models import Llama3_1_8B_Instruct +from abstentionbench.recipe.models import Llama3_1_8B_Instruct torch.cuda.empty_cache() diff --git a/tests/test_abstention_detector.py b/tests/test_abstention_detector.py index a223aee..10eaf75 100644 --- a/tests/test_abstention_detector.py +++ b/tests/test_abstention_detector.py @@ -4,10 +4,10 @@ This source code is licensed under the license found in the LICENSE file in the root directory of this source tree. """ -from recipe.evaluation import ContainsAbstentionKeywordAbstentionDetector, LLMJudgeAbstentionDetector -from recipe.abstention import Response -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt -from recipe.models import Llama3_1_8B_Instruct +from abstentionbench.recipe.evaluation import ContainsAbstentionKeywordAbstentionDetector, LLMJudgeAbstentionDetector +from abstentionbench.recipe.abstention import Response +from abstentionbench.recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from abstentionbench.recipe.models import Llama3_1_8B_Instruct import pytest diff --git a/tests/test_correctness_evaluator.py b/tests/test_correctness_evaluator.py index 5db8b61..e9d8636 100644 --- a/tests/test_correctness_evaluator.py +++ b/tests/test_correctness_evaluator.py @@ -6,10 +6,10 @@ """ import pytest -from recipe.abstention import Response -from recipe.abstention_datasets.abstract_abstention_dataset import Prompt -from recipe.evaluation import LLMJudgeCorrectnessEvaluator -from recipe.models import Llama3_1_8B_Instruct +from abstentionbench.recipe.abstention import Response +from abstentionbench.recipe.abstention_datasets.abstract_abstention_dataset import Prompt +from abstentionbench.recipe.evaluation import LLMJudgeCorrectnessEvaluator +from abstentionbench.recipe.models import Llama3_1_8B_Instruct class TestLLMJudgeCorrectnessEvaluator: diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 4635713..b2fea90 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -10,35 +10,35 @@ from hydra import compose, initialize from hydra.utils import instantiate -from recipe.abstention_datasets.abstract_abstention_dataset import DummyDataset, Prompt -from recipe.abstention_datasets.alcuna import ALCUNADataset -from recipe.abstention_datasets.bbq import BBQDataset -from recipe.abstention_datasets.big_bench import ( +from abstentionbench.recipe.abstention_datasets.abstract_abstention_dataset import DummyDataset, Prompt +from abstentionbench.recipe.abstention_datasets.alcuna import ALCUNADataset +from abstentionbench.recipe.abstention_datasets.bbq import BBQDataset +from abstentionbench.recipe.abstention_datasets.big_bench import ( BigBenchDisambiguateDataset, BigBenchKnownUnknownsDataset, ) -from recipe.abstention_datasets.coconot import CoCoNotDataset -from recipe.abstention_datasets.false_qa import FalseQADataset -from recipe.abstention_datasets.freshqa import FreshQADataset -from recipe.abstention_datasets.gpqa import GPQA -from recipe.abstention_datasets.gsm8k import GSM8K -from recipe.abstention_datasets.kuq import KUQDataset -from recipe.abstention_datasets.mediq import MediQDataset -from recipe.abstention_datasets.mmlu import ( +from abstentionbench.recipe.abstention_datasets.coconot import CoCoNotDataset +from abstentionbench.recipe.abstention_datasets.false_qa import FalseQADataset +from abstentionbench.recipe.abstention_datasets.freshqa import FreshQADataset +from abstentionbench.recipe.abstention_datasets.gpqa import GPQA +from abstentionbench.recipe.abstention_datasets.gsm8k import GSM8K +from abstentionbench.recipe.abstention_datasets.kuq import KUQDataset +from abstentionbench.recipe.abstention_datasets.mediq import MediQDataset +from abstentionbench.recipe.abstention_datasets.mmlu import ( MMLUHistory, MMLUHistoryGenerator, MMLUMath, MMLUMathGenerator, ) -from recipe.abstention_datasets.moralchoice import MoralChoiceDataset -from recipe.abstention_datasets.musique import MusiqueDataset -from recipe.abstention_datasets.nq_dataset import NQDataset -from recipe.abstention_datasets.qaqa import QAQADataset -from recipe.abstention_datasets.qasper import QASPERDataset -from recipe.abstention_datasets.self_aware import SelfAwareDataset -from recipe.abstention_datasets.situated_qa import SituatedQAGeoDataset -from recipe.abstention_datasets.squad import Squad2Dataset -from recipe.abstention_datasets.world_sense import WorldSenseDataset +from abstentionbench.recipe.abstention_datasets.moralchoice import MoralChoiceDataset +from abstentionbench.recipe.abstention_datasets.musique import MusiqueDataset +from abstentionbench.recipe.abstention_datasets.nq_dataset import NQDataset +from abstentionbench.recipe.abstention_datasets.qaqa import QAQADataset +from abstentionbench.recipe.abstention_datasets.qasper import QASPERDataset +from abstentionbench.recipe.abstention_datasets.self_aware import SelfAwareDataset +from abstentionbench.recipe.abstention_datasets.situated_qa import SituatedQAGeoDataset +from abstentionbench.recipe.abstention_datasets.squad import Squad2Dataset +from abstentionbench.recipe.abstention_datasets.world_sense import WorldSenseDataset class TestDummyDataset: diff --git a/tests/test_inference.py b/tests/test_inference.py index 8b9ed09..4d39643 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -4,9 +4,9 @@ This source code is licensed under the license found in the LICENSE file in the root directory of this source tree. """ -from recipe.abstention_datasets.abstract_abstention_dataset import DummyDataset, Prompt -from recipe.inference import InferencePipeline, RawResponse, RawResponses -from recipe.models import DummyModel +from abstentionbench.recipe.abstention_datasets.abstract_abstention_dataset import DummyDataset, Prompt +from abstentionbench.recipe.inference import InferencePipeline, RawResponse, RawResponses +from abstentionbench.recipe.models import DummyModel class TestResponses: diff --git a/tests/test_job_manager.py b/tests/test_job_manager.py index aad1af9..cc20100 100644 --- a/tests/test_job_manager.py +++ b/tests/test_job_manager.py @@ -4,7 +4,7 @@ This source code is licensed under the license found in the LICENSE file in the root directory of this source tree. """ -from recipe.job_manager import JobManager +from abstentionbench.recipe.job_manager import JobManager class TestJobManager: diff --git a/tests/test_models.py b/tests/test_models.py index a3d5177..0f9725a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -6,7 +6,7 @@ """ import pytest -from recipe.models import ( +from abstentionbench.recipe.models import ( DummyModel, Gemini15ProAPI, GPT4oAPI, From 7bd91a53b16bcd131106f28707ede968ced4a1d6 Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 15:10:15 -0400 Subject: [PATCH 2/6] bump version --- abstentionbench/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/abstentionbench/__init__.py b/abstentionbench/__init__.py index 2347e19..f2189fc 100644 --- a/abstentionbench/__init__.py +++ b/abstentionbench/__init__.py @@ -4,4 +4,4 @@ This package provides tools and datasets for evaluating LLM abstention capabilities. """ -__version__ = "1.0.0" \ No newline at end of file +__version__ = "1.0.1" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2a4f1b2..deec76f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "abstentionbench" -version = "1.0.0" +version = "1.0.1" description = "A Holistic Benchmark for LLM Abstention" readme = "README.md" requires-python = ">=3.10" From 48ca60a22e68bc25989e0387d0d59101c79ea552 Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 15:18:23 -0400 Subject: [PATCH 3/6] fix config files --- abstentionbench/configs/dataset/alcuna.yaml | 2 +- abstentionbench/configs/dataset/bbq.yaml | 2 +- abstentionbench/configs/dataset/big_bench_disambiguate.yaml | 2 +- abstentionbench/configs/dataset/big_bench_known_unknowns.yaml | 2 +- abstentionbench/configs/dataset/coconot.yaml | 2 +- abstentionbench/configs/dataset/dummy.yaml | 2 +- abstentionbench/configs/dataset/falseqa.yaml | 2 +- abstentionbench/configs/dataset/freshqa.yaml | 2 +- abstentionbench/configs/dataset/gpqa.yaml | 2 +- abstentionbench/configs/dataset/gsm8k.yaml | 2 +- abstentionbench/configs/dataset/kuq.yaml | 2 +- abstentionbench/configs/dataset/mediq.yaml | 2 +- abstentionbench/configs/dataset/mmlu_history.yaml | 2 +- abstentionbench/configs/dataset/mmlu_math.yaml | 2 +- abstentionbench/configs/dataset/moralchoice.yaml | 2 +- abstentionbench/configs/dataset/musique.yaml | 2 +- abstentionbench/configs/dataset/qaqa.yaml | 2 +- abstentionbench/configs/dataset/qasper.yaml | 2 +- abstentionbench/configs/dataset/self_aware.yaml | 2 +- abstentionbench/configs/dataset/situated_qa.yaml | 2 +- abstentionbench/configs/dataset/squad2.yaml | 2 +- abstentionbench/configs/dataset/umwp.yaml | 2 +- abstentionbench/configs/dataset/worldsense.yaml | 2 +- 23 files changed, 23 insertions(+), 23 deletions(-) diff --git a/abstentionbench/configs/dataset/alcuna.yaml b/abstentionbench/configs/dataset/alcuna.yaml index a877483..3a3b1ec 100644 --- a/abstentionbench/configs/dataset/alcuna.yaml +++ b/abstentionbench/configs/dataset/alcuna.yaml @@ -3,6 +3,6 @@ dataset_name: ALCUNADataset datamodule: - _target_: recipe.abstention_datasets.alcuna.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.alcuna.${dataset_name} data_dir: ${data_dir}/alcuna max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/bbq.yaml b/abstentionbench/configs/dataset/bbq.yaml index 9e8400d..86ceb3a 100644 --- a/abstentionbench/configs/dataset/bbq.yaml +++ b/abstentionbench/configs/dataset/bbq.yaml @@ -3,7 +3,7 @@ dataset_name: BBQDataset datamodule: - _target_: recipe.abstention_datasets.bbq.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.bbq.${dataset_name} data_dir: ${data_dir}/bbq data_url_base: "https://raw.githubusercontent.com/nyu-mll/BBQ/refs/heads/main/data/" max_num_samples: null diff --git a/abstentionbench/configs/dataset/big_bench_disambiguate.yaml b/abstentionbench/configs/dataset/big_bench_disambiguate.yaml index 262a1d5..c189c02 100644 --- a/abstentionbench/configs/dataset/big_bench_disambiguate.yaml +++ b/abstentionbench/configs/dataset/big_bench_disambiguate.yaml @@ -3,7 +3,7 @@ dataset_name: BigBenchDisambiguateDataset datamodule: - _target_: recipe.abstention_datasets.big_bench.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.big_bench.${dataset_name} data_dir: ${data_dir}/big_bench_disambiguate data_url: "https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/refs/heads/main/bbh/disambiguation_qa.json" max_num_samples: null diff --git a/abstentionbench/configs/dataset/big_bench_known_unknowns.yaml b/abstentionbench/configs/dataset/big_bench_known_unknowns.yaml index e68ccd2..b88b859 100644 --- a/abstentionbench/configs/dataset/big_bench_known_unknowns.yaml +++ b/abstentionbench/configs/dataset/big_bench_known_unknowns.yaml @@ -3,5 +3,5 @@ dataset_name: BigBenchKnownUnknownsDataset datamodule: - _target_: recipe.abstention_datasets.big_bench.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.big_bench.${dataset_name} max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/coconot.yaml b/abstentionbench/configs/dataset/coconot.yaml index 21b50cb..c533de6 100644 --- a/abstentionbench/configs/dataset/coconot.yaml +++ b/abstentionbench/configs/dataset/coconot.yaml @@ -3,5 +3,5 @@ dataset_name: CoCoNotDataset datamodule: - _target_: recipe.abstention_datasets.coconot.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.coconot.${dataset_name} max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/dummy.yaml b/abstentionbench/configs/dataset/dummy.yaml index 6439476..26afc8c 100644 --- a/abstentionbench/configs/dataset/dummy.yaml +++ b/abstentionbench/configs/dataset/dummy.yaml @@ -3,5 +3,5 @@ dataset_name: DummyDataset datamodule: - _target_: recipe.abstention_datasets.abstract_abstention_dataset.DummyDataset + _target_: abstentionbench.recipe.abstention_datasets.abstract_abstention_dataset.DummyDataset max_num_samples: 100 diff --git a/abstentionbench/configs/dataset/falseqa.yaml b/abstentionbench/configs/dataset/falseqa.yaml index cf3e3ca..c5a13ce 100644 --- a/abstentionbench/configs/dataset/falseqa.yaml +++ b/abstentionbench/configs/dataset/falseqa.yaml @@ -3,7 +3,7 @@ dataset_name: FalseQADataset datamodule: - _target_: recipe.abstention_datasets.false_qa.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.false_qa.${dataset_name} data_dir: ${data_dir}/falseqa data_url: "https://raw.githubusercontent.com/thunlp/FalseQA/refs/heads/main/dataset/test.csv" max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/freshqa.yaml b/abstentionbench/configs/dataset/freshqa.yaml index 1ec8955..a627884 100644 --- a/abstentionbench/configs/dataset/freshqa.yaml +++ b/abstentionbench/configs/dataset/freshqa.yaml @@ -3,7 +3,7 @@ dataset_name: FreshQADataset datamodule: - _target_: recipe.abstention_datasets.freshqa.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.freshqa.${dataset_name} path_baseline: ${data_dir}/freshqa/FreshQA_v10282024.csv path_updated: ${data_dir}/freshqa/FreshQA_v12182024.csv max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/gpqa.yaml b/abstentionbench/configs/dataset/gpqa.yaml index 86261dd..172955c 100644 --- a/abstentionbench/configs/dataset/gpqa.yaml +++ b/abstentionbench/configs/dataset/gpqa.yaml @@ -3,6 +3,6 @@ dataset_name: GPQA datamodule: - _target_: recipe.abstention_datasets.gpqa.GPQA + _target_: abstentionbench.recipe.abstention_datasets.gpqa.GPQA subset: "gpqa_diamond" max_num_samples: null diff --git a/abstentionbench/configs/dataset/gsm8k.yaml b/abstentionbench/configs/dataset/gsm8k.yaml index dacd65f..635ebf3 100644 --- a/abstentionbench/configs/dataset/gsm8k.yaml +++ b/abstentionbench/configs/dataset/gsm8k.yaml @@ -3,6 +3,6 @@ dataset_name: GSM8K datamodule: - _target_: recipe.abstention_datasets.gsm8k.GSM8K + _target_: abstentionbench.recipe.abstention_datasets.gsm8k.GSM8K split: "test" max_num_samples: null diff --git a/abstentionbench/configs/dataset/kuq.yaml b/abstentionbench/configs/dataset/kuq.yaml index beab60d..127343d 100644 --- a/abstentionbench/configs/dataset/kuq.yaml +++ b/abstentionbench/configs/dataset/kuq.yaml @@ -3,6 +3,6 @@ dataset_name: KUQDataset datamodule: - _target_: recipe.abstention_datasets.kuq.KUQDataset + _target_: abstentionbench.recipe.abstention_datasets.kuq.KUQDataset max_num_samples: null category_map_path: data/kuq/new-category-mapping.csv \ No newline at end of file diff --git a/abstentionbench/configs/dataset/mediq.yaml b/abstentionbench/configs/dataset/mediq.yaml index 9e6c6fe..c8268fd 100644 --- a/abstentionbench/configs/dataset/mediq.yaml +++ b/abstentionbench/configs/dataset/mediq.yaml @@ -3,7 +3,7 @@ dataset_name: MediQDataset datamodule: - _target_: recipe.abstention_datasets.mediq.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.mediq.${dataset_name} data_dir: ${data_dir}/mediq icraftmd_url: "https://raw.githubusercontent.com/stellalisy/mediQ/refs/heads/main/data/all_craft_md.jsonl" imedqa_url: "https://raw.githubusercontent.com/stellalisy/mediQ/refs/heads/main/data/all_dev_good.jsonl" diff --git a/abstentionbench/configs/dataset/mmlu_history.yaml b/abstentionbench/configs/dataset/mmlu_history.yaml index 0a012ee..7093f25 100644 --- a/abstentionbench/configs/dataset/mmlu_history.yaml +++ b/abstentionbench/configs/dataset/mmlu_history.yaml @@ -3,6 +3,6 @@ dataset_name: MMLUHistory datamodule: - _target_: recipe.abstention_datasets.mmlu.MMLUHistory + _target_: abstentionbench.recipe.abstention_datasets.mmlu.MMLUHistory split: "test" max_num_samples: null diff --git a/abstentionbench/configs/dataset/mmlu_math.yaml b/abstentionbench/configs/dataset/mmlu_math.yaml index 3123f86..2c6619a 100644 --- a/abstentionbench/configs/dataset/mmlu_math.yaml +++ b/abstentionbench/configs/dataset/mmlu_math.yaml @@ -3,6 +3,6 @@ dataset_name: MMLUMath datamodule: - _target_: recipe.abstention_datasets.mmlu.MMLUMath + _target_: abstentionbench.recipe.abstention_datasets.mmlu.MMLUMath split: "test" max_num_samples: null diff --git a/abstentionbench/configs/dataset/moralchoice.yaml b/abstentionbench/configs/dataset/moralchoice.yaml index 6accd7b..84b7980 100644 --- a/abstentionbench/configs/dataset/moralchoice.yaml +++ b/abstentionbench/configs/dataset/moralchoice.yaml @@ -3,7 +3,7 @@ dataset_name: MoralChoiceDataset datamodule: - _target_: recipe.abstention_datasets.moralchoice.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.moralchoice.${dataset_name} template_type: "ab" system_instruction: True max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/musique.yaml b/abstentionbench/configs/dataset/musique.yaml index 748ef48..23c66f5 100644 --- a/abstentionbench/configs/dataset/musique.yaml +++ b/abstentionbench/configs/dataset/musique.yaml @@ -3,6 +3,6 @@ dataset_name: MusiqueDataset datamodule: - _target_: recipe.abstention_datasets.musique.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.musique.${dataset_name} data_dir: ${data_dir}/musique max_num_samples: null diff --git a/abstentionbench/configs/dataset/qaqa.yaml b/abstentionbench/configs/dataset/qaqa.yaml index bfadebc..eb320e4 100644 --- a/abstentionbench/configs/dataset/qaqa.yaml +++ b/abstentionbench/configs/dataset/qaqa.yaml @@ -3,6 +3,6 @@ dataset_name: QAQADataset datamodule: - _target_: recipe.abstention_datasets.qaqa.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.qaqa.${dataset_name} data_dir: ${data_dir}/qaqa max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/qasper.yaml b/abstentionbench/configs/dataset/qasper.yaml index fab6b6c..869c0c3 100644 --- a/abstentionbench/configs/dataset/qasper.yaml +++ b/abstentionbench/configs/dataset/qasper.yaml @@ -7,6 +7,6 @@ dataset_name: QASPERDataset # empty responses. datamodule: - _target_: recipe.abstention_datasets.qasper.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.qasper.${dataset_name} data_dir: ${data_dir}/qasper max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/self_aware.yaml b/abstentionbench/configs/dataset/self_aware.yaml index 5ab45c7..0c34dae 100644 --- a/abstentionbench/configs/dataset/self_aware.yaml +++ b/abstentionbench/configs/dataset/self_aware.yaml @@ -3,7 +3,7 @@ dataset_name: SelfAwareDataset datamodule: - _target_: recipe.abstention_datasets.self_aware.SelfAwareDataset + _target_: abstentionbench.recipe.abstention_datasets.self_aware.SelfAwareDataset data_dir: ${data_dir}/selfaware data_url: "https://raw.githubusercontent.com/yinzhangyue/SelfAware/refs/heads/main/data/SelfAware.json" max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/situated_qa.yaml b/abstentionbench/configs/dataset/situated_qa.yaml index fb42c4e..9064cb8 100644 --- a/abstentionbench/configs/dataset/situated_qa.yaml +++ b/abstentionbench/configs/dataset/situated_qa.yaml @@ -3,5 +3,5 @@ dataset_name: SituatedQAGeoDataset datamodule: - _target_: recipe.abstention_datasets.situated_qa.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.situated_qa.${dataset_name} max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/squad2.yaml b/abstentionbench/configs/dataset/squad2.yaml index 8a89d50..7d25875 100644 --- a/abstentionbench/configs/dataset/squad2.yaml +++ b/abstentionbench/configs/dataset/squad2.yaml @@ -3,5 +3,5 @@ dataset_name: Squad2Dataset datamodule: - _target_: recipe.abstention_datasets.squad.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.squad.${dataset_name} max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/umwp.yaml b/abstentionbench/configs/dataset/umwp.yaml index 426ae95..76f6481 100644 --- a/abstentionbench/configs/dataset/umwp.yaml +++ b/abstentionbench/configs/dataset/umwp.yaml @@ -3,6 +3,6 @@ dataset_name: UMWP datamodule: - _target_: recipe.abstention_datasets.umwp.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.umwp.${dataset_name} data_dir: ${data_dir}/umwp max_num_samples: null \ No newline at end of file diff --git a/abstentionbench/configs/dataset/worldsense.yaml b/abstentionbench/configs/dataset/worldsense.yaml index e26168d..aeb2f3e 100644 --- a/abstentionbench/configs/dataset/worldsense.yaml +++ b/abstentionbench/configs/dataset/worldsense.yaml @@ -3,7 +3,7 @@ dataset_name: WorldSenseDataset datamodule: - _target_: recipe.abstention_datasets.world_sense.${dataset_name} + _target_: abstentionbench.recipe.abstention_datasets.world_sense.${dataset_name} data_dir: ${data_dir}/world_sense raw_data_url: "https://github.com/facebookresearch/worldsense/raw/refs/heads/main/data/worldsense/training_set/trials_10k.jsonl.bz2" max_num_samples: null From 81f986814e8587665858870156dbc3e7d1d8e201 Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 16:33:21 -0400 Subject: [PATCH 4/6] fix umwp indices path --- abstentionbench/recipe/abstention_datasets/umwp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abstentionbench/recipe/abstention_datasets/umwp.py b/abstentionbench/recipe/abstention_datasets/umwp.py index 6d3e814..6bc1bed 100644 --- a/abstentionbench/recipe/abstention_datasets/umwp.py +++ b/abstentionbench/recipe/abstention_datasets/umwp.py @@ -71,7 +71,7 @@ def _load_and_subset_data(self): # We cap AbstentionBench datasets to random 3500 questions. # This indices files indicates subset of random 3500/2 questions # (out of 5200/2 of UMWP). - indices_path = "data/UMWP_indices_answerable.json" + indices_path = os.path.join(get_data_dir(), "UMWP_indices_answerable.json") with open(to_absolute_path(indices_path), "r") as f: indices_list = json.load(f) answerable_ind = np.array(indices_list) From 05b74951f38de5511891ceba001fe676bc0b1b1a Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 16:36:52 -0400 Subject: [PATCH 5/6] fix kuq --- abstentionbench/configs/dataset/kuq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abstentionbench/configs/dataset/kuq.yaml b/abstentionbench/configs/dataset/kuq.yaml index 127343d..e8d3302 100644 --- a/abstentionbench/configs/dataset/kuq.yaml +++ b/abstentionbench/configs/dataset/kuq.yaml @@ -5,4 +5,4 @@ dataset_name: KUQDataset datamodule: _target_: abstentionbench.recipe.abstention_datasets.kuq.KUQDataset max_num_samples: null - category_map_path: data/kuq/new-category-mapping.csv \ No newline at end of file + category_map_path: ${data_dir}/kuq/new-category-mapping.csv \ No newline at end of file From 707c03d2367d2b4504b52730d0757b92efdd3df2 Mon Sep 17 00:00:00 2001 From: jeqcho Date: Sun, 7 Sep 2025 17:37:10 -0400 Subject: [PATCH 6/6] update remaining paths and readme --- README.md | 10 +++++----- abstentionbench/__init__.py | 2 +- abstentionbench/analysis/load_results.py | 2 +- abstentionbench/configs/default_pipeline.yaml | 4 ++-- abstentionbench/recipe/abstention_datasets/alcuna.py | 8 ++++---- abstentionbench/recipe/abstention_datasets/bbq.py | 2 +- abstentionbench/recipe/abstention_datasets/mediq.py | 2 +- abstentionbench/recipe/abstention_datasets/musique.py | 2 +- abstentionbench/recipe/abstention_datasets/qasper.py | 2 +- .../recipe/abstention_datasets/self_aware.py | 2 +- abstentionbench/recipe/job_manager.py | 2 +- pyproject.toml | 2 +- tests/test_datasets.py | 2 +- 13 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 5ddd1f9..fdb6ce4 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ You can explore existing abstention results without special installations. Simpl ```python import pandas as pd -df = pd.read_csv("analysis/abstention_performance.csv") +df = pd.read_csv("abstentionbench/analysis/abstention_performance.csv") df ``` @@ -151,7 +151,7 @@ To set up to run with FreshQA: 1. Review the list of available FreshQA Google Sheets on the project's [README](https://github.com/freshllms/freshqa?tab=readme-ov-file) 2. Pick a 'baseline' date, which should be before the model's pretraining cut-off, and an 'updated' date, which should be after the cut-off 3. For both baseline and updated, open the Google Sheet and export their contents as CSV (File > Download > Comma Separated Values) -4. Move these two CSVs into `data/freshqa/`, and update the paths in `configs/dataset/freshqa.yaml` accordingly +4. Move these two CSVs into `abstentionbench/data/freshqa/`, and update the paths in `abstentionbench/configs/dataset/freshqa.yaml` accordingly Note: To exactly replicate our work, use [FreshQA_v10282024](https://docs.google.com/spreadsheets/d/1j6qr14l8oK_7gJ_XdnTBi8Pj1NVt5yeQEBKxFPkKn_g/edit?gid=334049794#gid=334049794) and [FreshQA_v12182024](https://docs.google.com/spreadsheets/d/1llFQDYuwX95L7yYDQ4aLCwJmkEh9VOSHNu6g7HjT8e0/edit?gid=334049794#gid=334049794) as baseline and updated respectively. @@ -169,7 +169,7 @@ For a simple test of the end-to-end pipeline on your local machine, run: python main.py -m mode=local model=dummy abstention_detector=contains_abstention_keyword run_single_job_for_inference_and_judge=True ``` -Where the `dataset` arg corresponds to config file names in `configs/dataset/`, and `model` to `config/model/`. See below for supported models and datasets. +Where the `dataset` arg corresponds to config file names in `abstentionbench/configs/dataset/`, and `model` to `abstentionbench/configs/model/`. See below for supported models and datasets. By default, `main.py` runs the whole pipeline from inference, through abstention detection, to evaluation. The pipeline launches seaparate jobs for inference and the rest of the evaluations to support a separate LLM judge. To launch everything in a single job for debugging use the `run_single_job_for_inference_and_judge=True` flag. @@ -178,7 +178,7 @@ Model responses and evaluation results will be saved in `save_dir` specified in To run a fast subset on with at most 100 examples per dataset: ``` -python main.py -m mode=cluster dataset='glob(*,exclude=dummy)' model=llama_3_1_8B_instruct sweep_folder=fast-subset dataset_indices_path=data/fast-subset-indices.json +python main.py -m mode=cluster dataset='glob(*,exclude=dummy)' model=llama_3_1_8B_instruct sweep_folder=fast-subset dataset_indices_path=abstentionbench/data/fast-subset-indices.json ``` Note `mode=cluster` expects you're running the code on a cluster that supports SLURM. @@ -227,7 +227,7 @@ Note `mode=cluster` expects you're running the code on a cluster that supports S Note that all datasets with over a threshold 3.5k samples were truncated using uniform subsampling. These datasets are marked with "[subsampled]" in the list above. -Sample indices can be found in `data/subsampling-indices.json`. +Sample indices can be found in `abstentionbench/data/subsampling-indices.json`. You can disable this by setting the `dataset_indices_path` config attribute to `null`. diff --git a/abstentionbench/__init__.py b/abstentionbench/__init__.py index f2189fc..2347e19 100644 --- a/abstentionbench/__init__.py +++ b/abstentionbench/__init__.py @@ -4,4 +4,4 @@ This package provides tools and datasets for evaluating LLM abstention capabilities. """ -__version__ = "1.0.1" \ No newline at end of file +__version__ = "1.0.0" \ No newline at end of file diff --git a/abstentionbench/analysis/load_results.py b/abstentionbench/analysis/load_results.py index 2d927eb..b8dbe34 100644 --- a/abstentionbench/analysis/load_results.py +++ b/abstentionbench/analysis/load_results.py @@ -20,7 +20,7 @@ class Results: """Loads results. By default uses the paths specified in - analysis/results_path.py + abstentionbench/analysis/results_path.py Attributes: df: pandas dataframe that turns saved results into a table diff --git a/abstentionbench/configs/default_pipeline.yaml b/abstentionbench/configs/default_pipeline.yaml index ce8e6d5..ca85154 100644 --- a/abstentionbench/configs/default_pipeline.yaml +++ b/abstentionbench/configs/default_pipeline.yaml @@ -43,8 +43,8 @@ only_run_inference: False # name to list of indices. dataset_indices_subset: null # fast subset -# dataset_indices_path: data/fast-subset-indices.json -dataset_indices_path: data/subsampling-indices.json +# dataset_indices_path: abstentionbench/data/fast-subset-indices.json +dataset_indices_path: abstentionbench/data/subsampling-indices.json seed: 123456789 diff --git a/abstentionbench/recipe/abstention_datasets/alcuna.py b/abstentionbench/recipe/abstention_datasets/alcuna.py index 2cc238f..80faacc 100644 --- a/abstentionbench/recipe/abstention_datasets/alcuna.py +++ b/abstentionbench/recipe/abstention_datasets/alcuna.py @@ -50,12 +50,12 @@ def __init__( try: # Load the formatted dataset from disk - self.dataset = datasets.Dataset.load_from_disk(data_dir) + self.dataset = datasets.Dataset.load_from_disk(self.data_dir) except: logger.info("Preparing dataset") - questions_path = os.path.join(data_dir, "id2question.json") - metadata_path = os.path.join(data_dir, "meta_data.jsonl") + questions_path = os.path.join(self.data_dir, "id2question.json") + metadata_path = os.path.join(self.data_dir, "meta_data.jsonl") self._download_data() @@ -67,7 +67,7 @@ def __init__( self.dataset = self._prepare_dataset(questions_data, metadata) - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) def _download_data(self): file_id_and_file_names = [ diff --git a/abstentionbench/recipe/abstention_datasets/bbq.py b/abstentionbench/recipe/abstention_datasets/bbq.py index 2479bee..e1a9d59 100644 --- a/abstentionbench/recipe/abstention_datasets/bbq.py +++ b/abstentionbench/recipe/abstention_datasets/bbq.py @@ -63,7 +63,7 @@ def __init__( self.dataset = datasets.concatenate_datasets(all_subdatasets) - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) def __len__(self): return self.max_num_samples or len(self.dataset) diff --git a/abstentionbench/recipe/abstention_datasets/mediq.py b/abstentionbench/recipe/abstention_datasets/mediq.py index 4cf6652..efb4842 100644 --- a/abstentionbench/recipe/abstention_datasets/mediq.py +++ b/abstentionbench/recipe/abstention_datasets/mediq.py @@ -47,7 +47,7 @@ def __init__( self.dataset = datasets.Dataset.from_list(list(data)) - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) if exclude_sample_ids is not None: self.dataset = self.dataset.filter( diff --git a/abstentionbench/recipe/abstention_datasets/musique.py b/abstentionbench/recipe/abstention_datasets/musique.py index 20eecbb..b22fecf 100644 --- a/abstentionbench/recipe/abstention_datasets/musique.py +++ b/abstentionbench/recipe/abstention_datasets/musique.py @@ -18,7 +18,7 @@ class MusiqueDataset(NQDataset): def __init__( self, - data_dir='data/musique', + data_dir='abstentionbench/data/musique', file_name="musique/test.json", max_num_samples=None, ): diff --git a/abstentionbench/recipe/abstention_datasets/qasper.py b/abstentionbench/recipe/abstention_datasets/qasper.py index e273b74..fe05746 100644 --- a/abstentionbench/recipe/abstention_datasets/qasper.py +++ b/abstentionbench/recipe/abstention_datasets/qasper.py @@ -42,7 +42,7 @@ def __init__(self, data_dir=None, max_num_samples=None): self.dataset = self._prepare_dataset(dataset) - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) # Only keep questions where annotators agree on answerable/unanswerable self.dataset = self.dataset.filter(lambda x: x["is_unanswerable"] is not None) diff --git a/abstentionbench/recipe/abstention_datasets/self_aware.py b/abstentionbench/recipe/abstention_datasets/self_aware.py index 07f2819..9c10b09 100644 --- a/abstentionbench/recipe/abstention_datasets/self_aware.py +++ b/abstentionbench/recipe/abstention_datasets/self_aware.py @@ -33,7 +33,7 @@ def __init__( data = response.json() self.dataset = datasets.Dataset.from_list(data["example"]) - self.dataset.save_to_disk(data_dir) + self.dataset.save_to_disk(self.data_dir) def __len__(self): return self.max_num_samples or len(self.dataset) diff --git a/abstentionbench/recipe/job_manager.py b/abstentionbench/recipe/job_manager.py index d410b5a..fc29d03 100644 --- a/abstentionbench/recipe/job_manager.py +++ b/abstentionbench/recipe/job_manager.py @@ -5,7 +5,7 @@ LICENSE file in the root directory of this source tree. """ """ -python recipe/job_manager.py show_relaunch_commands_for_missing_jobs --sweep_dir=sweep-reasoning-2025-03-25 +python abstentionbench/recipe/job_manager.py show_relaunch_commands_for_missing_jobs --sweep_dir=sweep-reasoning-2025-03-25 """ import glob diff --git a/pyproject.toml b/pyproject.toml index deec76f..2a4f1b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "abstentionbench" -version = "1.0.1" +version = "1.0.0" description = "A Holistic Benchmark for LLM Abstention" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_datasets.py b/tests/test_datasets.py index b2fea90..d338b00 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -647,7 +647,7 @@ def test_len(self): "dataset_name,expected_len", [("dummy", 100), ("musique", 3266)] ) def test_config_instantiation(dataset_name, expected_len) -> None: - with initialize(version_base="1.2", config_path="../configs"): + with initialize(version_base="1.2", config_path="../abstentionbench/configs"): # config is relative to a module cfg = compose( config_name="default_pipeline.yaml", overrides=[f"dataset={dataset_name}"]