evaluate.py

import json

INPUT = "test_model_responses_after_gpt_judging.json" # contains judgment in "gpt_judge"
OUTPUT = "output_file.json"

total_dic = {
    "影视娱乐": 191,
    "教育培养": 147,
    "数理化生": 178,
    "历史国学": 186,
    "人物百科": 190,
    "政治法律": 155,
    "经济管理": 141,
    "计算机科学": 146,
    "医学": 128,
    "社会人文": 187,
    "农林牧渔": 138,
    "天文地理": 151,
    "运动旅游": 143,
    "数码汽车": 159,
    "工业工程": 149,
    "军武战争": 142,
    "网词网梗": 104,
    "工作生活": 131,
    "高新科技": 112,
    "信仰文化": 122
}
total_cnt = 3000

correct_cnt_dic = {}
correct_cnt = 0
incorrect_cnt = 0

with open(INPUT, "r") as f:
    data = json.load(f)

for d in data:
    if d["gpt_judge"].find("【正确】") != -1 and d["gpt_judge"].find("【错误】") == -1:
        d["judge"] = 1
    elif d["gpt_judge"].find("【错误】") != -1 and d["gpt_judge"].find("【正确】") == -1:
        d["judge"] = 0
    else:
        # You can uncomment the following lines to manually judge unclear cases
        # print("Fail in instruct following:")
        # print(d)
        # d["judge"] = int(input("1 for correct and 0 for incorrect"))
        d["judge"] = 0  # default to incorrect if unclear
    
    if d["domain"] not in correct_cnt_dic.keys():
        correct_cnt_dic[d["domain"]] = 0
    if d["judge"] == 1:
        correct_cnt += 1
        correct_cnt_dic[d["prompt_types"]] += 1
    else:
        incorrect_cnt += 1

print("Incorrect_cnt：", incorrect_cnt)
print("Correct_cnt：", correct_cnt)
print("Overall Acc:", correct_cnt / total_cnt)

print("Domain - Correct Count - Accuracy")
for key in total_dic.keys():
    print(key, correct_cnt_dic[key], correct_cnt_dic[key] / total_dic[key])
    correct_cnt_dic[key] = correct_cnt_dic[key] / total_dic[key]
    
sorted_dict = dict(sorted(correct_cnt_dic.items(), key=lambda x: x[1]))
print("Sorted Domain Accuracy:")
print(sorted_dict)

with open(OUTPUT, "w") as f:
    json.dump(data, f, ensure_ascii=False, indent=4)