Deep_learning_project/config.json at main · Cozy000000/Deep_learning_project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
{
  "models": {
    "ministral-7b": {
      "name": "ministral-7b",
      "model_name": "/data/Mistral-7B-Instruct-v0.3",
      "description": "Mistral AI的7B参数指令调优模型（本地版本）",
      "expected_performance": {
        "mmlu-pro": "0.45-0.55",
        "gsm8k": "0.60-0.70"
      }
    },
    "gemma-2-2b": {
      "name": "gemma-2-2b",
      "model_name": "/data/gemma-2-2b",
      "description": "Google Gemma 2的2B参数版本（本地版本）",
      "expected_performance": {
        "mmlu-pro": "0.35-0.45",
        "gsm8k": "0.40-0.50"
      }
    },
    "qwen2.5-3b": {
      "name": "qwen2.5-3b",
      "model_name": "/data/Qwen2.5-3B-Instruct",
      "description": "阿里云Qwen 2.5的3B参数版本（本地版本）",
      "expected_performance": {
        "mmlu-pro": "0.40-0.50",
        "gsm8k": "0.50-0.60"
      }
    },
    "llama3.1-8b": {
      "name": "llama3.1-8b",
      "model_name": "/data/Meta-Llama-3.1-8B-Instruct",
      "description": "Meta Llama 3.1的8B参数指令调优模型（本地版本）",
      "expected_performance": {
        "mmlu-pro": "0.55-0.65",
        "gsm8k": "0.70-0.80"
      }
    }
  },

  "datasets": {
    "mmlu": {
      "name": "MMLU",
      "description": "大规模多任务语言理解基准，包含57个学科的4选题",
      "num_samples": 14042,
      "answer_format": "multiple_choice",
      "evaluation_metric": "accuracy"
    },
    "mmlu-pro": {
      "name": "MMLU-Pro",
      "description": "多项选择推理任务，包含多个学科领域",
      "num_samples": 12032,
      "answer_format": "multiple_choice",
      "evaluation_metric": "accuracy"
    },
    "gsm8k": {
      "name": "GSM8K",
      "description": "小学数学应用题，需要多步推理",
      "num_samples": 1319,
      "answer_format": "numerical",
      "evaluation_metric": "exact_match"
    }
  },

  "inference_config": {
    "default": {
      "num_samples": 5,
      "temperature": 0.7,
      "top_p": 0.9,
      "max_new_tokens": 512,
      "description": "标准self-consistency配置"
    },
    "conservative": {
      "num_samples": 3,
      "temperature": 0.5,
      "top_p": 0.8,
      "max_new_tokens": 256,
      "description": "保守配置，更一致但可能创造性不足"
    },
    "diverse": {
      "num_samples": 7,
      "temperature": 0.9,
      "top_p": 0.95,
      "max_new_tokens": 768,
      "description": "多样化配置，更多样性但可能不够一致"
    }
  },

  "evaluation_settings": {
    "quick_test": {
      "num_samples_per_dataset": 50,
      "description": "快速测试，用于验证系统功能"
    },
    "development": {
      "num_samples_per_dataset": 200,
      "description": "开发测试，用于调试和初步验证"
    },
    "full_evaluation": {
      "num_samples_per_dataset": null,
      "description": "完整评估，使用所有样本"
    }
  },

  "output_settings": {
    "save_individual_responses": true,
    "save_token_usage": true,
    "save_timing_info": true,
    "generate_visualization": true,
    "generate_summary_report": true
  }
}