Deep_learning_project/check_models.py at main · Cozy000000/Deep_learning_project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#!/usr/bin/env python3
"""
本地模型路径验证脚本
检查所有配置的模型路径是否存在且可访问
"""

import os
import sys
from pathlib import Path

# 添加项目根目录到Python路径
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from src.config import get_config_manager

def check_model_paths():
    """检查所有模型路径是否存在"""

    print("🔍 检查本地模型路径...")
    print("=" * 50)

    config_manager = get_config_manager()
    models = config_manager.get_available_models()

    all_exist = True

    for model_name in models:
        model_config = config_manager.get_model_config(model_name)
        model_path = model_config["model_name"]

        print(f"\n📁 {model_name}:")
        print(f"   路径: {model_path}")

        # 检查路径是否存在
        if os.path.exists(model_path):
            print(f"   状态: ✅ 存在")

            # 检查是否包含必要文件
            required_files = ["config.json", "tokenizer.json"]
            optional_files = ["pytorch_model.bin", "model.safetensors", "tokenizer_config.json"]

            missing_required = []
            missing_optional = []

            for file in required_files:
                file_path = os.path.join(model_path, file)
                if not os.path.exists(file_path):
                    missing_required.append(file)

            for file in optional_files:
                file_path = os.path.join(model_path, file)
                if not os.path.exists(file_path):
                    missing_optional.append(file)

            if missing_required:
                print(f"   ⚠️  缺少必需文件: {', '.join(missing_required)}")
                all_exist = False
            else:
                print(f"   ✅ 包含必需文件")

            if missing_optional:
                print(f"   ℹ️  缺少可选文件: {', '.join(missing_optional)}")

        else:
            print(f"   状態: ❌ 不存在")
            all_exist = False

    print("\n" + "=" * 50)

    if all_exist:
        print("✅ 所有模型路径验证通过！")
        print("\n可以开始运行评估:")
        print("  python test_system.py")
        return True
    else:
        print("❌ 部分模型路径不存在，请检查配置。")
        print("\n建议操作:")
        print("1. 确认模型文件已下载到正确位置")
        print("2. 检查路径配置是否正确")
        print("3. 修改 src/models.py 中的 MODEL_CONFIGS")
        return False

def suggest_download_commands():
    """建议下载命令"""
    print("\n📥 模型下载建议:")
    print("-" * 30)

    print("# 使用 huggingface-cli 下载模型到本地:")
    print("huggingface-cli download mistralai/Ministral-8B-Instruct-2410 --local-dir /data/Mistral-7B-Instruct-v0.3")
    print("huggingface-cli download google/gemma-2-2b-it --local-dir /data/gemma-2-2b")
    print("huggingface-cli download Qwen/Qwen2.5-3B-Instruct --local-dir /data/Qwen2.5-3B-Instruct")

    print("\n# 或使用 git clone:")
    print("git lfs install")
    print("git clone https://huggingface.co/mistralai/Ministral-8B-Instruct-2410 /data/Mistral-7B-Instruct-v0.3")
    print("git clone https://huggingface.co/google/gemma-2-2b-it /data/gemma-2-2b")
    print("git clone https://huggingface.co/Qwen/Qwen2.5-3B-Instruct /data/Qwen2.5-3B-Instruct")

if __name__ == "__main__":
    success = check_model_paths()

    if not success:
        suggest_download_commands()

    exit(0 if success else 1)