XModBench/run_api.slurm at master · XingruiWang/XModBench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#!/bin/bash
#SBATCH --job-name=cpu
#SBATCH --output=log/job_%j.out
#SBATCH --error=log/job_%j.log
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=1
#SBATCH --exclude=ccvl34

echo "Running on host: $(hostname)"
echo "CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"

# Load user environment
source ~/.bashrc
module load conda
conda activate vlm

export audioBench='/home/xwang378/scratch/2025/AudioBench'

# Configuration
# MODEL="gemini-2.0-pro"
MODEL="gemini-2.5-pro"
# MODEL="reka"
MINI_BENCHMARK="false"

# Function to run evaluation
run_evaluation() {
    local model=$1
    local task=$2
    local subtask=$3
    local modality=$4

    local task_name="${task}/${subtask}_${modality}"

    echo "Running: Model=${model}, Task=${task_name}"

    if [ "$MINI_BENCHMARK" = "true" ]; then
        python $audioBench/scripts/run.py \
            --model $model \
            --task_name $task_name \
            --sample 5 \
            --mini_benchmark
    else
        python $audioBench/scripts/run.py \
            --model $model \
            --task_name $task_name \
            --sample 800
    fi
}


# Main execution - uncomment the tasks you want to run
TASKS_TO_RUN=(
    # Perception tasks
    # "perception/general"
    # "perception/finegrained"
    # "perception/instruments"
    # "perception/instruments_comp"
    "perception/natures"

    # # Spatial tasks
    # "spatial/arrangements"
    # "spatial/3D_movements"
    # "spatial/panaroma"

    # # Speech tasks
    # "speech/recognition"
    # "speech/translation"

    # # Temporal tasks
    # "temporal/count"
    # "temporal/calculation"
    # "temporal/order"

    # # External tasks
    # "external/music_genre_classification"
    # "external/emotion_classification"
#     "external/movie_matching"
#     "external/singer_identification"
)

# Run evaluations
# for MODEL in gemini-2.5-pro gemini-2.5-flash gemini-2.0-pro; do
for MODEL in gemini-2.0-pro; do
# for MODEL in reka; do
    for task_key in "${TASKS_TO_RUN[@]}"; do
        task=$(echo $task_key | cut -d'/' -f1)
        subtask=$(echo $task_key | cut -d'/' -f2)
        modalities="audio_vision vision_audio vision_text text_vision text_audio audio_text"

        echo "Processing task: $task_key"
        echo "Modalities: $modalities"
        echo "----------------------------------------"

        for modality in $modalities; do
            run_evaluation $MODEL $task $subtask $modality
        done

        echo "Completed task: $task_key"
        echo "========================================"
    done
done

echo "All evaluations completed!"