-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathrun.sh
More file actions
executable file
·96 lines (83 loc) · 8.02 KB
/
Copy pathrun.sh
File metadata and controls
executable file
·96 lines (83 loc) · 8.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/llama-3.2-1b --compiled-model-path /home/ubuntu/traced_model/llama-3.2-1b --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.2-3B --compiled-model-path /home/ubuntu/traced_model/Llama-3.2-3B --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.1-8B --compiled-model-path /home/ubuntu/traced_model/Llama-3.1-8B --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# python qwen_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Qwen3-1.7B --compiled-model-path /home/ubuntu/traced_model/Qwen3-4B --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# python qwen_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Qwen3-4B --compiled-model-path /home/ubuntu/traced_model/Qwen3-4B --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# python qwen_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Qwen3-8B --compiled-model-path /home/ubuntu/traced_model/Qwen3-8B --torch-dtype bfloat16 --batch-size 1 --max-context-length 128 --seq-len 256 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# #!/bin/bash
# Qwen
models=(
# "llama-3.2-1b"
# "Llama-3.2-3B"
# "Llama-3.1-8B"
Qwen3-1.7B
Qwen3-4B
)
# Define different compression ratios
ratios=(0.8)
# Run in loops
for model in "${models[@]}"; do
for ratio in "${ratios[@]}"; do
echo ">>> Running model $model with compress-ratio=$ratio"
python qwen_inference.py \
--model-type llama \
--task-type causal-lm \
run \
--model-path /home/ubuntu/models/$model \
--compiled-model-path /home/ubuntu/traced_model/$model \
--torch-dtype bfloat16 \
--batch-size 1 \
--max-context-length 32 \
--seq-len 64 \
--check-accuracy-mode logit-matching \
--divergence-difference-tol 0.005 \
--tol-map "{5: (1e-5, 0.02)}" \
--enable-bucketing \
--top-k 1 \
--pad-token-id 2 \
--prompt "I believe the meaning of life is" \
--prompt "The color of the sky is" \
--compress-ratio $ratio
done
done
# llama
models=(
"llama-3.2-1b"
"Llama-3.2-3B"
# # "Llama-3.1-8B"
# Qwen3-1.7B
# Qwen3-4B
)
# Define different compression ratios
ratios=(0.8)
# Run in loops
for model in "${models[@]}"; do
for ratio in "${ratios[@]}"; do
echo ">>> Running model $model with compress-ratio=$ratio"
python llama_inference.py \
--model-type llama \
--task-type causal-lm \
run \
--model-path /home/ubuntu/models/$model \
--compiled-model-path /home/ubuntu/traced_model/$model \
--torch-dtype bfloat16 \
--batch-size 1 \
--max-context-length 32 \
--seq-len 64 \
--check-accuracy-mode logit-matching \
--divergence-difference-tol 0.005 \
--tol-map "{5: (1e-5, 0.02)}" \
--enable-bucketing \
--top-k 1 \
--pad-token-id 2 \
--prompt "I believe the meaning of life is" \
--prompt "The color of the sky is" \
--compress-ratio $ratio
done
done
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/llama-3.2-1b --compiled-model-path /home/ubuntu/traced_model/llama-3.2-1b --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.2-3B --compiled-model-path /home/ubuntu/traced_model/Llama-3.2-3B --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.1-8B --compiled-model-path /home/ubuntu/traced_model/Llama-3.1-8B --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/llama-3.2-1b --compiled-model-path /home/ubuntu/traced_model/llama-3.2-1b --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.2-3B --compiled-model-path /home/ubuntu/traced_model/Llama-3.2-3B --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8
# # python llama_inference.py --model-type llama --task-type causal-lm run --model-path /home/ubuntu/models/Llama-3.1-8B --compiled-model-path /home/ubuntu/traced_model/Llama-3.1-8B --torch-dtype bfloat16 --batch-size 1 --max-context-length 256 --seq-len 512 --check-accuracy-mode logit-matching --divergence-difference-tol 0.005 --tol-map "{5: (1e-5, 0.02)}" --enable-bucketing --top-k 1 --pad-token-id 2 --prompt "I believe the meaning of life is" --prompt "The color of the sky is" --compress-ratio 0.8