From 0100a898132249f3d72da5a08b1b10be0718aeec Mon Sep 17 00:00:00 2001
From: zjjlivein <1834792141@qq.com>
Date: Fri, 21 Mar 2025 15:36:29 +0800
Subject: [PATCH] =?UTF-8?q?=E5=BC=80=E6=BA=90=E5=A4=A7=E6=A8=A1=E5=9E=8B?=
 =?UTF-8?q?=20=E5=A2=9E=E5=8A=A0Qwen=E7=9B=91=E6=8E=A7=20test=3Dmodel?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../PaddleNLP/llm/case/llm^llama.yaml.yaml    |  0
 .../PaddleNLP/llm/case/llm^qwen.yaml.yaml     | 35 +++++++
 models_restruct/PaddleNLP/llm/train/grpo.sh   | 91 +++++++++++++++++++
 models_restruct/PaddleNLP/llm/train/infer.sh  | 29 ++++++
 .../PaddleNLP/llm/train/predict.sh            | 33 +++++++
 5 files changed, 188 insertions(+)
 create mode 100644 models_restruct/PaddleNLP/llm/case/llm^llama.yaml.yaml
 create mode 100644 models_restruct/PaddleNLP/llm/case/llm^qwen.yaml.yaml
 create mode 100644 models_restruct/PaddleNLP/llm/train/grpo.sh
 create mode 100644 models_restruct/PaddleNLP/llm/train/infer.sh
 create mode 100644 models_restruct/PaddleNLP/llm/train/predict.sh

diff --git a/models_restruct/PaddleNLP/llm/case/llm^llama.yaml.yaml b/models_restruct/PaddleNLP/llm/case/llm^llama.yaml.yaml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/models_restruct/PaddleNLP/llm/case/llm^qwen.yaml.yaml b/models_restruct/PaddleNLP/llm/case/llm^qwen.yaml.yaml
new file mode 100644
index 0000000000..1cf00b4f19
--- /dev/null
+++ b/models_restruct/PaddleNLP/llm/case/llm^qwen.yaml.yaml
@@ -0,0 +1,35 @@
+case:
+  linux:
+    train:
+      -
+        name: grpo_training
+        path: llm/alignment/ppo
+        cmd: bash grpo.sh qwen 
+        result:
+          policy_loss:
+            base: "compare_path"
+            threshold: 0.1
+            evaluation: "=" # = base值转成str进行按位对齐   
+          reward:
+            base: "compare_path"
+            threshold: 0.1
+            evaluation: "-"
+          values:
+            base: "compare_path"
+            threshold: 0.1
+            evaluation: "-"
+          kl_divergence:
+            base: "compare_path"
+            threshold: 0.1
+            evaluation: "-"
+          interval_samples_per_second:
+            base: "compare_path"
+            threshold: 0.1
+            evaluation: "-"  
+      -
+        name: grpo_predict_dynamic_ngpus2_default_bf16 # 2卡动态图预测
+        path: llm/
+        cmd: bash ./predict.sh qwen 2 grpo
+      -
+        name: grpo_export_infer_ngpus2_bf16 # 2卡, sample预测 注意执行路径！
+        cmd: bash infer.sh qwen 2 grpo
\ No newline at end of file
diff --git a/models_restruct/PaddleNLP/llm/train/grpo.sh b/models_restruct/PaddleNLP/llm/train/grpo.sh
new file mode 100644
index 0000000000..2f2d2f026a
--- /dev/null
+++ b/models_restruct/PaddleNLP/llm/train/grpo.sh
@@ -0,0 +1,91 @@
+# work_path: PaddleNLP/llm
+# grpo 训练
+model_name=$1
+ngpus=${2:-8}
+ext_args=""
+
+# 1. 模型准备
+echo "清理显存"
+fuser -v /dev/nvidia* 2>/dev/null | awk '{for(i=1;i<=NF;i++) if ($i ~ /^[0-9]+$/) print $i}' | xargs kill -9 2>/dev/null
+sleep 3s
+echo "清理Checkpoints"
+rm -rf ../../checkpoints/${model_name}/grpo/* 2>/dev/null 
+
+if [[ ${model_name} == "qwen" ]]; then
+    model_name_or_path="Qwen/Qwen2.5-7B-Instruct-1M"
+elif [[ ${model_name} == "llama" ]]; then
+    model_name_or_path="meta-llama/Meta-Llama-3-8B"
+fi
+fi
+output_dir="../../checkpoints/${model_name}/grpo" # 以PaddleNLP/llm为根目录
+
+# 2. 数据准备 
+if [ ! -d "ppo-kk" ]; then
+    wget https://paddlenlp.bj.bcebos.com/datasets/examples/ppo-kk.tgz && tar zxf ppo-kk.tgz
+fi
+
+# 3. 设置环境变量
+if [ $ngpus -eq 0 ]; then  
+    DEVICE="0"  
+elif [ $ngpus -eq 1 ]; then  
+    DEVICE="1"  
+elif [ $ngpus -eq 2 ]; then  
+    DEVICE="2,3"  
+elif [ $ngpus -eq 4 ]; then  
+    DEVICE="4,5,6,7"  
+elif [ $ngpus -eq 8 ]; then  
+    DEVICE="0,1,2,3,4,5,6,7" 
+else  
+    echo "Unsupported number of GPUs"  
+    exit 1  
+fi  
+export CUDA_VISIBLE_DEVICES=${DEVICE}
+export PYTHONPATH=/workspace/PaddleNL/:$PYTHONPATH # 注意系统路径，请根据实际情况修改！！
+export PYTHONPATH=/workspace/PaddleNLP/llm:$PYTHONPATH
+
+# 4. 启动训练脚本
+echo "启动reward服务"
+python reward_server.py > reward_server.log 2>&1 &
+echo "开始训练:"
+
+python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" run_ppo.py ../../config/${model_name}/grpo_argument.json \
+    --train_datasets "Jsonfile::ppo-kk/34567ppl/train.jsonl" \
+    --eval_datasets "Jsonfile::ppo-kk/5ppl/test.jsonl" \
+    --actor_model_name_or_path ${model_name_or_path} \
+    --reward_model_name_or_path "" \
+    --output_dir ${output_dir} \
+    --max_steps 20 \
+    --save_steps 20 \
+    --tensor_parallel_degree 4 \
+    --per_device_prompt_batch_size 1 \
+    --per_device_train_batch_size 4 \
+    --max_length 1024 \
+    --max_prompt_len 512 \
+    --pipeline_parallel_degree 1 \
+    --sharding_parallel_degree 2 \
+    --sharding "stage1" \
+    --recompute 1 \
+    ${ext_args}
+
+echo "热启"
+python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" run_ppo.py ../../config/${model_name}/grpo_argument.json \
+    --train_datasets "Jsonfile::ppo-kk/34567ppl/train.jsonl" \
+    --eval_datasets "Jsonfile::ppo-kk/5ppl/test.jsonl" \
+    --actor_model_name_or_path ${model_name_or_path} \
+    --reward_model_name_or_path "" \
+    --output_dir ${output_dir} \
+    --max_steps 2 \
+    --save_steps 11 \
+    --tensor_parallel_degree 4 \
+    --per_device_prompt_batch_size 1 \
+    --per_device_train_batch_size 4 \
+    --max_length 1024 \
+    --max_prompt_len 512 \
+    --pipeline_parallel_degree 1 \
+    --sharding_parallel_degree 2 \
+    --sharding "stage1" \
+    --recompute 1 \
+    ${ext_args}
+
+echo "kill reward 服务"
+pkill -9 -f reward_server.py
\ No newline at end of file
diff --git a/models_restruct/PaddleNLP/llm/train/infer.sh b/models_restruct/PaddleNLP/llm/train/infer.sh
new file mode 100644
index 0000000000..0695f3507b
--- /dev/null
+++ b/models_restruct/PaddleNLP/llm/train/infer.sh
@@ -0,0 +1,29 @@
+# 静态图模型推理
+# work_path: PaddleNLP 后续可能修改
+model_name=$1
+ngpus=${2:-8}
+step_name=${3:-"grpo"}
+
+# 1.动转静模型路径
+if [ "$step_name" == "ppo" ] || [ "$step_name" == "grpo" ]; then 
+    model_name_or_path=./llm/checkpoints/$model_name/${step_name}/policy/checkpoint-20
+else
+    model_name_or_path=./llm/checkpoints/$model_name/${step_name}
+fi
+output_path="$model_name_or_path/inference"
+
+# 2. 静态图导出
+echo "静态图导出..."
+python llm/predict/export_model.py \
+    --model_name_or_path ${model_name_or_path} \
+    --inference_model \
+    --output_path ${output_path} \
+    --dtype float16
+
+# 3. 静态图推理
+echo "静态图推理..."
+python llm/predict/predictor.py \
+    --model_name_or_path ${output_path} \
+    --inference_model \
+    --dtype "float16" \
+    --mode "static"
\ No newline at end of file
diff --git a/models_restruct/PaddleNLP/llm/train/predict.sh b/models_restruct/PaddleNLP/llm/train/predict.sh
new file mode 100644
index 0000000000..c4b8f7b107
--- /dev/null
+++ b/models_restruct/PaddleNLP/llm/train/predict.sh
@@ -0,0 +1,33 @@
+# 动态图模型推理
+model_name=$1
+ngpus=${2:-8}
+step_name=${3:-"grpo"}
+# 1.设置模型路径
+if [ "$step_name" == "ppo" ] || [ "$step_name" == "grpo" ]; then 
+    model_name_or_path=./checkpoints/$model_name/${step_name}/policy/checkpoint-20
+else
+    model_name_or_path=./checkpoints/$model_name/${step_name}
+fi
+
+# 2.设置GPU
+if [ $ngpus -eq 0 ]; then  
+    DEVICE="0"  
+elif [ $ngpus -eq 1 ]; then  
+    DEVICE="1"  
+elif [ $ngpus -eq 2 ]; then  
+    DEVICE="2,3"  
+elif [ $ngpus -eq 4 ]; then  
+    DEVICE="4,5,6,7"  
+elif [ $ngpus -eq 8 ]; then  
+    DEVICE="0,1,2,3,4,5,6,7" 
+else  
+    echo "Unsupported number of GPUs"  
+    exit 1  
+fi  
+export CUDA_VISIBLE_DEVICES=${DEVICE}
+
+
+python ./predict/predictor.py \
+    --model_name_or_path $model_name_or_path \
+    --inference_model \
+    --dtype float16
\ No newline at end of file