L2D/exp_attack.sh at main · Mamba413/L2D · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/usr/bin/env bash
# Copyright (c) Jin Zhu.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# setup the environment
echo `date`, Setup the environment ...
set -e  # exit if error

# prepare folders
exp_path=exp_attack
mkdir -p $exp_path $exp_path/data/ $exp_path/results/
src_path=exp_prompt
src_data_path=$src_path/data
datasets="xsum squad writing"
M_test="claude-3-5-haiku"
M_train="claude-3-5-haiku"  ## it is fair as the comparison is conducted over the ML-based method
M2='gemma-9b-instruct'
paras="vanilla random t5"  # "vanilla" for no attack, "t5" for paraphrasing attack, or "random" for decoherence attack

for para in $paras; do
    data_path=$exp_path/data/$para
    res_path=$exp_path/results/$para
    mkdir -p $data_path $res_path

    # preparing dataset
    if [ "$para" != "vanilla" ]; then
        for D in $datasets; do
            for M in $source_models; do
                echo "$(date)", Preparing ${D}_${M} using paraphraser: $para
                python scripts/paraphrasing.py \
                    --dataset $D \
                    --dataset_file ${src_data_path}/${D}_${M}_polish \
                    --paraphraser $para \
                    --output_file ${data_path}/${D}_${M}_polish
            done
        done
    else
        echo "$(date)", Copying original data to $data_path
        cp -r "${src_data_path}/." "$data_path/"
    fi

    # evaluate RAIDAR (train on other LLMs)
    for D in $datasets; do
        train_dataset=""

        # collect training data from other LLMs
        for D1 in $datasets; do
            if [ "$D1" = "$D" ]; then
                continue  # 排除与测试集相同的 dataset
            fi

            # append three tasks for each (D1, M_train)
            if [ -z "$train_dataset" ]; then
                train_dataset="${data_path}/${D1}_${M_train}_polish&${data_path}/${D1}_${M_train}_rewrite&${data_path}/${D1}_${M_train}_expand"
            else
                train_dataset="${train_dataset}&${data_path}/${D1}_${M_train}_polish&${data_path}/${D1}_${M_train}_rewrite&${data_path}/${D1}_${M_train}_expand"
            fi
        done

        echo "Train data (RAIDAR): $train_dataset"
        python scripts/detect_raidar.py \
            --train_dataset ${train_dataset} \
            --eval_dataset $data_path/${D}_${M_test}_polish \
            --output_file $res_path/${D}_${M_test}_polish \
            --regen_number 2 \
            --batch_size 2
    done

    # evaluate L2D & ImBD
    for D in $datasets; do
        train_dataset=""
        for D1 in $datasets; do
            if [ "$D1" = "$D" ]; then
                continue  # 排除测试 dataset
            fi

            if [ -z "$train_dataset" ]; then
                train_dataset="${data_path}/${D1}_${M_train}_polish&${data_path}/${D1}_${M_train}_rewrite&${data_path}/${D1}_${M_train}_expand"
            else
                train_dataset="${train_dataset}&${data_path}/${D1}_${M_train}_polish&${data_path}/${D1}_${M_train}_rewrite&${data_path}/${D1}_${M_train}_expand"
            fi
        done

        echo "Train data (AdaDist/ImBD): $train_dataset"

        python scripts/detect_l2d.py \
            --datanum 500 \
            --base_model ${M2} \
            --train_dataset ${train_dataset} \
            --eval_after_train \
            --eval_dataset $data_path/${D}_${M_test}_polish \
            --output_file $res_path/${D}_${M_test}_polish \
            --regen_number 2 \
            --batch_size 2

        python scripts/detect_ImBD.py \
            --datanum 500 \
            --base_model ${M2} \
            --train_dataset ${train_dataset} \
            --eval_after_train \
            --eval_dataset $data_path/${D}_${M_test}_polish \
            --output_file $res_path/${D}_${M_test}_polish
    done
done