预训练数据处理遇到了一些问题

[yangxiaooooo]

我看到Readme里这张图里好像去掉了帧的头部

但是我自己预处理得到的数据没有去掉头部

我在代码里没有找到去除帧头部的代码，请问这怎么解决

[yangxiaooooo]

import os
import random
import pickle
import torch
from multiprocessing import Pool
from uer.utils.constants import *
from uer.utils.tokenizers import *
from uer.utils.misc import count_lines
from uer.utils.seed import set_seed

def mask_seq(src, tokenizer, whole_word_masking, span_masking, span_geo_prob, span_max_length):
vocab = tokenizer.vocab

for i in range(len(src) - 1, -1, -1):
    if src[i] != PAD_ID:
        break
src_no_pad = src[:i + 1]
tokens_index, src_no_pad = create_index(src_no_pad, tokenizer, whole_word_masking, span_masking, span_geo_prob, span_max_length)
if len(src_no_pad) < len(src):
    src = src_no_pad + (len(src) - len(src_no_pad)) * [PAD_ID]
else:
    src = src_no_pad

random.shuffle(tokens_index)
num_to_predict = max(1, int(round(len(src_no_pad) * 0.15))) 
tgt_mlm = []
for index_set in tokens_index:
    if len(tgt_mlm) >= num_to_predict:
        break
    if whole_word_masking:
        i = index_set[0]
        mask_len = index_set[1]
        if len(tgt_mlm) + mask_len > num_to_predict:
            continue

        for j in range(mask_len):
            token = src[i + j]
            tgt_mlm.append((i + j, token))
            prob = random.random()
            if prob < 0.8:
                src[i + j] = vocab.get(MASK_TOKEN)
            elif prob < 0.9:
                while True:
                    rdi = random.randint(1, len(vocab) - 1)
                    if rdi not in [vocab.get(CLS_TOKEN), vocab.get(SEP_TOKEN), vocab.get(MASK_TOKEN), PAD_ID]:
                        break
                src[i + j] = rdi
    elif span_masking:
        i = index_set[0]
        span_len = index_set[1]
        if len(tgt_mlm) + span_len > num_to_predict:
            continue

        for j in range(span_len):
            token = src[i + j]
            tgt_mlm.append((i + j, token))
        prob = random.random()
        if prob < 0.8:
            for j in range(span_len):
                src[i + j] = vocab.get(MASK_TOKEN)
        elif prob < 0.9:
            for j in range(span_len):
                while True:
                    rdi = random.randint(1, len(vocab) - 1)
                    if rdi not in [vocab.get(CLS_TOKEN), vocab.get(SEP_TOKEN), vocab.get(MASK_TOKEN), PAD_ID]:
                        break
                src[i + j] = rdi
    else:
        i = index_set[0]
        token = src[i]
        tgt_mlm.append((i, token))
        prob = random.random()
        if prob < 0.8: 
            src[i] = vocab.get(MASK_TOKEN)
        elif prob < 0.9: 
            while True:
                rdi = random.randint(1, len(vocab) - 1)
                if rdi not in [vocab.get(CLS_TOKEN), vocab.get(SEP_TOKEN), vocab.get(MASK_TOKEN), PAD_ID]:
                    break
            src[i] = rdi
tgt_mlm = sorted(tgt_mlm, key=lambda x: x[0])
return src, tgt_mlm

def create_index(src, tokenizer, whole_word_masking, span_masking, span_geo_prob, span_max_length):
tokens_index = []
span_end_position = -1
vocab = tokenizer.vocab
if whole_word_masking:
src_wwm = []
src_length = len(src)
has_cls, has_sep = False, False
if src[0] == vocab.get(CLS_TOKEN):
src = src[1:]
has_cls = True
if src[-1] == vocab.get(SEP_TOKEN):
src = src[:-1]
has_sep = True
sentence = "".join(tokenizer.convert_ids_to_tokens(src)).replace('[UNK]', '').replace('##', '')
import jieba
wordlist = jieba.cut(sentence)
if has_cls:
src_wwm += [vocab.get(CLS_TOKEN)]
for word in wordlist:
position = len(src_wwm)
src_wwm += tokenizer.convert_tokens_to_ids(tokenizer.tokenize(word))
if len(src_wwm) < src_length:
tokens_index.append([position, len(src_wwm)-position])
if has_sep:
src_wwm += [vocab.get(SEP_TOKEN)]
if len(src_wwm) > src_length:
src = src_wwm[:src_length]
else:
src = src_wwm
else:
for (i, token) in enumerate(src):
if token == vocab.get(CLS_TOKEN) or token == vocab.get(SEP_TOKEN) or token == PAD_ID:
continue
if not span_masking:
tokens_index.append([i])
else:
if i < span_end_position:
continue
span_len = get_span_len(span_max_length, span_geo_prob)
span_end_position = i + span_len
if span_end_position > len(src):
span_len = len(src) - i
tokens_index.append([i, span_len])
return tokens_index, src

def get_span_len(max_span_len, p):
geo_prob_cum = [0.0]
geo_prob = 1.0
for i in range(max_span_len + 1):
if i == 0:
continue
if i == 1:
geo_prob *= p
geo_prob_cum.append(geo_prob_cum[-1] + geo_prob)
else:
geo_prob *= (1 - p)
geo_prob_cum.append(geo_prob_cum[-1] + geo_prob)

prob = geo_prob_cum[-1] * random.random()
for i in range(len(geo_prob_cum) - 1):
    if prob >= geo_prob_cum[i] and prob < geo_prob_cum[i + 1]:
        current_span_len = i + 1
return current_span_len

def merge_dataset(dataset_path, workers_num):
# Merge datasets.
dataset_writer = open(dataset_path, "wb")
for i in range(workers_num):
tmp_dataset_reader = open("dataset-tmp-" + str(i) + ".pt", "rb")
while True:
tmp_data = tmp_dataset_reader.read(2^20)
if tmp_data:
dataset_writer.write(tmp_data)
else:
break
tmp_dataset_reader.close()
os.remove("dataset-tmp-" + str(i) + ".pt")
dataset_writer.close()

def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens):
""" truncate sequence pair to specific length """
while True:
total_length = len(tokens_a) + len(tokens_b)
if total_length <= max_num_tokens:
break
trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b
if random.random() < 0.5:
del trunc_tokens[0]
else:
trunc_tokens.pop()

def record_flow_start(corpus_path):
starts = []
with open(corpus_path, mode="r", encoding="utf-8") as f:
i = 0
while True:
line = f.readline()
if not line:
break
if line[:2] == "||":
starts.append(i)
i+=1
starts.append(i)
return starts

class Dataset(object):
def init(self, args, vocab, tokenizer):
self.vocab = vocab
self.tokenizer = tokenizer
self.corpus_path = args.corpus_path
self.dataset_path = args.dataset_path
self.seq_length = args.seq_length
self.seed = args.seed
self.dynamic_masking = args.dynamic_masking
self.whole_word_masking = args.whole_word_masking
self.span_masking = args.span_masking
self.span_geo_prob = args.span_geo_prob
self.span_max_length = args.span_max_length
self.docs_buffer_size = args.docs_buffer_size
self.dup_factor = args.dup_factor

def build_and_save(self, workers_num, split_by_flow=False):
    """
    Build dataset from the given corpus.
    Start workers_num processes and each process deals with a part of data.
    """
    lines_num = count_lines(self.corpus_path)
    print("Starting %d workers for building datasets ... " % workers_num)
    assert (workers_num >= 1)
    # 开多进程并行处理
    if workers_num == 1:
        self.worker(0, 0, lines_num)
    else:
        pool = Pool(workers_num)
        if split_by_flow:
            starts = record_flow_start(self.corpus_path)
        current_index = 0
        for i in range(workers_num):
            if split_by_flow:
                # start = starts[current_index]
                # for j in range(len(starts))[current_index:]:
                #     if starts[j]-starts[current_index]>perburst:
                #         current_index = j
                #         break
                # if i==workers_num-1:
                #     current_index = len(starts)-1
                # end = starts[current_index]
                start = starts[i*(len(starts)-1)//workers_num]
                end = starts[(i+1)*(len(starts)-1)//workers_num]
            else:
                start = i * lines_num // workers_num
                end = (i + 1) * lines_num // workers_num
            pool.apply_async(func=self.worker, args=[i, start, end])
        pool.close()
        pool.join()

    # Merge datasets.
    merge_dataset(self.dataset_path, workers_num)
    

def worker(self, proc_id, start, end):
    raise NotImplementedError()

class DataLoader(object):
def init(self, args, dataset_path, batch_size, proc_id, proc_num, shuffle=False):
self.tokenizer = args.tokenizer
self.batch_size = batch_size
self.instances_buffer_size = args.instances_buffer_size
self.proc_id = proc_id
self.proc_num = proc_num
self.shuffle = shuffle
self.dataset_reader = open(dataset_path, "rb")
self.read_count = 0
self.start = 0
self.end = 0
self.buffer = []
self.vocab = args.vocab
self.whole_word_masking = args.whole_word_masking
self.span_masking = args.span_masking
self.span_geo_prob = args.span_geo_prob
self.span_max_length = args.span_max_length

def _fill_buf(self):
    try:
        self.buffer = []
        while True:
            instance = pickle.load(self.dataset_reader)
            self.read_count += 1
            if (self.read_count - 1) % self.proc_num == self.proc_id: 
                self.buffer.append(instance)
                if len(self.buffer) >= self.instances_buffer_size:
                    break
    except EOFError:
        # Reach file end.
        self.dataset_reader.seek(0)

    if self.shuffle:
        random.shuffle(self.buffer)
    self.start = 0
    self.end = len(self.buffer)

def _empty(self):
    return self.start >= self.end

def __del__(self):
    self.dataset_reader.close()

class BertDataset(Dataset):
"""
Construct dataset for MLM and NSP tasks from the given corpus.
Each document consists of multiple sentences,
and each sentence occupies a single line.
Documents in corpus must be separated by empty lines.
"""

def __init__(self, args, vocab, tokenizer):
    super(BertDataset, self).__init__(args, vocab, tokenizer)
    self.short_seq_prob = args.short_seq_prob

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    set_seed(self.seed)
    docs_buffer = []
    document = []
    pos = 0
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    with open(self.corpus_path, mode="r", encoding="utf-8") as f:
        while pos < start:
            f.readline()
            pos += 1
        while True:
            line = f.readline()
            pos += 1
            if pos >= end:
                if len(docs_buffer) > 0:
                    instances = self.build_instances(docs_buffer)
                    for instance in instances:
                        pickle.dump(instance, dataset_writer)
                break

            if not line.strip(): 
                if len(document) >= 1:
                    docs_buffer.append(document)
                document = []
                if len(docs_buffer) == self.docs_buffer_size: 
                    print("Worker {}: Docs buffer is full, build instances...".format(proc_id))
                    # Build instances from documents.
                    instances = self.build_instances(docs_buffer)
                    # Save instances.
                    for instance in instances: 
                        pickle.dump(instance, dataset_writer)
                    # Clear buffer.
                    docs_buffer = []
                continue
            if line[:2] == "||":
                line = line[2:]
            sentence = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line)) 
            if len(sentence) > 0:
                document.append(sentence)

    dataset_writer.close()

def build_instances(self, all_documents):
    instances = []
    for _ in range(self.dup_factor):
        for doc_index in range(len(all_documents)):
            instances.extend(self.create_ins_from_doc(all_documents, doc_index))
    return instances

def create_ins_from_doc(self, all_documents, document_index):
    document = all_documents[document_index]
    max_num_tokens = self.seq_length - 3
    target_seq_length = max_num_tokens
    if random.random() < self.short_seq_prob:
        target_seq_length = random.randint(2, max_num_tokens)
    instances = []
    current_chunk = []
    current_length = 0
    i = 0
    while i < len(document):
        segment = document[i]
        current_chunk.append(segment)
        current_length += len(segment)
        if i == len(document) - 1 or current_length >= target_seq_length:
            if current_chunk:
                a_end = 1
                if len(current_chunk) >= 2: 
                    a_end = random.randint(1, len(current_chunk) - 1)

                tokens_a = [] #seg 1
                for j in range(a_end):
                    tokens_a.extend(current_chunk[j])

                tokens_b = [] ##seg 2
                is_random_next = 0

                if len(current_chunk) == 1 or random.random() < 0.5: 
                    is_random_next = 1
                    target_b_length = target_seq_length - len(tokens_a)

                    for _ in range(10):
                        random_document_index = random.randint(0, len(all_documents) - 1)
                        if random_document_index != document_index:
                            break

                    random_document = all_documents[random_document_index]
                    random_start = random.randint(0, len(random_document) - 1)
                    for j in range(random_start, len(random_document)):
                        tokens_b.extend(random_document[j])
                        if len(tokens_b) >= target_b_length:
                            break

                    num_unused_segments = len(current_chunk) - a_end 
                    i -= num_unused_segments

                else:
                    is_random_next = 0
                    for j in range(a_end, len(current_chunk)):
                        tokens_b.extend(current_chunk[j])

                truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)

                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))

                while len(src) != self.seq_length:
                    src.append(PAD_ID)

                if not self.dynamic_masking:
                    src, tgt_mlm = mask_seq(src, self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                    instance = (src, tgt_mlm, is_random_next, seg_pos)
                else:
                    instance = (src, is_random_next, seg_pos)

                instances.append(instance)
            current_chunk = []
            current_length = 0
        i += 1
    return instances

class BertFlowDataset(Dataset):
"""
Construct dataset for MLM and MIX tasks from the given corpus.
Each document consists of multiple paragraphs,
Each paragraph consists of multiple sentences,
and each sentence occupies a single line.
Paragraphs in corpus must be separated by empty lines.
Documents in corpus must be separated by empty lines.
"""

def __init__(self, args, vocab, tokenizer):
    super(BertFlowDataset, self).__init__(args, vocab, tokenizer)
    self.short_seq_prob = args.short_seq_prob

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    print(start,end)
    set_seed(self.seed)
    #flow流缓冲
    flow_buffer = []
    flow_proto = []
    # 数据包流缓冲
    docs_buffer = []
    document = []
    pos = 0
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    with open(self.corpus_path, mode="r", encoding="utf-8") as f:
        try:
            #with open(self.corpus_path[:-4]+"_extra.txt", mode="r", encoding="utf-8") as fe:
            while pos < start:
                f.readline()
                pos += 1
            while True:
                line = f.readline()
                if pos==start and line[:2] != "||":
                    print("not flow start...")
                pos += 1
                if pos > end:
                    if len(docs_buffer) >= 1:
                        flow_buffer.append(docs_buffer)
                        
                    if len(flow_buffer) > 0:
                        try:
                            instances = self.build_instances(flow_buffer,flow_proto)
                            print(instances)
                        except Exception as e:
                            print("has error2...", len(flow_buffer), len(flow_proto), e)
                        for instance in instances:
                            pickle.dump(instance, dataset_writer)
                    break

                if not line.strip():
                    if len(document) >= 1:
                        docs_buffer.append(document)
                    document = []
                    continue
                if line[:2] == "||" or not line: 
                    if len(docs_buffer) >= 1:
                        flow_buffer.append(docs_buffer)
                    docs_buffer = []
                
                    flow_buffer_size = 0
                    for d in flow_buffer:
                        flow_buffer_size += len(d)
                    if flow_buffer_size > self.docs_buffer_size or pos>=end: 
                        print("Worker %d is building instances ... " % proc_id, len(flow_buffer),len(flow_proto))
                        try:
                            instances = self.build_instances(flow_buffer,flow_proto)
                        except:
                            print("has error1...")
                        print("Worker {} has {} instances. ".format(proc_id,len(instances)))
                        # Save instances.
                        for instance in instances: 
                            pickle.dump(instance, dataset_writer)
                        flow_buffer = []
                        flow_proto = []
                    if pos>=end or not line:
                        break
                    line = line[2:]
                    if line[0]=="4":
                        if "bigram" in self.corpus_path:
                            if line[42:44] == "06":
                                flow_proto.append(0)
                            elif line[42:44] == "11":
                                flow_proto.append(1)
                            else:
                                print("not tcp or udp, ",line[42:44])
                        else:
                            # TCP
                            if line[18:20] == "06":
                                flow_proto.append(0)
                            # UDP
                            elif line[18:20] == "11":
                                flow_proto.append(1)
                            else:
                                print('error')
                    else:
                        print("find Ipv6!!")

                sentence = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line)) 
                if len(sentence) > 0:
                    document.append(sentence)

        except:
            print("has error...")
    print("Worker %d finished... " % proc_id)
    dataset_writer.close()

def build_instances(self, all_documents, flow_proto):
    assert len(all_documents) == len(flow_proto)
    instances = []
    for _ in range(self.dup_factor):
        for doc_index in range(len(all_documents)):
            # if doc_index%500==0:
            #     print(doc_index)
            instances.extend(self.create_ins_from_doc(all_documents, doc_index, flow_proto))
    return instances
def create_ins_from_doc(self, all_documents,document_index, flow_proto):
    document = all_documents[document_index] 
    max_num_tokens = self.seq_length - 3
    target_seq_length = max_num_tokens
    if random.random() < self.short_seq_prob: 
        target_seq_length = random.randint(2, max_num_tokens)
    instances = []
    i=0
    while i < len(document): 
        rnd = random.random()
        if i==len(document)-1 or rnd < 0.4: 
            a_end = 1
            if len(document[i]) >= 2: 
                a_end = random.randint(1, len(document[i]) - 1)

            tokens_a = [] #seg 1
            for j in range(a_end):
                tokens_a.extend(document[i][j])

            tokens_b = [] ##seg 2
            for j in range(a_end, len(document[i])):
                tokens_b.extend(document[i][j])
            
            truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)
            if random.random()<0.5:  #A1 A2
                label = 0 #
                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))
            else: #A2 A1
                label = 1
                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))
        elif rnd < 0.6: 
            tokens_a = []
            for j in range(len(document[i])):
                tokens_a.extend(document[i][j])
            next_burst_max_length = target_seq_length - len(tokens_a)

            for _ in range(20):
                random_document_index = random.randint(0, len(all_documents) - 1)
                if random_document_index != document_index:
                    break

            random_document = all_documents[random_document_index] 
            random_start = random.randint(0, len(random_document) - 1)
            burst_ind_end = random_start+1 #len(random_document) 
            tokens_b = []
            for burst_ind in range(random_start, burst_ind_end): 
                for j in range(len(random_document[burst_ind])): 
                    tokens_b.extend(random_document[burst_ind][j])
                    if len(tokens_b) >= next_burst_max_length:
                        break
            truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)
            if random_document_index == document_index:
                if random_start>=i:
                    label = 3 
                else:
                    label = 4
            else:
                label = 2 # A1A2 Z1Z2
            src = []
            src.append(self.vocab.get(CLS_TOKEN))
            src.extend(tokens_a)
            src.append(self.vocab.get(SEP_TOKEN))
            seg_pos = [len(src)]
            src.extend(tokens_b)
            src.append(self.vocab.get(SEP_TOKEN))
            seg_pos.append(len(src))
        else:
            tokens_a = []
            for j in range(len(document[i])):
                tokens_a.extend(document[i][j])
            i+=1
            tokens_b = []
            for j in range(len(document[i])):
                tokens_b.extend(document[i][j])
            truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)
            if random.random()<0.5:  #A1A2 B1B2
                label = 3 
                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))
            else: #B1B2 A1A2
                label = 4
                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))
        while len(src) != self.seq_length:
            src.append(PAD_ID)
        if not self.dynamic_masking:
            src, tgt_mlm = mask_seq(src, self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
            instance = (src, tgt_mlm, label, seg_pos)
        else:
            instance = (src, label, seg_pos)
        instance += (flow_proto[document_index],)
        instances.append(instance)
        i+=1 

    return instances

class BertDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt_mlm = []
        is_next = []
        seg = []

        masked_words_num = 0

        for ins in instances:
            if len(ins) == 4: 
                src.append(ins[0])
                masked_words_num += len(ins[1])
                tgt_mlm.append([0] * len(ins[0]))
                for mask in ins[1]:
                    tgt_mlm[-1][mask[0]] = mask[1]
                is_next.append(ins[2])
                seg.append([1] * ins[3][0] + [2] * (ins[3][1] - ins[3][0]) + [PAD_ID] * (len(ins[0]) - ins[3][1]))
            else:
                src_single, tgt_mlm_single = mask_seq(ins[0], self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                masked_words_num += len(tgt_mlm_single)
                src.append(src_single)
                tgt_mlm.append([0] * len(ins[0]))
                for mask in tgt_mlm_single:
                    tgt_mlm[-1][mask[0]] = mask[1]
                is_next.append(ins[1])
                seg.append([1] * ins[2][0] + [2] * (ins[2][1] - ins[2][0]) + [PAD_ID] * (len(ins[0]) - ins[2][1]))

        if masked_words_num == 0:
            continue

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt_mlm), \
            torch.LongTensor(is_next), \
            torch.LongTensor(seg)

class BertFlowDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt_mlm = []
        is_next = []
        seg = []
        proto = []

        masked_words_num = 0

        for ins in instances:
            if len(ins) == 5: 
                src.append(ins[0])
                masked_words_num += len(ins[1])
                tgt_mlm.append([0] * len(ins[0]))
                for mask in ins[1]:
                    tgt_mlm[-1][mask[0]] = mask[1]
                is_next.append(ins[2])
                seg.append([1] * ins[3][0] + [2] * (ins[3][1] - ins[3][0]) + [PAD_ID] * (len(ins[0]) - ins[3][1]))
                proto.append(ins[4])
            else:
                src_single, tgt_mlm_single = mask_seq(ins[0], self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                masked_words_num += len(tgt_mlm_single)
                src.append(src_single)
                tgt_mlm.append([0] * len(ins[0]))
                for mask in tgt_mlm_single:
                    tgt_mlm[-1][mask[0]] = mask[1]
                is_next.append(ins[1])
                seg.append([1] * ins[2][0] + [2] * (ins[2][1] - ins[2][0]) + [PAD_ID] * (len(ins[0]) - ins[2][1]))
                proto.append(ins[4])
        if masked_words_num == 0:
            continue

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt_mlm), \
            torch.LongTensor(is_next), \
            torch.LongTensor(seg), \
            torch.LongTensor(proto)

class MlmDataset(Dataset):
def init(self, args, vocab, tokenizer):
super(MlmDataset, self).init(args, vocab, tokenizer)
self.full_sentences = args.full_sentences

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    set_seed(self.seed)
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    docs_buffer = []
    for _ in range(self.dup_factor):
        pos = 0
        with open(self.corpus_path, mode="r", encoding="utf-8") as f:
            while pos < start:
                f.readline()
                pos += 1
            while True:
                line = f.readline()
                pos += 1

                document = [self.vocab.get(CLS_TOKEN)] + self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line)) + [self.vocab.get(SEP_TOKEN)]

                if self.full_sentences:
                    if len(document) > 0:
                        docs_buffer.append(document)
                    if len(docs_buffer) == self.docs_buffer_size:
                        # Build instances from documents.
                        all_documents = self.concatenate_docs(docs_buffer)
                        instances = self.build_instances(all_documents)
                        # Save instances.
                        for instance in instances:
                            pickle.dump(instance, dataset_writer)
                        # Clear buffer.
                        docs_buffer = []
                    if pos >= end:
                        if len(docs_buffer) > 0:
                            all_documents = self.concatenate_docs(docs_buffer)
                            instances = self.build_instances(all_documents)
                            # Save instances.
                            for instance in instances:
                                pickle.dump(instance, dataset_writer)
                        break
                else:
                    if len(document) > 0:
                        instances = self.build_instances(document)
                        # Save instances.
                        for instance in instances:
                            pickle.dump(instance, dataset_writer)

                if pos >= end:
                    break

    dataset_writer.close()

def concatenate_docs(self, docs_buffer):
    all_documents = []
    for i in range(len(docs_buffer)):
        all_documents += docs_buffer[i]
    return all_documents

def build_instances(self, all_documents):
    instances = []
    instances_num = len(all_documents) // self.seq_length
    for i in range(instances_num):
        src = all_documents[i * self.seq_length: (i + 1) * self.seq_length]
        seg_pos = [len(src)]

        if not self.dynamic_masking:
            src, tgt = mask_seq(src, self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
            instance = (src, tgt, seg_pos)
        else:
            instance = (src, seg_pos)

        instances.append(instance)

    src = all_documents[instances_num * self.seq_length:]
    seg_pos = [len(src)]

    while len(src) != self.seq_length:
        src.append(PAD_ID)

    if not self.dynamic_masking:
        src, tgt = mask_seq(src, self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
        instance = (src, tgt, seg_pos)
    else:
        instance = (src, seg_pos)

    instances.append(instance)
    return instances

class MlmDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt = []
        seg = []

        masked_words_num = 0

        for ins in instances:
            if len(ins) == 3:
                src.append(ins[0])
                masked_words_num += len(ins[1])
                tgt.append([0] * len(ins[0]))
                for mask in ins[1]:
                    tgt[-1][mask[0]] = mask[1]
                seg.append([1] * ins[2][0] + [PAD_ID] * (len(ins[0]) - ins[2][0]))
            else:
                src_single, tgt_single = mask_seq(ins[0], self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                masked_words_num += len(tgt_single)
                src.append(src_single)
                tgt.append([0] * len(ins[0]))
                for mask in tgt_single:
                    tgt[-1][mask[0]] = mask[1]
                seg.append([1] * ins[1][0] + [PAD_ID] * (len(ins[0]) - ins[1][0]))

        if masked_words_num == 0:
            continue

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt), \
            torch.LongTensor(seg)

class AlbertDataset(Dataset):
"""
Construct dataset for MLM and SOP tasks from the given corpus.
Each document consists of multiple sentences,
and each sentence occupies a single line.
Documents in corpus must be separated by empty lines.
"""

def __init__(self, args, vocab, tokenizer):
    super(AlbertDataset, self).__init__(args, vocab, tokenizer)
    self.short_seq_prob = args.short_seq_prob

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    set_seed(self.seed)
    document = []
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    for _ in range(self.dup_factor):
        pos = 0
        with open(self.corpus_path, mode="r", encoding="utf-8") as f:
            while pos < start:
                f.readline()
                pos += 1
            while True:
                line = f.readline()
                pos += 1
                if not line.strip():
                    if len(document) >= 1:
                        instances = self.build_instances(document)
                        for instance in instances:
                            pickle.dump(instance, dataset_writer)
                    document = []
                sentence = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line))
                if len(sentence) > 0:
                    document.append(sentence)
                if pos >= end - 1:
                    if len(document) >= 1:
                        instances = self.build_instances(document)
                        for instance in instances:
                            pickle.dump(instance, dataset_writer)
                    break
    dataset_writer.close()

def build_instances(self, document):
    instances = []
    instances.extend(self.create_ins_from_doc(document))
    return instances

def create_ins_from_doc(self, document):
    max_num_tokens = self.seq_length - 3
    target_seq_length = max_num_tokens
    if random.random() < self.short_seq_prob:
        target_seq_length = random.randint(2, max_num_tokens)
    instances = []
    current_chunk = []
    current_length = 0
    i = 0
    while i < len(document):
        segment = document[i]
        current_chunk.append(segment)
        current_length += len(segment)
        if i == len(document) - 1 or current_length >= target_seq_length:
            if current_chunk:
                a_end = 1
                if len(current_chunk) >= 2:
                    a_end = random.randint(1, len(current_chunk) - 1)

                tokens_a = []
                for j in range(a_end):
                    tokens_a.extend(current_chunk[j])

                tokens_b = []
                is_wrong_order = 0
                for j in range(a_end, len(current_chunk)):
                    tokens_b.extend(current_chunk[j])

                if random.random() < 0.5:
                    is_wrong_order = 1
                    tmp = tokens_a
                    tokens_a = tokens_b
                    tokens_b = tmp

                truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)

                src = []
                src.append(self.vocab.get(CLS_TOKEN))
                src.extend(tokens_a)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos = [len(src)]
                src.extend(tokens_b)
                src.append(self.vocab.get(SEP_TOKEN))
                seg_pos.append(len(src))

                while len(src) != self.seq_length:
                    src.append(PAD_ID)

                if not self.dynamic_masking:
                    src, tgt_mlm = mask_seq(src, self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                    instance = (src, tgt_mlm, is_wrong_order, seg_pos)
                else:
                    instance = (src, is_wrong_order, seg_pos)

                instances.append(instance)
            current_chunk = []
            current_length = 0
        i += 1
    return instances

class AlbertDataLoader(BertDataLoader):
'''
AlbertDataLoader can reuse the code of BertDataLoader.
'''
pass

class LmDataset(Dataset):
def worker(self, proc_id, start, end):
print("Worker %d is building dataset ... " % proc_id)
set_seed(self.seed)
dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
pos = 0
with open(self.corpus_path, mode="r", encoding="utf-8") as f:
while pos < start:
f.readline()
pos += 1
while True:
line = f.readline()
pos += 1

            document = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line))
            document = [self.vocab.get(CLS_TOKEN)] + document + [self.vocab.get(SEP_TOKEN)]

            instances_num = len(document) // (self.seq_length + 1)
            for i in range(instances_num):
                src = document[i * (self.seq_length + 1): (i + 1) * (self.seq_length + 1)]
                seg_pos = self.seq_length
                pickle.dump((src, seg_pos), dataset_writer)

            src = document[instances_num * (self.seq_length + 1):]
            if len(src) > 0:
                seg_pos = len(src)
                while len(src) != self.seq_length + 1:
                    src.append(PAD_ID)
                pickle.dump((src, seg_pos), dataset_writer)

            if pos >= end:
                break

    dataset_writer.close()

class LmDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt = []
        seg = []

        for ins in instances:
            src.append(ins[0][:-1])
            tgt.append(ins[0][1:])
            if ins[1] == len(ins[0]):
                seg.append([1] * (ins[1] - 1))
            else:
                seg.append([1] * ins[1] + [PAD_ID] * (len(ins[0]) - 1 - ins[1]))

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt), \
            torch.LongTensor(seg)

class BilmDataset(Dataset):
def worker(self, proc_id, start, end):
print("Worker %d is building dataset ... " % proc_id)
set_seed(self.seed)
dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
pos = 0
with open(self.corpus_path, mode="r", encoding="utf-8") as f:
while pos < start:
f.readline()
pos += 1
while True:
line = f.readline()
pos += 1

            document = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(line))

            instances_num = len(document) // self.seq_length
            for i in range(instances_num):
                src = document[i * self.seq_length: (i + 1) * self.seq_length]
                tgt_forward = src[1:] + [self.vocab.get(SEP_TOKEN)]
                tgt_backward = [self.vocab.get(CLS_TOKEN)] + src[:-1]
                seg = [1] * len(src)
                pickle.dump((src, tgt_forward, tgt_backward, seg), dataset_writer)

            src = document[instances_num * self.seq_length:]
            if len(src) < 1:
                continue
            tgt_forward = src[1:] + [self.vocab.get(SEP_TOKEN)]
            tgt_backward = [self.vocab.get(CLS_TOKEN)] + src[:-1]
            seg = [1] * len(src)
            while len(src) != self.seq_length:
                src.append(PAD_ID)
                tgt_forward.append(PAD_ID)
                tgt_backward.append(PAD_ID)
                seg.append(PAD_ID)
            pickle.dump((src, tgt_forward, tgt_backward, seg), dataset_writer)

            if pos >= end - 1:
                break

    dataset_writer.close()

class BilmDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt_forward = []
        tgt_backward = []
        seg = []

        for ins in instances:
            src.append(ins[0])
            tgt_forward.append(ins[1])
            tgt_backward.append(ins[2])
            seg.append(ins[3])

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt_forward), \
            torch.LongTensor(tgt_backward), \
            torch.LongTensor(seg)

class Seq2seqDataset(Dataset):
def init(self, args, vocab, tokenizer):
super(Seq2seqDataset, self).init(args, vocab, tokenizer)
self.tgt_seq_length = args.tgt_seq_length
self.src_vocab, self.src_tokenizer = vocab, tokenizer
self.tgt_tokenizer = args.tgt_tokenizer
self.tgt_vocab = self.tgt_tokenizer.vocab

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    set_seed(self.seed)
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    pos = 0
    with open(self.corpus_path, mode="r", encoding="utf-8") as f:
        while pos < start:
            f.readline()
            pos += 1
        while True:
            line = f.readline()
            pos += 1

            if len(line.strip().split("\t")) != 2:
                if pos >= end:
                    break
                continue
            document_src, document_tgt = line.strip().split("\t")
            src = self.src_tokenizer.convert_tokens_to_ids(self.src_tokenizer.tokenize(document_src))
            tgt = self.tgt_tokenizer.convert_tokens_to_ids(self.tgt_tokenizer.tokenize(document_tgt))

            src = [self.src_vocab.get(CLS_TOKEN)] + src + [self.src_vocab.get(SEP_TOKEN)]
            tgt = [self.tgt_vocab.get(CLS_TOKEN)] + tgt + [self.tgt_vocab.get(SEP_TOKEN)]
            seg = [1] * len(src)

            src, tgt, seg = src[:self.seq_length], tgt[:self.tgt_seq_length + 1], seg[:self.seq_length]
            while len(src) != self.seq_length:
                src.append(PAD_ID)
                seg.append(PAD_ID)
            while len(tgt) != self.tgt_seq_length + 1:
                tgt.append(PAD_ID)
            pickle.dump((src, tgt, seg), dataset_writer)

            if pos >= end:
                break

        dataset_writer.close()

class Seq2seqDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt_in = []
        tgt_out = []
        seg = []

        for ins in instances:
            src.append(ins[0])
            tgt_in.append(ins[1][:-1])
            tgt_out.append(ins[1][1:])
            seg.append(ins[2])

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt_in), \
            torch.LongTensor(tgt_out), \
            torch.LongTensor(seg)

class T5Dataset(MlmDataset):
'''
T5 can reuse the code of MlmDataset.
'''
pass

class T5DataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt_in = []
        tgt_out = []
        seg = []

        tgt_seq_length = 0

        for _, ins in enumerate(instances):
            if len(ins) == 3:
                src_single = ins[0]
                tgt_single = ins[1]
                seg.append([1] * ins[2][0] + [PAD_ID] * (len(ins[0]) - ins[2][0]))
            else:
                src_single, tgt_single = mask_seq(ins[0], self.tokenizer, self.whole_word_masking, self.span_masking, self.span_geo_prob, self.span_max_length)
                seg.append([1] * ins[1][0] + [PAD_ID] * (len(ins[0]) - ins[1][0]))

            MASK_ID = self.vocab.get(MASK_TOKEN)
            SENTINEL_ID = self.vocab.get(SENTINEL_TOKEN)

            for src_index, _ in tgt_single:
                if src_single[src_index] != MASK_ID:
                    src_single[src_index] = MASK_ID

            tgt_in_single = [self.vocab.get(CLS_TOKEN)]
            mask_index = 0
            src_with_sentinel = []
            for token_id in src_single:
                if token_id == MASK_ID:
                    if len(src_with_sentinel) > 0 and src_with_sentinel[-1] == (SENTINEL_ID - 1):
                        pass
                    else:
                        src_with_sentinel.append(SENTINEL_ID)
                        tgt_in_single.append(SENTINEL_ID)
                        SENTINEL_ID += 1
                    tgt_in_single.append(tgt_single[mask_index][1])
                    mask_index += 1
                else:
                    src_with_sentinel.append(token_id)
            tgt_in_single.append(SENTINEL_ID)
            tgt_in_single.append(self.vocab.get(SEP_TOKEN))

            while len(src_with_sentinel) < len(src_single):
                src_with_sentinel.append(PAD_ID)

            if len(tgt_in_single) > tgt_seq_length:
                tgt_seq_length = len(tgt_in_single)

            src.append(src_with_sentinel)
            tgt_in.append(tgt_in_single)
            tgt_out.append(tgt_in[-1][1:] + [PAD_ID])

        for i in range(len(tgt_in)):
            while len(tgt_in[i]) != tgt_seq_length:
                tgt_in[i].append(PAD_ID)
                tgt_out[i].append(PAD_ID)

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt_in), \
            torch.LongTensor(tgt_out), \
            torch.LongTensor(seg)

class ClsDataset(Dataset):
def worker(self, proc_id, start, end):
print("Worker %d is building dataset ... " % proc_id)
set_seed(self.seed)
f_write = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
pos = 0
with open(self.corpus_path, mode="r", encoding="utf-8") as f:
while pos < start:
line = f.readline()
pos += 1
while True:
line = f.readline()
pos += 1

            line = line.strip().split('\t')
            if len(line) == 2:
                label = int(line[0])
                text = " ".join(line[1:])
                src = [self.vocab.get(t) for t in self.tokenizer.tokenize(text)]
                src = [self.vocab.get(CLS_TOKEN)] + src
                tgt = label
                seg = [1] * len(src)
                if len(src) >= self.seq_length:
                    src = src[:self.seq_length]
                    seg = seg[:self.seq_length]
                else:
                    while len(src) != self.seq_length:
                        src.append(PAD_ID)
                        seg.append(PAD_ID)
                pickle.dump((src, tgt, seg), f_write)
            elif len(line) == 3:  # For sentence pair input.
                label = int(line[0])
                text_a, text_b = line[1], line[2]

                src_a = [self.vocab.get(t) for t in self.tokenizer.tokenize(text_a)]
                src_a = [self.vocab.get(CLS_TOKEN)] + src_a + [self.vocab.get(SEP_TOKEN)]
                src_b = [self.vocab.get(t) for t in self.tokenizer.tokenize(text_b)]
                src_b = src_b + [self.vocab.get(SEP_TOKEN)]

                src = src_a + src_b
                seg = [1] * len(src_a) + [2] * len(src_b)

                if len(src) >= self.seq_length:
                    src = src[:self.seq_length]
                    seg = seg[:self.seq_length]
                else:
                    while len(src) != self.seq_length:
                        src.append(PAD_ID)
                        seg.append(PAD_ID)
                pickle.dump((src, tgt, seg), f_write)
            else:
                pass

            if pos >= end - 1:
                break

    f_write.close()

class ClsDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt = []
        seg = []

        for ins in instances:
            src.append(ins[0])
            tgt.append(ins[1])
            seg.append(ins[2])

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt), \
            torch.LongTensor(seg)

class PrefixlmDataset(Dataset):

def worker(self, proc_id, start, end):
    print("Worker %d is building dataset ... " % proc_id)
    set_seed(self.seed)
    dataset_writer = open("dataset-tmp-" + str(proc_id) + ".pt", "wb")
    pos = 0
    with open(self.corpus_path, mode="r", encoding="utf-8") as f:
        while pos < start:
            f.readline()
            pos += 1
        while True:
            line = f.readline()
            pos += 1

            if len(line.strip().split("\t")) != 2:
                if pos >= end:
                    break
                continue
            document_src, document_tgt = line.strip().split("\t")
            src = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(document_src))
            tgt = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(document_tgt))
            src = [self.vocab.get(CLS_TOKEN)] + src + [self.vocab.get(SEP_TOKEN)]
            tgt = tgt + [self.vocab.get(SEP_TOKEN)]
            seg_pos = [len(src)]

            if seg_pos[0] >= self.seq_length:
                continue

            src = src + tgt
            tgt = [0] * seg_pos[0] + tgt[1:] + [PAD_ID]
            seg_pos.append(len(src))
            src, tgt = src[:self.seq_length], tgt[:self.seq_length]
            while len(src) != self.seq_length:
                src.append(PAD_ID)
                tgt.append(PAD_ID)
            if seg_pos[1] > self.seq_length:
                seg_pos[1] = self.seq_length

            pickle.dump((src, tgt, seg_pos), dataset_writer)

            if pos >= end:
                break

        dataset_writer.close()

class PrefixlmDataLoader(DataLoader):
def iter(self):
while True:
while self._empty():
self._fill_buf()
if self.start + self.batch_size >= self.end:
instances = self.buffer[self.start:]
else:
instances = self.buffer[self.start: self.start + self.batch_size]

        self.start += self.batch_size

        src = []
        tgt = []
        seg = []

        for ins in instances:
            src.append(ins[0])
            tgt.append(ins[1])
            seg.append([1] * ins[2][0] + [2] * (ins[2][1] - ins[2][0]) + [PAD_ID] * (len(ins[0]) - ins[2][1]))

        yield torch.LongTensor(src), \
            torch.LongTensor(tgt), \
            torch.LongTensor(seg)

这是我修改后的uer/utils/data.py文件

[whisperLiang]

大佬训练好了吗，效果怎么样？

[yangxiaooooo]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

[whisperLiang]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

太强了，想问下大佬有没有Cross-Platform (iOS and Android Apps)的数据集，我想对这个数据集进行预训练，但是网上下载地址失效了。

[SYX0817]

@whisperLiang 你好，我也碰到了这个问题，找不到Cross-Platform数据集的下载地址，请问你现在找到了吗，可以分享一下吗

[whisperLiang]

@whisperLiang 你好，我也碰到了这个问题，找不到Cross-Platform数据集的下载地址，请问你现在找到了吗，可以分享一下吗

没有找到，而且我发现这个方向是个坑，所以没有继续研究了

[Akwardxq]

@whisperLiang 你好，我也碰到了这个问题，找不到Cross-Platform数据集的下载地址，请问你现在找到了吗，可以分享一下吗

没有找到，而且我发现这个方向是个坑，所以没有继续研究了

你好，你指的方向是网络流量吗

[whisperLiang]

@whisperLiang 你好，我也碰到了这个问题，找不到Cross-Platform数据集的下载地址，请问你现在找到了吗，可以分享一下吗

没有找到，而且我发现这个方向是个坑，所以没有继续研究了

你好，你指的方向是网络流量吗

加密流量分类

[buxh2017]

你好，训练跑这个模型需要什么显卡才能跑？

[ZehuaShi]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

您好，我在使用preprocess.py生成dataset.pt的时候，总是遇到has error2...，然后退出。请问您知道这个应该怎么解决吗

[Akwardxq]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

您好，我在使用preprocess.py生成dataset.pt的时候，总是遇到has error2...，然后退出。请问您知道这个应该怎么解决吗

我第一步还在出问题，pretrain_data_gen.py这代码在我自己笔记本上运行能直接闪退。。

[Akwardxq]

该

我想知道你们怎么做pretrain_dataset_generation这第一步的？我下载的ISCX-NonVPN里面都是IPV6的流量包，get_bursts函数里全跳过了

[Akwardxq]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

大佬，我有问题...我想知道你怎么做pretrain_dataset_generation这第一步的？我下载的ISCX-NonVPN里面都是IPV6的流量包，get_bursts函数里全跳过了

[yangxiaooooo]

我没跑，转投ET-Bert了

[SylviaBCD]

大佬训练好了吗，效果怎么样？

我没完全跑一边，跑完预训练了，微调没跑

大佬，我有问题...我想知道你怎么做pretrain_dataset_generation这第一步的？我下载的ISCX-NonVPN里面都是IPV6的流量包，get_bursts函数里全跳过了

这个需要改一下utils.py里面的Splitexe文件地址，这是实验我跑了两周emmm