From e4e25df58c57a0a720d11952889ffb2cf6f0ebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=99=AF=E7=81=8F?= <1343084642@qq.com> Date: Wed, 8 Feb 2023 12:39:32 +0800 Subject: [PATCH] [d2l] dataloader test --- .../4_weight-decay.ipynb | 2 +- d2l/mindspore.py | 687 ++++++++++-------- 2 files changed, 367 insertions(+), 322 deletions(-) diff --git a/chapter_04_multilayer-perceptrons/4_weight-decay.ipynb b/chapter_04_multilayer-perceptrons/4_weight-decay.ipynb index 6b328c0..3069161 100644 --- a/chapter_04_multilayer-perceptrons/4_weight-decay.ipynb +++ b/chapter_04_multilayer-perceptrons/4_weight-decay.ipynb @@ -66,7 +66,7 @@ "outputs": [], "source": [ "n_train, n_test, num_inputs, batch_size = 20, 100, 200, 5\n", - "true_w, true_b = np.ones((num_inputs, 1)) * 0.01, 0.05\n", + "true_w, true_b = d2l.ones((num_inputs, 1)) * 0.01, 0.05\n", "train_data = d2l.synthetic_data(true_w, true_b, n_train)\n", "train_iter = d2l.load_array(train_data, batch_size)\n", "test_data = d2l.synthetic_data(true_w, true_b, n_test)\n", diff --git a/d2l/mindspore.py b/d2l/mindspore.py index 0efa27e..da096b3 100644 --- a/d2l/mindspore.py +++ b/d2l/mindspore.py @@ -29,12 +29,14 @@ DATA_HUB = dict() DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/' DATA_HUB['time_machine'] = (DATA_URL + 'timemachine.txt', - '090b5e7e70c295757f55df93cb0a180b9691891a') + '090b5e7e70c295757f55df93cb0a180b9691891a') DATA_HUB['fra-eng'] = (DATA_URL + 'fra-eng.zip', - '94646ad1522d915e7b0f9296181140edcf86a4f5') + '94646ad1522d915e7b0f9296181140edcf86a4f5') + class Timer: """记录多次运行时间。""" + def __init__(self): """Defined in :numref:`subsec_linear_model`""" self.times = [] @@ -61,8 +63,10 @@ def cumsum(self): """返回累计时间。""" return np.array(self.times).cumsum().tolist() + class Accumulator: """在`n`个变量上累加。""" + def __init__(self, n): self.data = [0.0] * n @@ -75,8 +79,10 @@ def reset(self): def __getitem__(self, idx): return self.data[idx] + class Animator: """在动画中绘制数据。""" + def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None, ylim=None, xscale='linear', yscale='linear', fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1, @@ -112,6 +118,7 @@ def add(self, x, y): display.display(self.fig) display.clear_output(wait=True) + class FashionMnist(): def __init__(self, path, kind): self.data, self.label = load_mnist(path, kind) @@ -122,17 +129,20 @@ def __getitem__(self, index): def __len__(self): return len(self.data) + class ArrayData(): def __init__(self, data): assert len(data) > 1 self.data = data def __getitem__(self, index): - return (i[index] for i in self.data) + # return (i[index] for i in self.data) + return [self.data[0][index], self.data[1][index]] def __len__(self): return len(self.data[0]) + class SGD(nn.Cell): def __init__(self, lr, batch_size, parameters): super().__init__() @@ -145,6 +155,7 @@ def construct(self, grads): ops.assign(self.parameters[idx], self.parameters[idx] - self.lr * grads[idx] / self.batch_size) return True + class Train(nn.Cell): def __init__(self, network, optimizer): super().__init__() @@ -158,6 +169,7 @@ def construct(self, *inputs): loss = ops.depend(loss, self.optimizer(grads)) return loss + class NetWithLoss(nn.Cell): def __init__(self, network, loss): super().__init__() @@ -169,6 +181,7 @@ def construct(self, *inputs): loss = self.loss(y_hat, inputs[-1]) return loss + class TrainCh8(nn.Cell): def __init__(self, network, optimizer, grad_op): super().__init__() @@ -184,6 +197,7 @@ def construct(self, *inputs): loss = ops.depend(loss, self.optimizer(grads)) return loss + class NetWithLossCh8(nn.Cell): def __init__(self, network, loss): super().__init__() @@ -207,6 +221,7 @@ def compute_kernel_size(inp_shape, output_size): kernel_height = math.ceil(kernel_height / output_size[1]) return (kernel_width, kernel_height) + class AdaptiveAvgPool2d(nn.Cell): def __init__(self, output_size=None): super().__init__() @@ -217,6 +232,7 @@ def construct(self, x): kernel_size = compute_kernel_size(inp_shape, self.output_size) return ops.AvgPool(kernel_size, kernel_size)(x) + class AdaptiveMaxPool2d(nn.Cell): def __init__(self, output_size=None): super().__init__() @@ -227,6 +243,7 @@ def construct(self, x): kernel_size = compute_kernel_size(inp_shape, self.output_size) return ops.MaxPool(kernel_size, kernel_size)(x) + class MaxPool2d(nn.Cell): def __init__(self, kernel_size, stride=None, padding=0): super().__init__() @@ -248,13 +265,16 @@ def construct(self, x): x = self.pad(x) return self.max_pool(x) + def linreg(x, w, b): return ops.matmul(x, w) + b + def squared_loss(y_hat, y): """均方损失。""" return (y_hat - y.reshape(y_hat.shape)) ** 2 / 2 + def load_mnist(path, kind='train'): import os import gzip @@ -263,10 +283,10 @@ def load_mnist(path, kind='train'): """Load MNIST data from `path`""" labels_path = os.path.join(path, '%s-labels-idx1-ubyte.gz' - %kind) + % kind) images_path = os.path.join(path, '%s-images-idx3-ubyte.gz' - %kind) + % kind) with gzip.open(labels_path, 'rb') as lbpath: labels = np.frombuffer(lbpath.read(), dtype=np.uint8, @@ -278,6 +298,7 @@ def load_mnist(path, kind='train'): return images, labels + def load_data_fashion_mnist(batch_size, resize=None, works=1): """将Fashion-MNIST数据集加载到内存中。""" data_path = "../data" @@ -298,6 +319,7 @@ def load_data_fashion_mnist(batch_size, resize=None, works=1): mnist_test = mnist_test.batch(batch_size, num_parallel_workers=works) return mnist_train, mnist_test + def get_fashion_mnist_labels(labels): """Return text labels for the Fashion-MNIST dataset. Defined in :numref:`sec_utils`""" @@ -305,16 +327,129 @@ def get_fashion_mnist_labels(labels): 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot'] return [text_labels[int(i)] for i in labels] + def load_array(data_arrays, batch_size, is_train=True): """Construct a PyTorch data iterator. Defined in :numref:`sec_utils`""" dataset = ArrayData(data_arrays) - data_column_size = len(data_arrays) - - dataset = ds.GeneratorDataset(source=dataset, column_names=[str(i) for i in range(data_column_size)], shuffle=is_train) - dataset = dataset.batch(batch_size) + # data_column_size = len(data_arrays) + # dataset = ds.GeneratorDataset(source=dataset, column_names=[str(i) for i in range(data_column_size)], shuffle=is_train) + dataset = GeneratorTensorDataset(dataset=dataset, batch_size=batch_size, shuffle=is_train) return dataset + +def default_collate(data): + unit_x = [] + unit_y = [] + for unit in data: + unit_x.append(unit[0]) + unit_y.append(unit[1]) + return d2l.stack(unit_x, 0), d2l.stack(unit_y, 0) + + +class GeneratorTensorDataset(): + def __init__(self, dataset, batch_size=1, shuffle=None, sampler=None, batch_sampler=None, collate_fn=None, + drop_last=False): + self.dataset = dataset + if sampler is None: + if shuffle: + sampler = RandomSampler(dataset) + else: + sampler = SequentialSampler(dataset) + + if batch_sampler is None: + batch_sampler = BatchSampler(sampler, batch_size, drop_last) + + self.batch_size = batch_size + self.drop_last = drop_last + self.sampler = sampler + self.batch_sampler = iter(batch_sampler) + if collate_fn is None: + collate_fn = default_collate + self.collate_fn = collate_fn + + def __next__(self): + index = next(self.batch_sampler) + data = [self.dataset[idx] for idx in index] + data = self.collate_fn(data) + return data + + def __iter__(self): + return self + + +class Sampler(object): + def __init__(self, data_source): + pass + + def __iter__(self): + raise NotImplementedError + + def __len__(self): + return len(self.data_source) + + +class SequentialSampler(Sampler): + def __init__(self, data_source): + super(SequentialSampler, self).__init__(data_source) + self.data_source = data_source + + def __iter__(self): + return iter(range(len(self.data_source))) + + def __len__(self): + return len(self.data_source) + + +class RandomSampler(Sampler): + def __init__(self, data_source, num_samples=None) -> None: + super(RandomSampler, self).__init__(data_source) + self.data_source = data_source + self._num_samples = num_samples + + if not isinstance(self.num_samples, int) or self.num_samples <= 0: + raise ValueError("num_samples should be a positive integer " + "value, but got num_samples={}".format(self.num_samples)) + + @property + def num_samples(self) -> int: + if self._num_samples is None: + return len(self.data_source) + return self._num_samples + + def __iter__(self): + n = len(self.data_source) + for _ in range(self.num_samples // n): + yield from np.random.permutation(n).tolist() + yield from np.random.permutation(n).tolist()[:self.num_samples % n] + + def __len__(self): + return self.num_samples + + +class BatchSampler(Sampler): + def __init__(self, sampler, batch_size, drop_last): + self.sampler = sampler + self.batch_size = batch_size + self.drop_last = drop_last + + def __iter__(self): + batch = [] + for idx in self.sampler: + batch.append(idx) + if len(batch) == self.batch_size: + yield batch + batch = [] + if len(batch) > 0 and not self.drop_last: + yield batch + + def __len__(self): + if self.drop_last: + return len(self.sampler) // self.batch_size + else: + return (len(self.sampler) + self.batch_size - 1) // self.batch_size + + def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5): """Plot a list of images. Defined in :numref:`sec_utils`""" @@ -333,27 +468,31 @@ def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5): ax.set_title(titles[i]) return axes + def use_svg_display(): """Use the svg format to display a plot in Jupyter. Defined in :numref:`sec_calculus`""" display.set_matplotlib_formats('svg') + def set_figsize(figsize=(3.5, 2.5)): """Set the figure size for matplotlib. Defined in :numref:`sec_calculus`""" use_svg_display() plt.rcParams['figure.figsize'] = figsize + def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend): """Set the axes for matplotlib. Defined in :numref:`sec_calculus`""" axes.set_xlabel(xlabel), axes.set_ylabel(ylabel) axes.set_xscale(xscale), axes.set_yscale(yscale) - axes.set_xlim(xlim), axes.set_ylim(ylim) + axes.set_xlim(xlim), axes.set_ylim(ylim) if legend: axes.legend(legend) axes.grid() + def plot(X, Y=None, xlabel=None, ylabel=None, legend=[], xlim=None, ylim=None, xscale='linear', yscale='linear', fmts=('-', 'm--', 'g-.', 'r:'), figsize=(3.5, 2.5), axes=None): @@ -376,15 +515,20 @@ def has_one_axis(X): # True if `X` (tensor or list) has 1 axis if axes is None: axes = plt.gca() axes.cla() for x, y, fmt in zip(X, Y, fmts): - axes.plot(x,y,fmt) if len(x) else axes.plot(y,fmt) + axes.plot(x, y, fmt) if len(x) else axes.plot(y, fmt) set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend) + def synthetic_data(w, b, num_examples): """生成 y = Xw + b + 噪声。""" - X = np.random.normal(0, 1, (num_examples, len(w))) - y = np.matmul(X, w) + b - y += np.random.normal(0, 0.01, y.shape) - return X.astype(np.float32), y.reshape((-1, 1)).astype(np.float32) + # X = np.random.normal(0, 1, (num_examples, len(w))) + X = d2l.normal((num_examples, len(w)), 0, 1) + # y = np.matmul(X, w) + b + y = d2l.matmul(X, w) + b + # y += np.random.normal(0, 0.01, y.shape) + y += d2l.normal(y.shape, 0, 0.01) + return X, y.reshape((-1, 1)) + def accuracy(y_hat, y): """计算预测正确的数量。""" @@ -393,6 +537,7 @@ def accuracy(y_hat, y): cmp = y_hat.asnumpy() == y.asnumpy() return float(cmp.sum()) + def evaluate_accuracy(net, dataset): """计算在指定数据集上模型的精度。""" metric = Accumulator(2) @@ -400,59 +545,55 @@ def evaluate_accuracy(net, dataset): metric.add(accuracy(net(X), y), y.size) return metric[0] / metric[1] + def train_epoch_ch3(net, dataset, loss, optim): """训练模型一个迭代周期（定义见第3章）。""" - # 定义前向网络 - def forward_fn(x, y): - y_hat = net(x) - l = loss(y_hat, y) - return l - batch_size = dataset.get_batch_size() + net_with_loss = nn.WithLossCell(net, loss) + net_train = nn.TrainOneStepCell(net_with_loss, optim) metric = Accumulator(3) - for X, y in dataset: - grad_fn = mindspore.value_and_grad(forward_fn, grad_position=None, weights=optim.parameters) - l, grads = grad_fn(X, y) + batch_size = dataset.get_batch_size() + + for X, y in dataset.create_tuple_iterator(): + l = net_train(X, y) y_hat = net(X) - optim(grads) metric.add(float(l.asnumpy()), accuracy(y_hat, y), y.size) return metric[0] / metric[2] * batch_size, metric[1] / metric[2] + def train_ch3(net, train_dataset, test_dataset, loss, num_epochs, optim): """训练模型（定义见第3章）。""" animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9], legend=['train loss', 'train acc', 'test acc']) - net.set_train() for epoch in range(num_epochs): train_metrics = train_epoch_ch3(net, train_dataset, loss, optim) - test_acc = evaluate_accuracy(net, test_dataset) print(train_metrics) + test_acc = evaluate_accuracy(net, test_dataset) animator.add(epoch + 1, train_metrics + (test_acc,)) train_loss, train_acc = train_metrics + def predict_ch3(net, dataset, n=6): """预测标签（定义见第3章）。""" for X, y in dataset.create_tuple_iterator(): break trues = get_fashion_mnist_labels(y.asnumpy()) preds = get_fashion_mnist_labels(net(X).argmax(axis=1).asnumpy()) - titles = [true +'\n' + pred for true, pred in zip(trues, preds)] + titles = [true + '\n' + pred for true, pred in zip(trues, preds)] show_images( X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n]) -def evaluate_loss(net, data_iter, loss): - """Evaluate the loss of a model on the given dataset. - Defined in :numref:`sec_model_selection`""" - if isinstance(net, nn.Cell): - net.set_train(False) - metric = d2l.Accumulator(2) # Sum of losses, no. of examples - for X, y in data_iter: - out = net(X) - y = d2l.reshape(y, out.shape) - l = loss(out, y) - metric.add(d2l.reduce_sum(l), l.size) +def evaluate_loss(net, loss, dataset): + """Evaluate the loss of a model on the given dataset. + Defined in :numref:`sec_utils`""" + metric = Accumulator(2) # Sum of losses, no. of examples + for X, y in dataset: + z = net(X) + l = loss(z, y) + metric.add(l.sum().asnumpy(), l.size) return metric[0] / metric[1] + def corr2d(X, K): """计算二维互相关运算。""" h, w = K.shape @@ -462,6 +603,7 @@ def corr2d(X, K): Y[i, j] = (X[i:i + h, j:j + w] * K).sum() return Y + def evaluate_accuracy_gpu(net, dataset, device=None): """使用GPU计算模型在数据集上的精度。""" net.set_train(False) @@ -470,34 +612,24 @@ def evaluate_accuracy_gpu(net, dataset, device=None): metric.add(accuracy(net(X), y), y.size) return metric[0] / metric[1] + def train_ch6(net, train_dataset, test_dataset, num_epochs, lr): """用GPU训练模型(在第六章定义)。""" optim = nn.SGD(net.trainable_params(), learning_rate=lr) - loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') - - # 定义前向传播函数 - def forward_fn(x, y): - y_hat = net(x) - loss = loss_fn(y_hat, y) - return loss, y_hat - grad_fn = ops.value_and_grad(forward_fn, None, weights=net.trainable_params(), has_aux=True) - - # 定义模型单步训练 - def train(X, Y, optim): - (loss, y_hat), grads = grad_fn(X, Y) - loss = ops.depend(loss, optim(grads)) - return loss, y_hat - + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + net_with_loss = nn.WithLossCell(net, loss) + train = nn.TrainOneStepCell(net_with_loss, optim) animator = Animator(xlabel='epoch', xlim=[1, num_epochs], - legend=['train loss', 'train acc', 'test acc']) + legend=['train loss', 'train acc', 'test acc']) timer, num_batches = Timer(), train_dataset.get_dataset_size() for epoch in range(num_epochs): metric = Accumulator(3) net.set_train() for i, (X, y) in enumerate(train_dataset.create_tuple_iterator()): timer.start() - loss, y_hat = train(X, y, optim) - metric.add(loss.asnumpy() * X.shape[0], accuracy(y_hat, y), X.shape[0]) + l = train(X, y) + y_hat = net(X) + metric.add(l.asnumpy() * X.shape[0], accuracy(y_hat, y), X.shape[0]) timer.stop() train_l = metric[0] / metric[2] train_acc = metric[1] / metric[2] @@ -510,6 +642,7 @@ def train(X, Y, optim): f'test acc {test_acc:.3f}') print(f'{metric[2] * num_epochs / timer.sum():.1f} examples/sec') + def download(name, cache_dir=os.path.join('..', 'data')): """下载一个DATA_HUB中的文件，返回本地文件名。 Defined in :numref:`sec_kaggle_house`""" @@ -533,6 +666,7 @@ def download(name, cache_dir=os.path.join('..', 'data')): f.write(r.content) return fname + def download_extract(name, folder=None): """下载并解压zip/tar文件。 Defined in :numref:`sec_kaggle_house`""" @@ -548,6 +682,7 @@ def download_extract(name, folder=None): fp.extractall(base_dir) return os.path.join(base_dir, folder) if folder else data_dir + def download_all(): """下载DATA_HUB中的所有文件。 Defined in :numref:`sec_kaggle_house`""" @@ -561,6 +696,7 @@ def read_time_machine(): lines = f.readlines() return [re.sub('[^A-Za-z]+', ' ', line).strip().lower() for line in lines] + def tokenize(lines, token='word'): """将文本行拆分为单词或字符词元。""" if token == 'word': @@ -570,27 +706,28 @@ def tokenize(lines, token='word'): else: print('错误：未知词元类型：' + token) + class Vocab: """文本词表""" + def __init__(self, tokens=None, min_freq=0, reserved_tokens=None): + """Defined in :numref:`sec_text_preprocessing`""" if tokens is None: tokens = [] if reserved_tokens is None: reserved_tokens = [] # 按出现频率排序 counter = count_corpus(tokens) - self._token_freqs = sorted(counter.items(), key=lambda x: x[1], - reverse=True) + self.token_freqs = sorted(counter.items(), key=lambda x: x[1], + reverse=True) # 未知词元的索引为0 - self.idx_to_token = [''] + reserved_tokens - self.token_to_idx = {token: idx - for idx, token in enumerate(self.idx_to_token)} - for token, freq in self._token_freqs: - if freq < min_freq: - break - if token not in self.token_to_idx: - self.idx_to_token.append(token) - self.token_to_idx[token] = len(self.idx_to_token) - 1 + self.unk, uniq_tokens = 0, [''] + reserved_tokens + uniq_tokens += [token for token, freq in self.token_freqs + if freq >= min_freq and token not in uniq_tokens] + self.idx_to_token, self.token_to_idx = [], dict() + for token in uniq_tokens: + self.idx_to_token.append(token) + self.token_to_idx[token] = len(self.idx_to_token) - 1 def __len__(self): return len(self.idx_to_token) @@ -605,61 +742,45 @@ def to_tokens(self, indices): return self.idx_to_token[indices] return [self.idx_to_token[index] for index in indices] - @property - def unk(self): # 未知词元的索引为0 - return 0 - - @property - def token_freqs(self): - return self._token_freqs def count_corpus(tokens): """统计词元的频率。""" - # 这里的tokens是1D列表或2D列表 if len(tokens) == 0 or isinstance(tokens[0], list): - # 将词元列表展平成一个列表 tokens = [token for line in tokens for token in line] return collections.Counter(tokens) + def load_corpus_time_machine(max_tokens=-1): """返回时光机器数据集的词元索引列表和词表。""" lines = read_time_machine() tokens = tokenize(lines, 'char') vocab = Vocab(tokens) - # 因为时光机器数据集中的每个文本行不一定是一个句子或一个段落， - # 所以将所有文本行展平到一个列表中 corpus = [vocab[token] for line in tokens for token in line] if max_tokens > 0: corpus = corpus[:max_tokens] return corpus, vocab + def seq_data_iter_random(corpus, batch_size, num_steps): """使用随机抽样生成一个小批量子序列。""" - # 从随机偏移量开始对序列进行分区，随机范围包括num_steps-1 corpus = corpus[random.randint(0, num_steps - 1):] - # 减去1，是因为我们需要考虑标签 num_subseqs = (len(corpus) - 1) // num_steps - # 长度为num_steps的子序列的起始索引 initial_indices = list(range(0, num_subseqs * num_steps, num_steps)) - # 在随机抽样的迭代过程中， - # 来自两个相邻的、随机的、小批量中的子序列不一定在原始序列上相邻 random.shuffle(initial_indices) def data(pos): - # 返回从pos位置开始的长度为num_steps的序列 return corpus[pos: pos + num_steps] num_batches = num_subseqs // batch_size for i in range(0, batch_size * num_batches, batch_size): - # 在这里，initial_indices包含子序列的随机起始索引 initial_indices_per_batch = initial_indices[i: i + batch_size] X = [data(j) for j in initial_indices_per_batch] Y = [data(j + 1) for j in initial_indices_per_batch] yield mindspore.Tensor(X, mindspore.int32), mindspore.Tensor(Y, mindspore.int32) - + + def seq_data_iter_sequential(corpus, batch_size, num_steps): """使用顺序分区生成一个小批量子序列。""" - # 从随机偏移量开始划分序列 offset = random.randint(0, num_steps) num_tokens = ((len(corpus) - offset - 1) // batch_size) * batch_size Xs = mindspore.Tensor(corpus[offset: offset + num_tokens], mindspore.int32) @@ -671,18 +792,22 @@ def seq_data_iter_sequential(corpus, batch_size, num_steps): Y = Ys[:, i: i + num_steps] yield X, Y + class SeqDataLoader: """加载序列数据的迭代器。""" + def __init__(self, batch_size, num_steps, use_random_iter, max_tokens): if use_random_iter: - self.data_iter_fn = d2l.seq_data_iter_random + self.data_iter_fn = seq_data_iter_random else: - self.data_iter_fn = d2l.seq_data_iter_sequential - self.corpus, self.vocab = d2l.load_corpus_time_machine(max_tokens) + self.data_iter_fn = seq_data_iter_sequential + self.corpus, self.vocab = load_corpus_time_machine(max_tokens) self.batch_size, self.num_steps = batch_size, num_steps def __iter__(self): return self.data_iter_fn(self.corpus, self.batch_size, self.num_steps) + + def load_data_time_machine(batch_size, num_steps, use_random_iter=False, max_tokens=10000): """返回时光机器数据集的迭代器和词表。""" @@ -690,20 +815,22 @@ def load_data_time_machine(batch_size, num_steps, batch_size, num_steps, use_random_iter, max_tokens) return data_iter, data_iter.vocab + def predict_ch8(prefix, num_preds, net, vocab): """在`prefix`后面生成新字符。""" net.set_train(False) state = net.begin_state(batch_size=1) outputs = [vocab[prefix[0]]] - get_input = lambda: d2l.reshape(Tensor([outputs[-1]], mindspore.int32), (1,1)) - for y in prefix[1:]: # 预热期 + get_input = lambda: mindspore.Tensor([outputs[-1]], mindspore.int32).reshape((1, 1)) + for y in prefix[1:]: _, state = net(get_input(), state) outputs.append(vocab[y]) - for _ in range(num_preds): # 预测num_preds步 + for _ in range(num_preds): y, state = net(get_input(), state) outputs.append(int(y.argmax(axis=1).reshape(1).asnumpy())) return ''.join([vocab.idx_to_token[i] for i in outputs]) + class TrainCh8(nn.Cell): def __init__(self, network, optimizer, theta): super().__init__() @@ -719,6 +846,7 @@ def construct(self, *inputs): loss = ops.depend(loss, self.optimizer(grads)) return loss + class NetWithLossCh8(nn.Cell): def __init__(self, network, loss): super().__init__() @@ -730,55 +858,34 @@ def construct(self, *inputs): loss = self.loss(y_hat, inputs[-1]) return loss -def grad_clipping(grads, theta): - """裁剪梯度。""" - norm = ops.sqrt(sum(ops.sum((g ** 2)) for g in grads)) - if norm > theta: - for g in grads: - g[:] *= theta / norm -def train_epoch_ch8(net, train_iter, loss, updater, use_random_iter): +def train_epoch_ch8(net, state, train_iter): """训练网络一个迭代周期（定义见第8章）。""" - state, timer = None, d2l.Timer() - metric = d2l.Accumulator(2) # 训练损失之和，词元数量 - # 定义前向函数 - def forward_fn(x, state, y): - y_hat, state = net(x, state) - l = loss(y_hat, y).mean() - return l - # 获取梯度函数 - grad_fn = ops.value_and_grad(forward_fn, None, weights=net.trainable_params()) - net.set_train() + timer = Timer() + metric = Accumulator(2) for X, Y in train_iter: - if state is None or use_random_iter: - # 在第一次迭代或使用随机抽样时初始化state - state = net.begin_state(batch_size=X.shape[0]) y = Y.T.reshape(-1) - (l), grads = grad_fn(X, state, y) - grad_clipping(grads, 1) - if isinstance(updater, nn.Optimizer): - updater(grads) - else: - # 因为已经调用了mean函数 - updater(batch_size=1) - metric.add(l.asnumpy() * d2l.size(y), d2l.size(y)) + l = net(X, state, y) + metric.add(l.asnumpy() * y.size, y.size) return math.exp(metric[0] / metric[1]), metric[1] / timer.stop() -def train_ch8(net, train_iter, vocab, lr, num_epochs, use_random_iter=False): - """训练模型（定义见第8章）""" - loss = nn.CrossEntropyLoss() - animator = d2l.Animator(xlabel='epoch', ylabel='perplexity', - legend=['train'], xlim=[10, num_epochs]) - # 初始化 - if isinstance(net, nn.Cell): - updater = nn.SGD(net.trainable_params(), lr) - else: - updater = lambda batch_size: d2l.sgd(net.params, lr, batch_size) + +def train_ch8(net, train_iter, vocab, lr, num_epochs): + """训练模型（定义见第8章）。""" + animator = Animator(xlabel='epoch', ylabel='perplexity', + legend=['train'], xlim=[10, num_epochs]) + + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + optim = nn.SGD(net.trainable_params(), lr) + net_with_loss = NetWithLossCh8(net, loss) + train = TrainCh8(net_with_loss, optim, 1) + predict = lambda prefix: predict_ch8(prefix, 50, net, vocab) - # 训练和预测 + + state = net.begin_state(train_iter.batch_size) for epoch in range(num_epochs): ppl, speed = train_epoch_ch8( - net, train_iter, loss, updater, use_random_iter) + train, state, train_iter) if (epoch + 1) % 10 == 0: print(predict('time traveller')) animator.add(epoch + 1, [ppl]) @@ -787,97 +894,109 @@ def train_ch8(net, train_iter, vocab, lr, num_epochs, use_random_iter=False): print(predict('time traveller')) print(predict('traveller')) + class RNNModel(nn.Cell): """循环神经网络模型。""" - def __init__(self, rnn_layer, vocab_size, **kwargs): + + def __init__(self, rnn_layer, vocab_size, num_hiddens, **kwargs): super(RNNModel, self).__init__(**kwargs) self.rnn = rnn_layer self.vocab_size = vocab_size - self.num_hiddens = self.rnn.hidden_size - # 如果RNN是双向的（之后将介绍），num_directions应该是2，否则应该是1 + self.num_hiddens = num_hiddens if not self.rnn.bidirectional: self.num_directions = 1 self.linear = nn.Dense(self.num_hiddens, self.vocab_size) else: self.num_directions = 2 self.linear = nn.Dense(self.num_hiddens * 2, self.vocab_size) - + self.on_value = Tensor(1.0, mindspore.float32) + self.off_value = Tensor(0.0, mindspore.float32) + def construct(self, inputs, state): - X = ops.one_hot(inputs.T, self.vocab_size, d2l.tensor(1.0), d2l.tensor(0.0)) + X = ops.OneHot()(inputs.T, self.vocab_size, self.on_value, self.off_value) Y, state = self.rnn(X, state) - # 全连接层首先将Y的形状改为(时间步数*批量大小,隐藏单元数) - # 它的输出形状是(时间步数*批量大小,词表大小)。 output = self.linear(Y.reshape((-1, Y.shape[-1]))) return output, state def begin_state(self, batch_size=1): if not isinstance(self.rnn, nn.LSTM): - # nn.GRU以张量作为隐状态 - return ops.zeros((self.num_directions * self.rnn.num_layers, - batch_size, self.num_hiddens)) + return mnp.zeros((self.num_directions * self.rnn.num_layers, + batch_size, self.num_hiddens)) else: - # nn.LSTM以元组作为隐状态 - return (ops.zeros(( - self.num_directions * self.rnn.num_layers, - batch_size, self.num_hiddens)), - ops.zeros(( + return (mnp.zeros(( + self.num_directions * self.rnn.num_layers, + batch_size, self.num_hiddens)), + mnp.zeros(( self.num_directions * self.rnn.num_layers, batch_size, self.num_hiddens))) + class RNNModelScratch(nn.Cell): """从零开始实现的循环神经网络模型""" + def __init__(self, vocab_size, num_hiddens, get_params, init_state, forward_fn): super().__init__() self.vocab_size, self.num_hiddens = vocab_size, num_hiddens self.params = get_params(vocab_size, num_hiddens) self.init_state, self.forward_fn = init_state, forward_fn - + self.on_value = Tensor(1.0, mindspore.float32) + self.off_value = Tensor(0.0, mindspore.float32) + def construct(self, X, state): - X = ops.one_hot(X.T, self.vocab_size, Tensor(1.0, mindspore.float32), Tensor(0.0, mindspore.float32)) + X = ops.OneHot()(X.T, self.vocab_size, self.on_value, self.off_value) return self.forward_fn(X, state, self.params) def begin_state(self, batch_size): return self.init_state(batch_size, self.num_hiddens) + class Encoder(nn.Cell): """编码器-解码器架构的基本编码器接口""" + def __init__(self, **kwargs): super(Encoder, self).__init__(**kwargs) - def construct(self, X, *args): + def construct(self, X): raise NotImplementedError + class Decoder(nn.Cell): """编码器-解码器架构的基本解码器接口""" + def __init__(self, **kwargs): super(Decoder, self).__init__(**kwargs) - def init_state(self, enc_outputs, *args): + def init_state(self, enc_outputs): raise NotImplementedError def construct(self, X, state): raise NotImplementedError + class EncoderDecoder(nn.Cell): """编码器-解码器架构的基类""" + def __init__(self, encoder, decoder, **kwargs): super(EncoderDecoder, self).__init__(**kwargs) self.encoder = encoder self.decoder = decoder - def construct(self, enc_X, dec_X, *args): - enc_outputs = self.encoder(enc_X, *args) - dec_state = self.decoder.init_state(enc_outputs, *args) - return self.decoder(dec_X, dec_state) + def construct(self, enc_X, dec_X, X_valid_len): + enc_outputs = self.encoder(enc_X, X_valid_len) + dec_state = self.decoder.init_state(enc_outputs, X_valid_len) + dec_outputs, state, _ = self.decoder(dec_X, dec_state) + return dec_outputs, state + def read_data_nmt(): """载入“英语－法语”数据集""" data_dir = download_extract('fra-eng') with open(os.path.join(data_dir, 'fra.txt'), 'r', - encoding='utf-8') as f: + encoding='utf-8') as f: return f.read() + def tokenize_nmt(text, num_examples=None): """词元化“英语－法语”数据数据集""" source, target = [], [] @@ -890,49 +1009,50 @@ def tokenize_nmt(text, num_examples=None): target.append(parts[1].split(' ')) return source, target + def preprocess_nmt(text): """预处理“英语－法语”数据集。""" + def no_space(char, prev_char): return char in set(',.!?') and prev_char != ' ' - # 使用空格替换不间断空格 - # 使用小写字母替换大写字母 text = text.replace('\u202f', ' ').replace('\xa0', ' ').lower() - # 在单词和标点符号之间插入空格 out = [' ' + char if i > 0 and no_space(char, text[i - 1]) else char for i, char in enumerate(text)] return ''.join(out) + def truncate_pad(line, num_steps, padding_token): """截断或填充文本序列""" if len(line) > num_steps: - return line[:num_steps] # 截断 - return line + [padding_token] * (num_steps - len(line)) # 填充 + return line[:num_steps] + return line + [padding_token] * (num_steps - len(line)) + def build_array_nmt(lines, vocab, num_steps): """将机器翻译的文本序列转换成小批量""" lines = [vocab[l] for l in lines] lines = [l + [vocab['']] for l in lines] - array = np.array([truncate_pad( - l, num_steps, vocab['']) for l in lines], dtype=np.int32) - valid_len = d2l.reduce_sum( - d2l.astype(array != vocab[''], np.int32), 1) + array = np.array([truncate_pad(l, num_steps, vocab['']) for l in lines]) + valid_len = (array != vocab['']).astype(np.int32).sum(1) return array, valid_len + def load_data_nmt(batch_size, num_steps, num_examples=600): """返回翻译数据集的迭代器和词表""" text = preprocess_nmt(read_data_nmt()) source, target = tokenize_nmt(text, num_examples) - src_vocab = d2l.Vocab(source, min_freq=2, - reserved_tokens=['', '', '']) - tgt_vocab = d2l.Vocab(target, min_freq=2, - reserved_tokens=['', '', '']) + src_vocab = Vocab(source, min_freq=2, + reserved_tokens=['', '', '']) + tgt_vocab = Vocab(target, min_freq=2, + reserved_tokens=['', '', '']) src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps) tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps) data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len) - data_iter = d2l.load_array(data_arrays, batch_size) + data_iter = load_array(data_arrays, batch_size) return data_iter, src_vocab, tgt_vocab + def show_heatmaps(matrices, xlabel, ylabel, titles=None, figsize=(2.5, 2.5), cmap='Reds'): """Show heatmaps of matrices. @@ -940,7 +1060,7 @@ def show_heatmaps(matrices, xlabel, ylabel, titles=None, figsize=(2.5, 2.5), use_svg_display() num_rows, num_cols = len(matrices), len(matrices[0]) fig, axes = plt.subplots(num_rows, num_cols, figsize=figsize, - sharex=True, sharey=True, squeeze=False) + sharex=True, sharey=True, squeeze=False) for i, (row_axes, row_matrices) in enumerate(zip(axes, matrices)): for j, (ax, matrix) in enumerate(zip(row_axes, row_matrices)): pcm = ax.imshow(matrix.asnumpy(), cmap=cmap) @@ -952,39 +1072,41 @@ def show_heatmaps(matrices, xlabel, ylabel, titles=None, figsize=(2.5, 2.5), ax.set_title(titles[j]) fig.colorbar(pcm, ax=axes, shrink=0.6) -class Seq2SeqEncoder(d2l.Encoder): + +class Seq2SeqEncoder(Encoder): """用于序列到序列学习的循环神经网络编码器""" + def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, dropout=0., **kwargs): super(Seq2SeqEncoder, self).__init__(**kwargs) - # 嵌入层 self.embedding = nn.Embedding(vocab_size, embed_size) self.rnn = nn.GRU(embed_size, num_hiddens, num_layers, dropout=dropout) - + self.num_hiddens = num_hiddens + self.num_layers = num_layers + def construct(self, X, X_len=None): - # 输出'X'的形状：(batch_size,num_steps,embed_size) + hx = mnp.zeros((self.num_layers, X.shape[0], self.num_hiddens), mindspore.float32) X = self.embedding(X) - # 在循环神经网络模型中，第一个轴对应于时间步 - X = X.permute(1, 0, 2) - output, state = self.rnn(X) + X = X.transpose(1, 0, 2) + output, state = self.rnn(X, hx) return output, state + class MaskedSoftmaxCELoss(nn.Cell): """带遮蔽的softmax交叉熵损失函数""" + def __init__(self): super().__init__() - self.softmax_ce_loss = nn.CrossEntropyLoss() + self.softmax_ce_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True) - # pred的形状：(batch_size,num_steps,vocab_size) - # label的形状：(batch_size,num_steps) - # valid_len的形状：(batch_size,) def construct(self, pred, label, valid_len): - weights = ops.ones_like(label) + weights = mnp.ones_like(label) weights = sequence_mask(weights, valid_len) - unweighted_loss = self.softmax_ce_loss(pred.permute(0, 2, 1), label) - weighted_loss = (unweighted_loss * weights).mean(axis=1) - return weighted_loss + unweighted_loss = self.softmax_ce_loss(pred.view(-1, pred.shape[-1]), label.view(-1)) + weighted_loss = (unweighted_loss.view(label.shape) * weights).mean(axis=1) + return weighted_loss.sum() + class NetWithLossCh8_Seq2seq(nn.Cell): def __init__(self, network, loss): @@ -997,36 +1119,31 @@ def construct(self, *inputs): loss = self.loss(y_hat, inputs[-2], inputs[-1]) return loss + def train_seq2seq(net, data_iter, lr, num_epochs, tgt_vocab): """训练序列到序列模型""" optimizer = nn.Adam(net.trainable_params(), lr) loss = MaskedSoftmaxCELoss() - animator = d2l.Animator(xlabel='epoch', ylabel='loss', - xlim=[10, num_epochs]) - def forward_fn(X, dec_input, X_valid_len, Y, Y_valid_len): - pred, _ = net(X, dec_input, X_valid_len) - l = loss(pred, Y, Y_valid_len) - return l - grad_fn = ops.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=False) - + net_with_loss = NetWithLossCh8_Seq2seq(net, loss) + train = TrainCh8(net_with_loss, optimizer, 1) + animator = Animator(xlabel='epoch', ylabel='loss', + xlim=[10, num_epochs]) for epoch in range(num_epochs): - timer = d2l.Timer() - metric = d2l.Accumulator(2) # 训练损失总和，词元数量 - net.set_train() + timer = Timer() + metric = Accumulator(2) for batch in data_iter: - X, X_valid_len, Y, Y_valid_len = [x.astype(d2l.int32) for x in batch] - # print(X.shape, X_valid_len, Y.shape, Y_valid_len) - bos = mindspore.Tensor([tgt_vocab['']] * Y.shape[0], dtype=mindspore.int32).reshape(-1, 1) - dec_input = ops.concat([bos, Y[:, :-1]], 1) # 强制教学 - l, grads = grad_fn(X, dec_input, X_valid_len, Y, Y_valid_len) - optimizer(grads) + X, X_valid_len, Y, Y_valid_len = [x.astype(mindspore.int32) for x in batch] + bos = mindspore.Tensor([tgt_vocab['']] * Y.shape[0], mindspore.int32).reshape(-1, 1) + dec_input = mnp.concatenate([bos, Y[:, :-1]], 1) + l = train(X, dec_input, X_valid_len, Y, Y_valid_len) num_tokens = Y_valid_len.sum() - metric.add(l.sum(), num_tokens) + metric.add(l.sum().asnumpy(), num_tokens.asnumpy()) if (epoch + 1) % 10 == 0: animator.add(epoch + 1, (metric[0] / metric[1],)) print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} ' - f'tokens/sec') + f'tokens/sec') + def bleu(pred_seq, label_seq, k): """计算BLEU""" @@ -1044,42 +1161,39 @@ def bleu(pred_seq, label_seq, k): score *= math.pow(num_matches / (len_pred - n + 1), math.pow(0.5, n)) return score + def predict_seq2seq(net, src_sentence, src_vocab, tgt_vocab, num_steps, save_attention_weights=False): """序列到序列模型的预测""" - # 在预测时将net设置为评估模式 net.set_train(False) src_tokens = src_vocab[src_sentence.lower().split(' ')] + [ src_vocab['']] - enc_valid_len = mindspore.Tensor([len(src_tokens)]) - src_tokens = d2l.truncate_pad(src_tokens, num_steps, src_vocab['']) - # 添加批量轴 - enc_X = ops.unsqueeze(mindspore.Tensor(src_tokens, mindspore.int32), 0) + enc_valid_len = mindspore.Tensor([len(src_tokens)], mindspore.int32) + src_tokens = truncate_pad(src_tokens, num_steps, src_vocab['']) + enc_X = mnp.expand_dims(mindspore.Tensor(src_tokens, mindspore.int32), 0) enc_outputs = net.encoder(enc_X, enc_valid_len) dec_state = net.decoder.init_state(enc_outputs, enc_valid_len) - # 添加批量轴 - dec_X = ops.unsqueeze(mindspore.Tensor([tgt_vocab['']], mindspore.int32), 0) + dec_X = mnp.expand_dims(mindspore.Tensor([tgt_vocab['']], mindspore.int32), 0) output_seq, attention_weight_seq = [], [] for _ in range(num_steps): - Y, dec_state = net.decoder(dec_X, dec_state) - # 我们使用具有预测最高可能性的词元，作为解码器在下一时间步的输入 + Y, dec_state, attention_weights = net.decoder(dec_X, dec_state) dec_X = Y.argmax(axis=2) pred = int(dec_X.squeeze(0).asnumpy()) - # 保存注意力权重（稍后讨论） if save_attention_weights: - attention_weight_seq.append(net.decoder.attention_weights) - # 一旦序列结束词元被预测，输出序列的生成就完成了 + attention_weight_seq.append(attention_weights) if pred == tgt_vocab['']: break output_seq.append(pred) return ' '.join(tgt_vocab.to_tokens(output_seq)), attention_weight_seq + def sequence_mask(X, valid_len, value=0): """在序列中屏蔽不相关的项""" maxlen = X.shape[1] - mask = ops.arange((maxlen), dtype=mindspore.float32)[None, :] < valid_len[:, None] + mask = mnp.arange((maxlen), dtype=mindspore.int32)[None, :] < valid_len[:, None] X[~mask] = value return X + def masked_softmax(X, valid_lens): """通过在最后一个轴上掩蔽元素来执行 softmax 操作""" if valid_lens is None: @@ -1091,11 +1205,13 @@ def masked_softmax(X, valid_lens): else: valid_lens = valid_lens.reshape(-1) X = sequence_mask(X.reshape(-1, shape[-1]), valid_lens, - value=-1e6) + value=-1e6) return ops.Softmax(-1)(X.reshape(shape)) + class AdditiveAttention(nn.Cell): """加性注意力""" + def __init__(self, key_size, query_size, num_hiddens, dropout, **kwargs): super(AdditiveAttention, self).__init__(**kwargs) self.W_k = nn.Dense(key_size, num_hiddens, has_bias=False) @@ -1112,20 +1228,24 @@ def construct(self, queries, keys, values, valid_lens): outputs = ops.BatchMatMul()(self.dropout(attention_weights), values) return outputs, attention_weights + class DotProductAttention(nn.Cell): """缩放点积注意力""" + def __init__(self, dropout, **kwargs): super(DotProductAttention, self).__init__(**kwargs) self.dropout = nn.Dropout(1 - dropout) def construct(self, queries, keys, values, valid_lens=None): d = queries.shape[-1] - scores = ops.BatchMatMul()(queries, keys.swapaxes(1,2)) / ops.Sqrt()(ops.ScalarToTensor()(d, mindspore.float32)) + scores = ops.BatchMatMul()(queries, keys.swapaxes(1, 2)) / ops.Sqrt()( + ops.ScalarToTensor()(d, mindspore.float32)) attention_weights = masked_softmax(scores, valid_lens) outputs = ops.BatchMatMul()(self.dropout(attention_weights), values) return outputs, attention_weights + def transpose_qkv(X, num_heads): """为了多注意力头的并行计算而变换形状。""" X = X.reshape(X.shape[0], X.shape[1], num_heads, -1) @@ -1134,14 +1254,17 @@ def transpose_qkv(X, num_heads): return X.reshape(-1, X.shape[2], X.shape[3]) + def transpose_output(X, num_heads): """逆转 `transpose_qkv` 函数的操作。""" X = X.reshape(-1, num_heads, X.shape[1], X.shape[2]) X = X.transpose(0, 2, 1, 3) return X.reshape(X.shape[0], X.shape[1], -1) + class MultiHeadAttention(nn.Cell): """多头注意力""" + def __init__(self, key_size, query_size, value_size, num_hiddens, num_heads, dropout, has_bias=False, **kwargs): super(MultiHeadAttention, self).__init__(**kwargs) @@ -1166,8 +1289,10 @@ def construct(self, queries, keys, values, valid_lens): output_concat = transpose_output(output, self.num_heads) return self.W_o(output_concat), attention_weights + class PositionalEncoding(nn.Cell): """位置编码""" + def __init__(self, num_hiddens, dropout, max_len=1000): super(PositionalEncoding, self).__init__() self.dropout = nn.Dropout(1 - dropout) @@ -1182,9 +1307,11 @@ def construct(self, X): X = X + self.P[:, :X.shape[1], :] return self.dropout(X) + class Dense(nn.Dense): def __init__(self, in_channels, out_channels, has_bias=True, activation=None): - super().__init__(in_channels, out_channels, weight_init='xavier_uniform', bias_init='zeros', has_bias=has_bias, activation=activation) + super().__init__(in_channels, out_channels, weight_init='xavier_uniform', bias_init='zeros', has_bias=has_bias, + activation=activation) self.reset_parameters() def reset_parameters(self): @@ -1193,18 +1320,22 @@ def reset_parameters(self): bound = 1 / math.sqrt(fan_in) self.bias.set_data(initializer(Uniform(bound), [self.out_channels])) + class Embedding(nn.Embedding): - def __init__(self, vocab_size, embedding_size, use_one_hot=False, embedding_table='normal', dtype=mindspore.float32, padding_idx=None): + def __init__(self, vocab_size, embedding_size, use_one_hot=False, embedding_table='normal', dtype=mindspore.float32, + padding_idx=None): if embedding_table == 'normal': embedding_table = Normal(1.0) super().__init__(vocab_size, embedding_size, use_one_hot, embedding_table, dtype, padding_idx) + @classmethod - def from_pretrained_embedding(cls, embeddings:Tensor, freeze=True, padding_idx=None): + def from_pretrained_embedding(cls, embeddings: Tensor, freeze=True, padding_idx=None): rows, cols = embeddings.shape embedding = cls(rows, cols, embedding_table=embeddings, padding_idx=padding_idx) embedding.embedding_table.requires_grad = not freeze return embedding + def train_transformer(net, data_iter, lr, num_epochs, tgt_vocab): """训练序列到序列模型""" @@ -1213,7 +1344,7 @@ def train_transformer(net, data_iter, lr, num_epochs, tgt_vocab): net_with_loss = NetWithLossCh8_Seq2seq(net, loss) train = TrainCh8(net_with_loss, optimizer, 1) animator = Animator(xlabel='epoch', ylabel='loss', - xlim=[10, num_epochs]) + xlim=[10, num_epochs]) for epoch in range(num_epochs): timer = Timer() metric = Accumulator(2) @@ -1229,7 +1360,8 @@ def train_transformer(net, data_iter, lr, num_epochs, tgt_vocab): if (epoch + 1) % 10 == 0: animator.add(epoch + 1, (metric[0] / metric[1],)) print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} ' - f'tokens/sec') + f'tokens/sec') + def predict_transformer(net, src_sentence, src_vocab, tgt_vocab, num_steps, save_attention_weights=False): """序列到序列模型的预测""" @@ -1254,6 +1386,7 @@ def predict_transformer(net, src_sentence, src_vocab, tgt_vocab, num_steps, save output_seq.append(pred) return ' '.join(tgt_vocab.to_tokens(output_seq)), attention_weight_seq, encoder_attention_weight + def train_2d(trainer, steps=20, f_grad=None): """Optimize a 2D objective function with a customized trainer. @@ -1270,110 +1403,26 @@ def train_2d(trainer, steps=20, f_grad=None): print(f'epoch {i + 1}, x1: {float(x1):f}, x2: {float(x2):f}') return results + def show_trace_2d(f, results): """Show the trace of 2D variables during optimization. Defined in :numref:`subsec_gd-learningrate`""" d2l.set_figsize() d2l.plt.plot(*zip(*results), '-o', color='#ff7f0e') - x1, x2 = d2l.meshgrid(d2l.arange(-5.5, 1.0, 0.1), - d2l.arange(-3.0, 1.0, 0.1)) + x1, x2 = d2l.meshgrid((d2l.arange(-5.5, 1.0, 0.1), + d2l.arange(-3.0, 1.0, 0.1))) d2l.plt.contour(x1, x2, f(x1, x2), colors='#1f77b4') d2l.plt.xlabel('x1') d2l.plt.ylabel('x2') -d2l.DATA_HUB['airfoil'] = (d2l.DATA_URL + 'airfoil_self_noise.dat', - '76e5be1548fd8222e5074cf0faae75edff8cf93f') - -def get_data_ch11(batch_size=10, n=1500): - """Defined in :numref:`sec_minibatches`""" - data = np.genfromtxt(d2l.download('airfoil'), - dtype=np.float32, delimiter='\t') - data = (data - data.mean(axis=0)) / data.std(axis=0) - data_iter = d2l.load_array((data[:n, :-1], data[:n, -1]), - batch_size, is_train=True) - return data_iter, data.shape[1]-1 - -def train_ch11(trainer_fn, states, hyperparams, data_iter, - feature_dim, num_epochs=2): - """Defined in :numref:`sec_minibatches`""" - # Initialization - w = mindspore.Parameter(d2l.normal(mean=0.0, stddev=0.01, shape=(feature_dim, 1))) - b = mindspore.Parameter(d2l.zeros((1))) - net, loss = lambda X: d2l.linreg(X, w, b), d2l.squared_loss - loss_fn = lambda x, y: loss(net(x), y).mean() - grad_fn = mindspore.value_and_grad(loss_fn, None, [w, b]) - # Train - animator = d2l.Animator(xlabel='epoch', ylabel='loss', - xlim=[0, num_epochs], ylim=[0.22, 0.35]) - n, timer = 0, d2l.Timer() - for _ in range(num_epochs): - for X, y in data_iter: - l, [dw, db] = grad_fn(X, y) - trainer_fn([w, b], [dw, db], states, hyperparams) - n += X.shape[0] - if n % 200 == 0: - timer.stop() - animator.add(n/X.shape[0]/data_iter.get_dataset_size(), - (d2l.evaluate_loss(net, data_iter, loss),)) - timer.start() - print(f'loss: {animator.Y[0][-1]:.3f}, {timer.avg():.3f} sec/epoch') - return timer.cumsum(), animator.Y[0] - -def train_concise_ch11(trainer_fn, hyperparams, data_iter, num_epochs=4): - """Defined in :numref:`sec_minibatches`""" - # Initialization - net = nn.Dense(5, 1) - def init_weights(m): - if type(m) == nn.Dense: - m.weight.set_data(initializer(Normal(0.01), m.weight.shape)) - net.apply(init_weights) - - optimizer = trainer_fn(net.trainable_params(), **hyperparams) - loss = nn.MSELoss(reduction='none') - forward_fn = lambda X, y: loss(net(X), y).mean() - - # Get gradient function - grad_fn = mindspore.value_and_grad(forward_fn, None, net.trainable_params()) - animator = d2l.Animator(xlabel='epoch', ylabel='loss', - xlim=[0, num_epochs], ylim=[0.22, 0.35]) - n, timer = 0, d2l.Timer() - for _ in range(num_epochs): - for X, y in data_iter: - output, grads = grad_fn(X, y) - optimizer(grads) - n += X.shape[0] - if n % 200 == 0: - timer.stop() - # MSELoss计算平方误差时不带系数1/2 - animator.add(n / X.shape[0] / data_iter.get_dataset_size(), - (d2l.evaluate_loss(net, data_iter, loss) / 2,)) - timer.start() - print(f'loss: {animator.Y[0][-1]:.3f}, {timer.avg():.3f} sec/epoch') - - -def show_list_len_pair_hist(legend, xlabel, ylabel, xlist, ylist): - """绘制列表长度对的直方图""" - d2l.set_figsize() - _, _, patches = d2l.plt.hist( - [[len(l) for l in xlist], [len(l) for l in ylist]]) - d2l.plt.xlabel(xlabel) - d2l.plt.ylabel(ylabel) - for patch in patches[1].patches: - patch.set_hatch('/') - d2l.plt.legend(legend) abs = ops.abs arange = ops.arange -randn = ops.randn concat = ops.concat -int32 = mindspore.int32 float32 = mindspore.float32 ones = ops.ones zeros = ops.zeros -inner = ops.inner -mv = ops.mv -mm = ops.mm matmul = ops.matmul stack = ops.stack sin = ops.sin @@ -1382,27 +1431,23 @@ def show_list_len_pair_hist(legend, xlabel, ylabel, xlist, ylist): cosh = ops.cosh tanh = ops.tanh exp = ops.exp -square = ops.square -sqrt = ops.sqrt -sign = ops.sign meshgrid = ops.meshgrid linspace = ops.linspace zeros_like = ops.zeros_like +sqrt = ops.sqrt log = ops.log maximum = ops.maximum relu = ops.relu sigmoid = ops.sigmoid norm = ops.norm -cat = ops.cat +sum = ops.sum pow = lambda x, y: ops.pow(x, y) clip_by_value = lambda x, clip_value_min, clip_value_max: ops.clip_by_value(x, clip_value_min, clip_value_max) uniform = lambda shape, minval, maxval: ops.uniform(shape, tensor(minval), tensor(maxval), dtype=float32) rand = lambda size, *args: ops.rand(size, dtype=float32) randn = lambda size, *args: ops.randn(size, dtype=float32) tensor = lambda x: mindspore.Tensor(x, dtype=mindspore.float32) -normal = lambda shape, mean, stddev, *args : ops.normal(shape, tensor(mean), tensor(stddev), *args) +normal = lambda shape, mean, stddev, *args: ops.normal(shape, tensor(mean), tensor(stddev), *args) reduce_sum = lambda x, *args, **kwargs: x.sum(*args, **kwargs) reshape = lambda x, *args, **kwargs: x.reshape(*args, **kwargs) transpose = lambda x, *args, **kwargs: x.t(*args, **kwargs) -size = lambda x, *args, **kwargs: x.numel(*args, **kwargs) -astype = lambda x, *args, **kwargs: x.astype(*args, **kwargs)