9、transformer
<pre><code>
# 变压器模型背后的核心思想是自我注意，即关注输入序列的不同位置，从而计算该序列的表示形式。
# Transformer创建了大量的自我注意层，下面将在按比例缩放的点积注意和多头注意小节中进行解释。
# 转换器模型使用自关注层栈来处理可变大小的输入，而不是使用RNNs或CNNs。
# 这种通用体系结构有许多优点:它不假设跨数据的时间/空间关系。
# 这对于处理一组对象(例如，星际争霸单位)是非常理想的。
# 层输出可以并行计算，而不是像RNN那样的串联计算。
# 遥远的项目可以相互影响对方的输出，而无需经过许多rn步骤或卷积层(例如，请参阅场景内存变压器)。
# 它可以学习长期依赖。
# 这在许多顺序任务中都是一个挑战。
# 这种体系结构的缺点是:对于时间序列，时间步长的输出是从整个历史记录计算出来的，而不是只计算输入和当前隐藏状态。
# 这可能效率较低。如果输入确实具有时间/空间关系，比如文本，则必须添加一些位置编码，否则模型将有效地看到一袋单词。
# 在训练了本笔记本中的模型后，您将能够输入一个葡萄牙语句子并返回英语翻译。
from __future__ import absolute_import, division, print_function, unicode_literals
# !pip install -q tensorflow-gpu==2.0.0-beta1
# pip install tensorflow_datasets
import tensorflow_datasets as tfds
import tensorflow as tf
import time
import numpy as np
import matplotlib.pyplot as plt
# ## Setup input pipeline
# Use [TFDS](https://www.tensorflow.org/datasets) to load the [Portugese-English translation dataset](https://github.com/neulab/word-embeddings-for-nmt) from the [TED Talks Open Translation Project](https://www.ted.com/participate/translate).
#
# This dataset contains approximately 50000 training examples, 1100 validation examples, and 2000 test examples.
# 加载数据，在home目录下
examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,
as_supervised=True)
train_examples, val_examples = examples['train'], examples['validation']
# Create a custom subwords tokenizer from the training dataset.
# 从训练数据中创建字典
tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(
(en.numpy() for pt, en in train_examples), target_vocab_size=2**13)
tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus(
(pt.numpy() for pt, en in train_examples), target_vocab_size=2**13)
# 演示 编解码过程
sample_string = 'Transformer is awesome.'
tokenized_string = tokenizer_en.encode(sample_string)
print ('Tokenized string is {}'.format(tokenized_string))
original_string = tokenizer_en.decode(tokenized_string)
print ('The original string: {}'.format(original_string))
assert original_string == sample_string
# 保存生成的 tokenizer
# import pickle ##导入pickle模块
# pickle_file = open('tokenizer_en.pkl','wb') ##注意打开方式一定要二进制形式打开
# pickle.dump(tokenizer_en, pickle_file) ##把列表永久保存到文件中
# pickle_file.close()
# pickle_file = open('tokenizer_pt.pkl','wb') ##注意打开方式一定要二进制形式打开
# pickle.dump(tokenizer_pt, pickle_file) ##把列表永久保存到文件中
# pickle_file.close()
# The tokenizer encodes the string by breaking it into subwords if the word is not in its dictionary.
# 如果字符串不在其字典中，记号赋予器将其分解为子单词，从而对字符串进行编码。
for ts in tokenized_string:
print ('{} ----> {}'.format(ts, tokenizer_en.decode([ts])))
# 7915 ----> T
# 1248 ----> ran
# 7946 ----> s
# 7194 ----> former
# 13 ----> is
# 2799 ----> awesome
# 7877 ----> .
BUFFER_SIZE = 20000
BATCH_SIZE = 64
# Add a start and end token to the input and target.
# 添加一个起始 结束位置
def encode(lang1, lang2):
lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(lang1.numpy()) + [tokenizer_pt.vocab_size+1]
lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(lang2.numpy()) + [tokenizer_en.vocab_size+1]
return lang1, lang2
# Note: To keep this example small and relatively fast, drop examples with a length of over 40 tokens.
# 去除长度超过40的句子
MAX_LENGTH = 40
def filter_max_length(x, y, max_length=MAX_LENGTH):
return tf.logical_and(tf.size(x) <= max_length,
tf.size(y) <= max_length)
# Operations inside `.map()` run in graph mode and receive a graph tensor that do not have a numpy attribute.
# The `tokenizer` expects a string or Unicode symbol to encode it into integers.
# Hence, you need to run the encoding inside a `tf.py_function`,
# which receives an eager tensor having a numpy attribute that contains the string value.
# map()中的操作以图形模式运行，并接收一个没有numpy属性的图形张量。
# 记号赋予器期望字符串或Unicode符号将其编码为整数。因此，您需要在tf中运行编码。py_function，
# 它接收一个热切张量，该张量具有包含字符串值的numpy属性。
def tf_encode(pt, en):
return tf.py_function(encode, [pt, en], [tf.int64, tf.int64])
# 过滤训练 、 验证数据
train_dataset = train_examples.map(tf_encode)
train_dataset = train_dataset.filter(filter_max_length)
# cache the dataset to memory to get a speedup while reading from it.
train_dataset = train_dataset.cache()
train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(
BATCH_SIZE, padded_shapes=([-1], [-1]))
# 创建一个“Dataset”，该“Dataset”预先从该数据集中获取元素。
train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)
val_dataset = val_examples.map(tf_encode)
val_dataset = val_dataset.filter(filter_max_length).padded_batch(
BATCH_SIZE, padded_shapes=([-1], [-1]))
# 从迭代器返回下一项。如果给出了默认值和迭代器
# 已耗尽，它将返回默认值，而不是引发StopIteration报错（）。
pt_batch, en_batch = next(iter(val_dataset))
# pt_batch, en_batch
# <tf.Tensor: id=311410, shape=(64, 40), dtype=int64, numpy=
# <tf.Tensor: id=311411, shape=(64, 40), dtype=int64, numpy=
# ## Positional encoding
#
# Since this model doesn't contain any recurrence or convolution,
# positional encoding is added to give the model some information about the relative position of the words in the sentence.
# The positional encoding vector is added to the embedding vector.
# Embeddings represent a token in a d-dimensional space where tokens with similar meaning will be closer to each other.
# But the embeddings do not encode the relative position of words in a sentence. So after adding the positional encoding,
# words will be closer to each other based on the *similarity of their meaning and their position in the sentence*,
# in the d-dimensional space.
#
# See the notebook on [positional encoding](https://github.com/tensorflow/examples/blob/master/community/en/position_encoding.ipynb) to learn more about it. The formula for calculating the positional encoding is as follows:
# 由于该模型不包含任何递归或卷积，因此添加了位置编码，从而为该模型提供关于单词在句子中的相对位置的信息。
# 将位置编码向量添加到嵌入向量中。嵌入表示在d维空间中的令牌，其中具有相似含义的令牌将彼此更接近。但是嵌入并不编码句子中单词的相对位置。因此，在添加位置编码之后，单词之间的距离会因为它们的意思和在句子中的位置的相似性而变得更近
# 得到角度
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))
return pos * angle_rates
# 位置编码
def positional_encoding(position, d_model):
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
# apply sin to even indices in the array; 2i
sines = np.sin(angle_rads[:, 0::2])
# apply cos to odd indices in the array; 2i+1
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = np.concatenate([sines, cosines], axis=-1)
pos_encoding = pos_encoding[np.newaxis, ...]
return tf.cast(pos_encoding, dtype=tf.float32)
pos_encoding = positional_encoding(50, 512)
print (pos_encoding.shape)
# plt.pcolormesh(pos_encoding[0], cmap='RdBu')
# plt.xlabel('Depth')
# plt.xlim((0, 512))
# plt.ylabel('Position')
# plt.colorbar()
# plt.show()
# ## Masking
# 屏蔽顺序批中的所有pad令牌。它确保模型不将填充作为输入。掩码指示pad值0出现的位置:在这些位置输出1，否则输出0。
# Mask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input.
# The mask indicates where pad value `0` is present: it outputs a `1` at those locations, and a `0` otherwise.
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
# add extra dimensions so that we can add the padding
# to the attention logits.
return seq[:, tf.newaxis, tf.newaxis, :] # (batch_size, 1, 1, seq_len)
x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])
create_padding_mask(x)
# <tf.Tensor: id=311427, shape=(3, 1, 1, 5), dtype=float32, numpy=***
# The look-ahead mask is used to mask the future tokens in a sequence.
# In other words, the mask indicates which entries should not be used.
#
# This means that to predict the third word, only the first and second word will be used.
# Similarly to predict the fourth word, only the first, second and the third word will be used and so on.
# 前瞻性掩码用于按顺序掩码未来的令牌。换句话说，掩码指示不应该使用哪些条目。 这意味着预测第三个单词时，
# 只使用第一个和第二个单词。与预测第四个单词相似，只有第一个、第二个和第三个单词会被使用，以此类推
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
return mask # (seq_len, seq_len)
x = tf.random.uniform((1, 3))
temp = create_look_ahead_mask(x.shape[1])
# temp
# <tf.Tensor: id=311443, shape=(3, 3), dtype=float32, numpy=
# array([[0., 1., 1.],
# [0., 0., 1.],
# [0., 0., 0.]], dtype=float32)>
# ## Scaled dot product attention
# 叉乘注意力
# The attention function used by the transformer takes three inputs: Q (query), K (key), V (value).
# The equation used to calculate the attention weights is:
# The dot-product attention is scaled by a factor of square root of the depth.
# This is done because for large values of depth,
# the dot product grows large in magnitude pushing the softmax function where it has small gradients resulting in a very hard softmax.
# For example, consider that `Q` and `K` have a mean of 0 and variance of 1.
# Their matrix multiplication will have a mean of 0 and variance of `dk`.
# Hence, *square root of `dk`* is used for scaling (and not any other number) because the matmul of `Q` and `K` should have a mean of 0 and variance of 1, so that we get a gentler softmax.
#
# The mask is multiplied with -1e9 (close to negative infinity).
# This is done because the mask is summed with the scaled matrix multiplication of Q and K and is applied immediately before a softmax.
# The goal is to zero out these cells, and large negative inputs to softmax are near zero in the output.
# 变压器使用的注意函数有三个输入:Q(查询)、K(键)、V(值)。计算注意权重的公式为:
# 点积注意力是由深度的平方根来衡量的。之所以这样做，是因为对于较大的深度值，点积的大小会增大，从而推动softmax函数，使其具有较小的梯度，从而产生非常硬的softmax。
# 例如，假设Q和K的均值为0，方差为1。它们的矩阵乘法的均值为0，方差为dk。因此，dk的平方根用于缩放(而不是其他任何数)，因为Q和K的matmul的均值应该是0，方差应该是1，这样我们就得到了一个更温和的softmax。
# 掩模乘以-1e9(接近负无穷)。之所以这样做，是因为掩码是由Q和K的缩放矩阵乘法求和的，并且应用于softmax之前。目标是使这些单元格归零，而softmax的大量负输入在输出中接近于零。
def scaled_dot_product_attention(q, k, v, mask):
"""Calculate the attention weights.
q, k, v must have matching leading dimensions.
k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.
The mask has different shapes depending on its type(padding or look ahead)
but it must be broadcastable for addition.
Args:
q: query shape == (..., seq_len_q, depth)
k: key shape == (..., seq_len_k, depth)
v: value shape == (..., seq_len_v, depth_v)
mask: Float tensor with shape broadcastable
to (..., seq_len_q, seq_len_k). Defaults to None.
Returns:
output, attention_weights
"""
matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k)
# scale matmul_qk
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
# add the mask to the scaled tensor.
if mask is not None:
scaled_attention_logits += (mask * -1e9)
# softmax is normalized on the last axis (seq_len_k) so that the scores
# add up to 1.
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # (..., seq_len_q, seq_len_k)
output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v)
return output, attention_weights
# As the softmax normalization is done on K, its values decide the amount of importance given to Q.
#
# The output represents the multiplication of the attention weights and the V (value) vector.
# This ensures that the words we want to focus on are kept as is and the irrelevant words are flushed out.
# 当对K进行softmax归一化时，它的值决定了Q的重要性。
# 输出表示注意权重与V(值)向量的乘积。这确保了我们想要关注的单词保持原样，不相关的单词被清除。
def print_out(q, k, v):
temp_out, temp_attn = scaled_dot_product_attention(
q, k, v, None)
print ('Attention weights are:')
print (temp_attn)
print ('Output is:')
print (temp_out)
np.set_printoptions(suppress=True)
temp_k = tf.constant([[10,0,0],
[0,10,0],
[0,0,10],
[0,0,10]], dtype=tf.float32) # (4, 3)
temp_v = tf.constant([[ 1,0],
[ 10,0],
[ 100,5],
[1000,6]], dtype=tf.float32) # (4, 2)
# Attention weights are:
# tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)
# Output is:
# tf.Tensor([[10. 0.]], shape=(1, 2), dtype=float32)
# This `query` aligns with the second `key`,
# so the second `value` is returned.
# 这个查询与一个重复键对齐(第三和第四)，
# 所有相关值都取平均值。
temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32) # (1, 3)
print_out(temp_q, temp_k, temp_v)
# Attention weights are:
# tf.Tensor([[0. 0. 0.5 0.5]], shape=(1, 4), dtype=float32)
# Output is:
# tf.Tensor([[550. 5.5]], shape=(1, 2), dtype=float32)
# This query aligns with a repeated key (third and fourth),
# so all associated values get averaged.
temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32) # (1, 3)
print_out(temp_q, temp_k, temp_v)
# Attention weights are:
# tf.Tensor([[0. 0. 0.5 0.5]], shape=(1, 4), dtype=float32)
# Output is:
# tf.Tensor([[550. 5.5]], shape=(1, 2), dtype=float32)
# This query aligns equally with the first and second key,
# so their values get averaged.
temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32) # (1, 3)
print_out(temp_q, temp_k, temp_v)
# Attention weights are:
# tf.Tensor([[0.5 0.5 0. 0. ]], shape=(1, 4), dtype=float32)
# Output is:
# tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)
# Pass all the queries together.
temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32) # (3, 3)
print_out(temp_q, temp_k, temp_v)
# Attention weights are:
# tf.Tensor(
# [[0. 0. 0.5 0.5]
# [0. 1. 0. 0. ]
# [0.5 0.5 0. 0. ]], shape=(3, 4), dtype=float32)
# Output is:
# tf.Tensor(
# [[550. 5.5]
# [ 10. 0. ]
# [ 5.5 0. ]], shape=(3, 2), dtype=float32)
# ## Multi-head attention
# # 多头注意包括四个部分:
# 线性层，并分裂成头。
# 按比例缩小的点积的注意。
# 连接头。
# 最终线性层。
#
# Multi-head attention consists of four parts:
# * Linear layers and split into heads.
# * Scaled dot-product attention.
# * Concatenation of heads.
# * Final linear layer.
# Each multi-head attention block gets three inputs; Q (query), K (key), V (value). These are put through linear (Dense) layers and split up into multiple heads.
#
# The `scaled_dot_product_attention` defined above is applied to each head (broadcasted for efficiency). An appropriate mask must be used in the attention step. The attention output for each head is then concatenated (using `tf.transpose`, and `tf.reshape`) and put through a final `Dense` layer.
#
# Instead of one single attention head, Q, K, and V are split into multiple heads because it allows the model to jointly attend to information at different positions from different representational spaces. After the split each head has a reduced dimensionality, so the total computation cost is the same as a single head attention with full dimensionality.
# 每个多头注意块有三个输入;Q(查询)，K(键)，V(值)。它们通过线性(密集的)层被分成多个头。
# 上面定义的scaled_dot_product_attention应用于每个头部(为提高效率而广播)。注意步骤中必须使用适当的口罩。
# 然后连接每个头部的注意力输出(使用tf)。转置，tf.整形)，并通过最后一个致密层。Q、K、V不是一个注意头，而是被分成多个注意头，
# 因为它允许模型从不同的表示空间的不同位置共同注意信息。在分割后，每个头的维数都减小了，因此总的计算成本与全维的单头注意相同。
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
self.wk = tf.keras.layers.Dense(d_model)
self.wv = tf.keras.layers.Dense(d_model)
self.dense = tf.keras.layers.Dense(d_model)
def split_heads(self, x, batch_size):
"""Split the last dimension into (num_heads, depth).
Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)
"""
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q) # (batch_size, seq_len, d_model)
k = self.wk(k) # (batch_size, seq_len, d_model)
v = self.wv(v) # (batch_size, seq_len, d_model)
q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)
k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth)
v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth)
# scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
# attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
scaled_attention, attention_weights = scaled_dot_product_attention(
q, k, v, mask)
scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) # (batch_size, seq_len_q, num_heads, depth)
concat_attention = tf.reshape(scaled_attention,
(batch_size, -1, self.d_model)) # (batch_size, seq_len_q, d_model)
output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model)
return output, attention_weights
# Create a `MultiHeadAttention` layer to try out. At each location in the sequence, `y`, the `MultiHeadAttention` runs all 8 attention heads across all other locations in the sequence, returning a new vector of the same length at each location.
# 创建一个多注意力层来尝试。在序列的每个位置y, MultiHeadAttention运行序列中所有其他位置的所有8个注意头，在每个位置返回一个长度相同的新向量。
temp_mha = MultiHeadAttention(d_model=512, num_heads=8)
y = tf.random.uniform((1, 60, 512)) # (batch_size, encoder_sequence, d_model)
out, attn = temp_mha(y, k=y, q=y, mask=None)
# out.shape, attn.shape
# (TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))
# ## Point wise feed forward network
# 点向前馈网络
# Point wise feed forward network consists of two fully-connected layers with a ReLU activation in between.
# 点向前馈网络由两个完全连接的层组成，层之间有一个ReLU激活。
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'), # (batch_size, seq_len, dff)
tf.keras.layers.Dense(d_model) # (batch_size, seq_len, d_model)
])
sample_ffn = point_wise_feed_forward_network(512, 2048)
# sample_ffn(tf.random.uniform((64, 50, 512))).shape
# TensorShape([64, 50, 512])
# ## Encoder and decoder
# 编码解码
# <img src="https://www.tensorflow.org/images/tutorials/transformer/transformer.png" width="600" alt="transformer">
# The transformer model follows the same general pattern as a standard [sequence to sequence with attention model](nmt_with_attention.ipynb).
#
# * The input sentence is passed through `N` encoder layers that generates an output for each word/token in the sequence.
# * The decoder attends on the encoder's output and its own input (self-attention) to predict the next word.
# 输入语句通过N个编码层传递，这些编码层为序列中的每个单词/令牌生成输出。
# 解码器关注编码器的输出和它自己的输入(自我注意)来预测下一个单词。
# ### Encoder layer
# Each encoder layer consists of sublayers:
# 1. Multi-head attention (with padding mask)
# 2. Point wise feed forward networks.
# Each of these sublayers has a residual connection around it followed by a layer normalization.
# Residual connections help in avoiding the vanishing gradient problem in deep networks.
# The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis.
# There are N encoder layers in the transformer.
# 每个编码器层由子层组成:
# 多头注意(带填充蒙版)
# 点向前馈网络。
# 每个子层周围都有一个剩余连接，然后进行层标准化。在深度网络中，剩余连接有助于避免梯度消失问题。
# 每个子层的输出是LayerNorm(x +子层(x))。规范化是在d_model(最后一个)轴上完成的。变压器中有N个编码器层。
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
attn_output, _ = self.mha(x, x, x, mask) # (batch_size, input_seq_len, d_model)
attn_output = self.dropout1(attn_output, training=training)
out1 = self.layernorm1(x + attn_output) # (batch_size, input_seq_len, d_model)
ffn_output = self.ffn(out1) # (batch_size, input_seq_len, d_model)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = self.layernorm2(out1 + ffn_output) # (batch_size, input_seq_len, d_model)
return out2
sample_encoder_layer = EncoderLayer(512, 8, 2048)
sample_encoder_layer_output = sample_encoder_layer(
tf.random.uniform((64, 43, 512)), False, None)
# sample_encoder_layer_output.shape # (batch_size, input_seq_len, d_model)
# TensorShape([64, 43, 512])
# ### Decoder layer
# Each decoder layer consists of sublayers:
# 1. Masked multi-head attention (with look ahead mask and padding mask)
# 2. Multi-head attention (with padding mask). V (value) and K (key) receive the *encoder output* as inputs. Q (query) receives the *output from the masked multi-head attention sublayer.*
# 3. Point wise feed forward networks
# Each of these sublayers has a residual connection around it followed by a layer normalization. The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis.
# There are N decoder layers in the transformer.
#
# As Q receives the output from decoder's first attention block, and K receives the encoder output,
# the attention weights represent the importance given to the decoder's input based on the encoder's output.
# In other words, the decoder predicts the next word by looking at the encoder output and self-attending to its own output.
# See the demonstration above in the scaled dot product attention section.
# 每个解码器层由子层组成:
# 蒙面多头注意(带有前瞻蒙版和填充蒙版)
# 多头注意(带填充蒙版)。V(值)和K(键)接收编码器输出作为输入。Q (query)接收蒙面多线程注意力子层的输出。
# 点向前馈网络
# 每个子层周围都有一个剩余连接，然后进行层标准化。每个子层的输出是LayerNorm(x +子层(x))。规范化是在d_model(最后一个)轴上完成的。
# 变压器中有N个译码层。
# 当Q接收到
class DecoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(DecoderLayer, self).__init__()
self.mha1 = MultiHeadAttention(d_model, num_heads)
self.mha2 = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.dropout3 = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
# enc_output.shape == (batch_size, input_seq_len, d_model)
attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask) # (batch_size, target_seq_len, d_model)
attn1 = self.dropout1(attn1, training=training)
out1 = self.layernorm1(attn1 + x)
attn2, attn_weights_block2 = self.mha2(
enc_output, enc_output, out1, padding_mask) # (batch_size, target_seq_len, d_model)
attn2 = self.dropout2(attn2, training=training)
out2 = self.layernorm2(attn2 + out1) # (batch_size, target_seq_len, d_model)
ffn_output = self.ffn(out2) # (batch_size, target_seq_len, d_model)
ffn_output = self.dropout3(ffn_output, training=training)
out3 = self.layernorm3(ffn_output + out2) # (batch_size, target_seq_len, d_model)
return out3, attn_weights_block1, attn_weights_block2
sample_decoder_layer = DecoderLayer(512, 8, 2048)
sample_decoder_layer_output, _, _ = sample_decoder_layer(
tf.random.uniform((64, 50, 512)), sample_encoder_layer_output,
False, None, None)
# sample_decoder_layer_output.shape # (batch_size, target_seq_len, d_model)
# TensorShape([64, 50, 512])
# ### Encoder
#
# The `Encoder` consists of:
# 1. Input Embedding
# 2. Positional Encoding
# 3. N encoder layers
#
# The input is put through an embedding which is summed with the positional encoding.
# The output of this summation is the input to the encoder layers. The output of the encoder is the input to the decoder.
# 编码器包括:
# 输入嵌入
# 位置编码
# N层编码器
# 输入通过嵌入来完成，嵌入用位置编码求和。这个求和的输出是编码器层的输入。编码器的输出是译码器的输入。
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
rate=0.1):
super(Encoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model)
self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
seq_len = tf.shape(x)[1]
# adding embedding and position encoding.
x = self.embedding(x) # (batch_size, input_seq_len, d_model)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
return x # (batch_size, input_seq_len, d_model)
sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8,
dff=2048, input_vocab_size=8500)
sample_encoder_output = sample_encoder(tf.random.uniform((64, 62)),
training=False, mask=None)
print (sample_encoder_output.shape) # (batch_size, input_seq_len, d_model)
# (64, 62, 512)
# ### Decoder
# The `Decoder` consists of:
# 1. Output Embedding
# 2. Positional Encoding
# 3. N decoder layers
#
# The target is put through an embedding which is summed with the positional encoding.
# The output of this summation is the input to the decoder layers. The output of the decoder is the input to the final linear layer.
# 解码器包括:
# 输出嵌入
# 位置编码
# N译码器层
# 对目标进行嵌入，嵌入后用位置编码求和。这个总和的输出是译码器层的输入。解码器的输出是最终线性层的输入。
class Decoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,
rate=0.1):
super(Decoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)
self.pos_encoding = positional_encoding(target_vocab_size, self.d_model)
self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
seq_len = tf.shape(x)[1]
attention_weights = {}
x = self.embedding(x) # (batch_size, target_seq_len, d_model)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x, block1, block2 = self.dec_layers[i](x, enc_output, training,
look_ahead_mask, padding_mask)
attention_weights['decoder_layer{}_block1'.format(i+1)] = block1
attention_weights['decoder_layer{}_block2'.format(i+1)] = block2
# x.shape == (batch_size, target_seq_len, d_model)
return x, attention_weights
# 45]:
sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8,
dff=2048, target_vocab_size=8000)
output, attn = sample_decoder(tf.random.uniform((64, 26)),
enc_output=sample_encoder_output,
training=False, look_ahead_mask=None,
padding_mask=None)
# output.shape, attn['decoder_layer2_block2'].shape
# (TensorShape([64, 26, 512]), TensorShape([64, 8, 26, 62]))
# ## Create the Transformer
# Transformer consists of the encoder, decoder and a final linear layer.
# The output of the decoder is the input to the linear layer and its output is returned.
# 变压器由编码器、解码器和最后的线性层组成。解码器的输出是线性层的输入，其输出被返回。
class Transformer(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
target_vocab_size, rate=0.1):
super(Transformer, self).__init__()
self.encoder = Encoder(num_layers, d_model, num_heads, dff,
input_vocab_size, rate)
self.decoder = Decoder(num_layers, d_model, num_heads, dff,
target_vocab_size, rate)
self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def call(self, inp, tar, training, enc_padding_mask,
look_ahead_mask, dec_padding_mask):
enc_output = self.encoder(inp, training, enc_padding_mask) # (batch_size, inp_seq_len, d_model)
# dec_output.shape == (batch_size, tar_seq_len, d_model)
dec_output, attention_weights = self.decoder(
tar, enc_output, training, look_ahead_mask, dec_padding_mask)
final_output = self.final_layer(dec_output) # (batch_size, tar_seq_len, target_vocab_size)
return final_output, attention_weights
sample_transformer = Transformer(
num_layers=2, d_model=512, num_heads=8, dff=2048,
input_vocab_size=8500, target_vocab_size=8000)
temp_input = tf.random.uniform((64, 62))
temp_target = tf.random.uniform((64, 26))
fn_out, _ = sample_transformer(temp_input, temp_target, training=False,
enc_padding_mask=None,
look_ahead_mask=None,
dec_padding_mask=None)
# fn_out.shape # (batch_size, tar_seq_len, target_vocab_size)
# TensorShape([64, 26, 8000])
# ## Set hyperparameters
# To keep this example small and relatively fast, the values for *num_layers, d_model, and dff* have been reduced.
#
# The values used in the base model of transformer were; *num_layers=6*, *d_model = 512*, *dff = 2048*. See the [paper](https://arxiv.org/abs/1706.03762) for all the other versions of the transformer.
#
# Note: By changing the values below, you can get the model that achieved state of the art on many tasks.
# 为了保持这个示例较小且相对较快，已经减少了num_layers、d_model和dff的值。
# 变压器基本模型中使用的值为;num_layers=6, d_model = 512, dff = 2048。有关变压器的所有其他版本，请参阅本文。
# 注意:通过更改下面的值，您可以得到在许多任务上实现了最先进状态的模型。
num_layers = 4
d_model = 128
dff = 512
num_heads = 8
input_vocab_size = tokenizer_pt.vocab_size + 2
target_vocab_size = tokenizer_en.vocab_size + 2
dropout_rate = 0.1
# ## Optimizer
# 根据本文公式，使用Adam优化器和自定义学习速率调度程序
# Use the Adam optimizer with a custom learning rate scheduler according to the formula in the [paper](https://arxiv.org/abs/1706.03762).
class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, d_model, warmup_steps=4000):
super(CustomSchedule, self).__init__()
self.d_model = d_model
self.d_model = tf.cast(self.d_model, tf.float32)
self.warmup_steps = warmup_steps
def __call__(self, step):
arg1 = tf.math.rsqrt(step)
arg2 = step * (self.warmup_steps ** -1.5)
return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
learning_rate = CustomSchedule(d_model)
optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,
epsilon=1e-9)
temp_learning_rate_schedule = CustomSchedule(d_model)
# plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))
# plt.ylabel("Learning Rate")
# plt.xlabel("Train Step")
# ## Loss and metrics
# Since the target sequences are padded, it is important to apply a padding mask when calculating the loss.
# 由于填充了目标序列，因此在计算损失时应用填充掩码非常重要。
#
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True, reduction='none')
def loss_function(real, pred):
mask = tf.math.logical_not(tf.math.equal(real, 0))
loss_ = loss_object(real, pred)
mask = tf.cast(mask, dtype=loss_.dtype)
loss_ *= mask
return tf.reduce_mean(loss_)
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name='train_accuracy')
# ## Training and checkpointing
#
transformer = Transformer(num_layers, d_model, num_heads, dff,
input_vocab_size, target_vocab_size, dropout_rate)
def create_masks(inp, tar):
# Encoder padding mask
enc_padding_mask = create_padding_mask(inp)
# Used in the 2nd attention block in the decoder.
# This padding mask is used to mask the encoder outputs.
dec_padding_mask = create_padding_mask(inp)
# Used in the 1st attention block in the decoder.
# It is used to pad and mask future tokens in the input received by
# the decoder.
look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])
dec_target_padding_mask = create_padding_mask(tar)
combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)
return enc_padding_mask, combined_mask, dec_padding_mask
# Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every `n` epochs.
checkpoint_path = "./checkpoints/train"
ckpt = tf.train.Checkpoint(transformer=transformer,
optimizer=optimizer)
ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)
# if a checkpoint exists, restore the latest checkpoint.
if ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint)
print ('Latest checkpoint restored!!')
# The target is divided into tar_inp and tar_real. tar_inp is passed as an input to the decoder. `tar_real` is that same input shifted by 1: At each location in `tar_input`, `tar_real` contains the next token that should be predicted.
#
# For example, `sentence` = "SOS A lion in the jungle is sleeping EOS"
#
# `tar_inp` = "SOS A lion in the jungle is sleeping"
#
# `tar_real` = "A lion in the jungle is sleeping EOS"
#
# The transformer is an auto-regressive model: it makes predictions one part at a time, and uses its output so far to decide what to do next.
#
# During training this example uses teacher-forcing (like in the [text generation tutorial](./text_generation.ipynb)). Teacher forcing is passing the true output to the next time step regardless of what the model predicts at the current time step.
#
# As the transformer predicts each word, *self-attention* allows it to look at the previous words in the input sequence to better predict the next word.
#
# To prevent the model from peaking at the expected output the model uses a look-ahead mask.
# 目标分为tar_inp和tar_real。tar_inp作为输入传递给解码器。tar_real是移动了1的相同输入:在tar_input中的每个位置，
# tar_real包含应该预测的下一个令牌。
# 例如,句子= " SOS狮子在丛林里睡EOS”tar_inp =“SOS狮子在丛林里睡觉”tar_real =“狮子在丛林里睡EOS”
# 变压器是一个自回归模型:它使预测的一部分,并使用其输出到目前为止决定下一步要做什么。在培训过程中，
# 这个例子使用了教师强制(就像在文本生成教程中一样)。教师强迫是将真实的输出传递到下一个时间步，而不管模型在当前时间步预测什么。
# 当转换器预测每个单词时，
# 自我注意允许它查看输入序列中的前一个单词，从而更好地预测下一个单词。为了防止模型在预期输出时达到峰值，模型使用了一个前瞻性掩码。
EPOCHS = 20
# The @tf.function trace-compiles train_step into a TF graph for faster
# execution. The function specializes to the precise shape of the argument
# tensors. To avoid re-tracing due to the variable sequence lengths or variable
# batch sizes (the last batch is smaller), use input_signature to specify
# more generic shapes.
# @tf。函数trace-将train_step编译到TF图中，以便更快
# #执行。该函数专门用于参数的精确形状
# #张量。避免由于可变的序列长度或变量而重新跟踪
# #批大小(最后一个批比较小)，使用input_signature来指定
# 更通用的形状。
train_step_signature = [
tf.TensorSpec(shape=(None, None), dtype=tf.int64),
tf.TensorSpec(shape=(None, None), dtype=tf.int64),
]
@tf.function(input_signature=train_step_signature)
def train_step(inp, tar):
tar_inp = tar[:, :-1]
tar_real = tar[:, 1:]
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)
with tf.GradientTape() as tape:
predictions, _ = transformer(inp, tar_inp,
True,
enc_padding_mask,
combined_mask,
dec_padding_mask)
loss = loss_function(tar_real, predictions)
gradients = tape.gradient(loss, transformer.trainable_variables)
optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))
train_loss(loss)
train_accuracy(tar_real, predictions)
# Portuguese is used as the input language and English is the target language.
# 输入语言为葡萄牙语，目标语言为英语。
#
for epoch in range(EPOCHS):
start = time.time()
train_loss.reset_states()
train_accuracy.reset_states()
# inp -> portuguese, tar -> english
for (batch, (inp, tar)) in enumerate(train_dataset):
train_step(inp, tar)
if batch % 50 == 0:
print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(
epoch + 1, batch, train_loss.result(), train_accuracy.result()))
if (epoch + 1) % 5 == 0:
ckpt_save_path = ckpt_manager.save()
print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,
ckpt_save_path))
print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1,
train_loss.result(),
train_accuracy.result()))
print ('Time taken for 1 epoch: {} secs\n'.format(time.time() - start))
# Epoch 20 Batch 650 Loss 0.5743 Accuracy 0.3407
# Epoch 20 Batch 700 Loss 0.5739 Accuracy 0.3412
# Saving checkpoint for epoch 20 at ./checkpoints/train\ckpt-4
# Epoch 20 Loss 0.5736 Accuracy 0.3412
# Time taken for 1 epoch: 53.0600483417511 secs
# ## Evaluate
# The following steps are used for evaluation:
#
# * Encode the input sentence using the Portuguese tokenizer (`tokenizer_pt`). Moreover, add the start and end token so the input is equivalent to what the model is trained with. This is the encoder input.
# * The decoder input is the `start token == tokenizer_en.vocab_size`.
# * Calculate the padding masks and the look ahead masks.
# * The `decoder` then outputs the predictions by looking at the `encoder output` and its own output (self-attention).
# * Select the last word and calculate the argmax of that.
# * Concatentate the predicted word to the decoder input as pass it to the decoder.
# * In this approach, the decoder predicts the next word based on the previous words it predicted.
#
# Note: The model used here has less capacity to keep the example relatively faster so the predictions maybe less right. To reproduce the results in the paper, use the entire dataset and base transformer model or transformer XL, by changing the hyperparameters above.
# 下面的步骤用于求值:使用葡萄牙记号赋予器(tokenizer_pt)对输入语句进行编码。
# 此外，添加start和end令牌，这样输入就相当于用什么来训练模型。这是编码器的输入。
# 解码器输入是start令牌== tokenizer_en.vocab_size。计算填充蒙版和前瞻性蒙版。
# 然后解码器通过查看编码器的输出和它自己的输出(自我注意)来输出预测。选择最后一个单词并计算它的argmax。
# 将预测的单词连接到解码器输入，并将其传递给解码器。在这种方法中，解码器根据之前预测的单词预测下一个单词。
# 注意:这里使用的模型保持示例相对较快的能力较小，
# 因此预测可能不太准确。为了重现本文的结果，使用整个数据集和基础变压器模型或变压器XL，通过改变上述超参数。
def evaluate(inp_sentence):
start_token = [tokenizer_pt.vocab_size]
end_token = [tokenizer_pt.vocab_size + 1]
# inp sentence is portuguese, hence adding the start and end token
inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token
encoder_input = tf.expand_dims(inp_sentence, 0)
# as the target is english, the first word to the transformer should be the
# english start token.
decoder_input = [tokenizer_en.vocab_size]
output = tf.expand_dims(decoder_input, 0)
for i in range(MAX_LENGTH):
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
encoder_input, output)
# predictions.shape == (batch_size, seq_len, vocab_size)
predictions, attention_weights = transformer(encoder_input,
output,
False,
enc_padding_mask,
combined_mask,
dec_padding_mask)
# select the last word from the seq_len dimension
predictions = predictions[: ,-1:, :] # (batch_size, 1, vocab_size)
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
# return the result if the predicted_id is equal to the end token
if tf.equal(predicted_id, tokenizer_en.vocab_size+1):
return tf.squeeze(output, axis=0), attention_weights
# concatentate the predicted_id to the output which is given to the decoder
# as its input.
output = tf.concat([output, predicted_id], axis=-1)
return tf.squeeze(output, axis=0), attention_weights
def plot_attention_weights(attention, sentence, result, layer):
fig = plt.figure(figsize=(16, 8))
sentence = tokenizer_pt.encode(sentence)
attention = tf.squeeze(attention[layer], axis=0)
for head in range(attention.shape[0]):
ax = fig.add_subplot(2, 4, head+1)
# plot the attention weights
ax.matshow(attention[head][:-1, :], cmap='viridis')
fontdict = {'fontsize': 10}
ax.set_xticks(range(len(sentence)+2))
ax.set_yticks(range(len(result)))
ax.set_ylim(len(result)-1.5, -0.5)
ax.set_xticklabels(
['<start>']+[tokenizer_pt.decode([i]) for i in sentence]+['<end>'],
fontdict=fontdict, rotation=90)
ax.set_yticklabels([tokenizer_en.decode([i]) for i in result
if i < tokenizer_en.vocab_size],
fontdict=fontdict)
ax.set_xlabel('Head {}'.format(head+1))
plt.tight_layout()
plt.show()
def translate(sentence, plot=''):
result, attention_weights = evaluate(sentence)
predicted_sentence = tokenizer_en.decode([i for i in result
if i < tokenizer_en.vocab_size])
print('Input: {}'.format(sentence))
print('Predicted translation: {}'.format(predicted_sentence))
if plot:
plot_attention_weights(attention_weights, sentence, result, plot)
# 64]:
translate("este é um problema que temos que resolver.")
print ("Real translation: this is a problem we have to solve .")
# 65]:
translate("os meus vizinhos ouviram sobre esta ideia.")
print ("Real translation: and my neighboring homes heard about this idea .")
# 66]:
translate("vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.")
print ("Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .")
# You can pass different layers and attention blocks of the decoder to the `plot` parameter.
# 67]:
translate("este é o primeiro livro que eu fiz.", plot='decoder_layer4_block2')
print ("Real translation: this is the first book i've ever done.")
# ## Summary
#
# In this tutorial, you learned about positional encoding, multi-head attention, the importance of masking and how to create a transformer.
#
# Try using a different dataset to train the transformer. You can also create the base transformer or transformer XL by changing the hyperparameters above. You can also use the layers defined here to create [BERT](https://arxiv.org/abs/1810.04805) and train state of the art models. Futhermore, you can implement beam search to get better predictions.
# 在本教程中，您学习了位置编码、多头注意、掩蔽的重要性以及如何创建转换器。
# 尝试使用不同的数据集来训练转换器。您还可以通过更改上面的超参数来创建基本transformer或transformer XL。
# 您还可以使用这里定义的层来创建BERT并训练艺术模型的状态。此外，您可以实现波束搜索来获得更好的预测。</code></pre>