pytorch-- Attention Mechanism

1. paper: Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation

Encoder

　　每个时刻输入一个词，隐藏层状态根据公式ht=f(ht−1,xt)改变。其中激活函数f可以是sigmod,tanh,ReLU,sotfplus等。
　　读完序列的每一个词之后，会得到一个固定长度向量c=tanh(VhN)
Decoder

　　由结构图可以看出，t时刻的隐藏层状态ht由ht−1,yt−1,c决定：ht=f(ht−1,yt−1,c)，其中h0=tanh(V′c)
　　最后的输出yt是由ht,yt−1,c决定
　　P=(yt|yt−1,yt−2,...,y1,c)=g(ht,yt−1,c)

以上,f,gf,g都是激活函数，其中g一般是softmax

对此我在pytoch环境下进行实现seq2seq最初版的模型：

(参考：https://github.com/graykode/nlp-tutorial)

 import numpy as np
 import torch
 import torch.nn as nn
 from torch.autograd import Variable

 dtype = torch.FloatTensor
 # S: Symbol that shows starting of decoding input
 # E: Symbol that shows ending of decoding output
 # P: Symbol that will fill in blank sequence if current batch data size is short than time steps

 char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz']
 num_dic = {n: i for i, n in enumerate(char_arr)}

 seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']]

 # Seq2Seq Parameter
 n_step = 5
 n_hidden = 128
 n_class = len(num_dic)    #
 batch_size = len(seq_data)    #

 def make_batch(seq_data):
     input_batch, output_batch, target_batch = [], [], []

     for seq in seq_data:
         for i in range(2):
             seq[i] = seq[i] + 'P' * (n_step - len(seq[i]))

         input = [num_dic[n] for n in seq[0]]
         output = [num_dic[n] for n in ('S' + seq[1])]
         target = [num_dic[n] for n in (seq[1] + 'E')]

         input_batch.append(np.eye(n_class)[input])
         output_batch.append(np.eye(n_class)[output])
         target_batch.append(target) # not one-hot

     # make tensor
     return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch))

 # Model
 class Seq2Seq(nn.Module):
     def __init__(self):
         super(Seq2Seq, self).__init__()

         self.enc_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)
         self.dec_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)
         self.fc = nn.Linear(n_hidden, n_class)

     def forward(self, enc_input, enc_hidden, dec_input):
         enc_input = enc_input.transpose(0, 1) # enc_input: [max_len(=n_step, time step), batch_size, n_class]
         dec_input = dec_input.transpose(0, 1) # dec_input: [max_len(=n_step, time step), batch_size, n_class]

         # enc_states : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]
         _, enc_states = self.enc_cell(enc_input, enc_hidden)
         # outputs : [max_len+1(=6), batch_size, num_directions(=1) * n_hidden(=128)]
         outputs, _ = self.dec_cell(dec_input, enc_states)

         model = self.fc(outputs) # model : [max_len+1(=6), batch_size, n_class]
         return model

 input_batch, output_batch, target_batch = make_batch(seq_data)

 model = Seq2Seq()
 criterion = nn.CrossEntropyLoss()
 optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

 for epoch in range(5000):
     # make hidden shape [num_layers * num_directions, batch_size, n_hidden]
     hidden = Variable(torch.zeros(1, batch_size, n_hidden))

     # input_batch : [batch_size, max_len(=n_step, time step), n_class]
     # output_batch : [batch_size, max_len+1(=n_step, time step) (becase of 'S' or 'E'), n_class]
     # target_batch : [batch_size, max_len+1(=n_step, time step)], not one-hot
     output = model(input_batch, hidden, output_batch)
     # output : [max_len+1, batch_size, n_class]
     output = output.transpose(0, 1) # [batch_size, max_len+1(=6), n_class]
     loss = 0
     for i in range(0, len(target_batch)):
         # output[i] : [max_len+1, n_class, target_batch[i] : max_len+1]
         loss += criterion(output[i], target_batch[i])
     if (epoch + 1) % 1000 == 0:
         print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))

     optimizer.zero_grad()
     loss.backward()
     optimizer.step()

 # Test
 def translate(word):
     input_batch, output_batch, _ = make_batch([[word, 'P' * len(word)]])

     # make hidden shape [num_layers * num_directions, batch_size, n_hidden]
     hidden = Variable(torch.zeros(1, 1, n_hidden))
     output = model(input_batch, hidden, output_batch)
     # output : [max_len+1(=6), batch_size(=1), n_class]

     predict = output.data.max(2, keepdim=True)[1] # select n_class dimension
     decoded = [char_arr[i] for i in predict]
     end = decoded.index('E')
     translated = ''.join(decoded[:end])

     return translated.replace('P', '')

 print('test')
 print('man ->', translate('man'))
 print('mans ->', translate('mans'))
 print('king ->', translate('king'))
 print('black ->', translate('black'))
 print('upp ->', translate('upp'))

之后，在seq2seq模型基础上，提出了attention机制。

论文： NEURAL MACHINE TRANSLATION BY JOINTLY LEARNING TO ALIGN AND TRANSLATE