pytorch --Rnn语言模型(LSTM，BiLSTM) -- 《Recurrent neural network based language model》

论文通过实现RNN来完成了文本分类。

论文地址：88888888

模型结构图：

原理自行参考论文，code and comment（https://github.com/graykode/nlp-tutorial）:

 # -*- coding: utf-8 -*-
 # @time : 2019/11/9  15:12

 import numpy as np
 import torch
 import torch.nn as nn
 import torch.optim as optim
 from torch.autograd import Variable

 dtype = torch.FloatTensor

 sentences = [ "i like dog", "i love coffee", "i hate milk"]

 word_list = " ".join(sentences).split()
 word_list = list(set(word_list))
 word_dict = {w: i for i, w in enumerate(word_list)}
 number_dict = {i: w for i, w in enumerate(word_list)}
 n_class = len(word_dict)

 # TextRNN Parameter
 batch_size = len(sentences)
 n_step = 2 # number of cells(= number of Step)
 n_hidden = 5 # number of hidden units in one cell

 def make_batch(sentences):
     input_batch = []
     target_batch = []

     for sen in sentences:
         word = sen.split()
         input = [word_dict[n] for n in word[:-1]]
         target = word_dict[word[-1]]

         input_batch.append(np.eye(n_class)[input])
         target_batch.append(target)

     return input_batch, target_batch

 # to Torch.Tensor
 input_batch, target_batch = make_batch(sentences)
 input_batch = Variable(torch.Tensor(input_batch))
 target_batch = Variable(torch.LongTensor(target_batch))

 class TextRNN(nn.Module):
     def __init__(self):
         super(TextRNN, self).__init__()

         self.rnn = nn.RNN(input_size=n_class, hidden_size=n_hidden,batch_first=True)
         self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))
         self.b = nn.Parameter(torch.randn([n_class]).type(dtype))

     def forward(self, hidden, X):
         if self.rnn.batch_first == True:
             # X [batch_size,time_step,word_vector]
             outputs, hidden = self.rnn(X, hidden)

             # outputs [batch_size, time_step, hidden_size*num_directions]
             output = outputs[:, -1, :]  # [batch_size, num_directions(=1) * n_hidden]
             model = torch.mm(output, self.W) + self.b  # model : [batch_size, n_class]
             return model
         else:
             X = X.transpose(0, 1) # X : [n_step, batch_size, n_class]
             outputs, hidden = self.rnn(X, hidden)
             # outputs : [n_step, batch_size, num_directions(=1) * n_hidden]
             # hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]

             output = outputs[-1,:,:] # [batch_size, num_directions(=1) * n_hidden]
             model = torch.mm(output, self.W) + self.b # model : [batch_size, n_class]
             return model

 model = TextRNN()

 criterion = nn.CrossEntropyLoss()
 optimizer = optim.Adam(model.parameters(), lr=0.001)

 # Training
 for epoch in range(5000):
     optimizer.zero_grad()

     # hidden : [num_layers * num_directions, batch, hidden_size]
     hidden = Variable(torch.zeros(1, batch_size, n_hidden))
     # input_batch : [batch_size, n_step, n_class]
     output = model(hidden, input_batch)

     # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot)
     loss = criterion(output, target_batch)
     if (epoch + 1) % 1000 == 0:
         print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))

     loss.backward()
     optimizer.step()

 # Predict
 hidden_initial = Variable(torch.zeros(1, batch_size, n_hidden))
 predict = model(hidden_initial, input_batch).data.max(1, keepdim=True)[1]
 print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])

LSTM unit的RNN模型：

 import numpy as np
 import torch
 import torch.nn as nn
 import torch.optim as optim
 from torch.autograd import Variable

 dtype = torch.FloatTensor

 char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz']
 word_dict = {n: i for i, n in enumerate(char_arr)}
 number_dict = {i: w for i, w in enumerate(char_arr)}
 n_class = len(word_dict)  # number of class(=number of vocab)

 seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star']

 # TextLSTM Parameters
 n_step = 3
 n_hidden = 128

 def make_batch(seq_data):
     input_batch, target_batch = [], []

     for seq in seq_data:
         input = [word_dict[n] for n in seq[:-1]]  # 'm', 'a' , 'k' is input
         target = word_dict[seq[-1]]  # 'e' is target
         input_batch.append(np.eye(n_class)[input])
         target_batch.append(target)

     return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch))

 class TextLSTM(nn.Module):
     def __init__(self):
         super(TextLSTM, self).__init__()

         self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden)
         self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))
         self.b = nn.Parameter(torch.randn([n_class]).type(dtype))

     def forward(self, X):
         input = X.transpose(0, 1)  # X : [n_step, batch_size, n_class]

         hidden_state = Variable(
             torch.zeros(1, len(X), n_hidden))  # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]
         cell_state = Variable(
             torch.zeros(1, len(X), n_hidden))  # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]

         outputs, (_, _) = self.lstm(input, (hidden_state, cell_state))
         outputs = outputs[-1]  # [batch_size, n_hidden]
         model = torch.mm(outputs, self.W) + self.b  # model : [batch_size, n_class]
         return model

 input_batch, target_batch = make_batch(seq_data)

 model = TextLSTM()

 criterion = nn.CrossEntropyLoss()
 optimizer = optim.Adam(model.parameters(), lr=0.001)

 # Training
 for epoch in range(1000):

     output = model(input_batch)
     loss = criterion(output, target_batch)
     if (epoch + 1) % 100 == 0:
         print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))
     optimizer.zero_grad()
     loss.backward()
     optimizer.step()

 inputs = [sen[:3] for sen in seq_data]

 predict = model(input_batch).data.max(1, keepdim=True)[1]
 print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()])

BiLSTM RNN model:

 import numpy as np
 import torch
 import torch.nn as nn
 import torch.optim as optim
 from torch.autograd import Variable
 import torch.nn.functional as F

 dtype = torch.FloatTensor

 sentence = (
     'Lorem ipsum dolor sit amet consectetur adipisicing elit '
     'sed do eiusmod tempor incididunt ut labore et dolore magna '
     'aliqua Ut enim ad minim veniam quis nostrud exercitation'
 )

 word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))}
 number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))}
 n_class = len(word_dict)
 max_len = len(sentence.split())
 n_hidden = 5

 def make_batch(sentence):
     input_batch = []
     target_batch = []

     words = sentence.split()
     for i, word in enumerate(words[:-1]):
         input = [word_dict[n] for n in words[:(i + 1)]]
         input = input + [0] * (max_len - len(input))
         target = word_dict[words[i + 1]]
         input_batch.append(np.eye(n_class)[input])
         target_batch.append(target)

     return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch))

 class BiLSTM(nn.Module):
     def __init__(self):
         super(BiLSTM, self).__init__()

         self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden, bidirectional=True)
         self.W = nn.Parameter(torch.randn([n_hidden * 2, n_class]).type(dtype))
         self.b = nn.Parameter(torch.randn([n_class]).type(dtype))

     def forward(self, X):
         input = X.transpose(0, 1)  # input : [n_step, batch_size, n_class]

         hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden))   # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]
         cell_state = Variable(torch.zeros(1*2, len(X), n_hidden))     # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]

         outputs, (_, _) = self.lstm(input, (hidden_state, cell_state))
         outputs = outputs[-1]  # [batch_size, n_hidden]
         model = torch.mm(outputs, self.W) + self.b  # model : [batch_size, n_class]
         return model

 input_batch, target_batch = make_batch(sentence)

 model = BiLSTM()

 criterion = nn.CrossEntropyLoss()
 optimizer = optim.Adam(model.parameters(), lr=0.001)

 # Training
 for epoch in range(10000):
     output = model(input_batch)
     loss = criterion(output, target_batch)
     if (epoch + 1) % 1000 == 0:
         print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))

     optimizer.zero_grad()
     loss.backward()
     optimizer.step()

 predict = model(input_batch).data.max(1, keepdim=True)[1]
 print(sentence)
 print([number_dict[n.item()] for n in predict.squeeze()])