05 Transformer 中的前馈神经网络（FFN）的实现

2：20：理论链接

博客配套视频链接: https://space.bilibili.com/383551518?spm_id_from=333.1007.0.0 b 站直接看

配套 github 链接：https://github.com/nickchen121/Pre-training-language-model

配套博客链接：https://www.cnblogs.com/nickchen121/p/15105048.html

class PositionWiseFeedForward(nn.Module):

    """
    w2(relu(w1(layer_norm(x))+b1))+b2
    """

    def __init__(self, d_model, d_ff, dropout=0.1):
        super(PositionWiseFeedForward, self).__init__()
        self.w_1 = nn.Linear(d_model, d_ff)
        self.w_2 = nn.Linear(d_ff, d_model)
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
        self.dropout_1 = nn.Dropout(dropout)
        self.relu = nn.ReLU()
        self.dropout_2 = nn.Dropout(dropout)

    def forward(self, x):
        inter = self.dropout_1(self.relu(self.w_1(self.layer_norm(x))))
        output = self.dropout_2(self.w_2(inter))
        return output