From 02bdf0631d5f0c2918b2babee5b0c855cc96a172 Mon Sep 17 00:00:00 2001 From: Xue10 Date: Mon, 30 Sep 2024 08:16:43 +0800 Subject: [PATCH] fix layernorm --- model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model.py b/model.py index afdaee5..92892e9 100644 --- a/model.py +++ b/model.py @@ -15,9 +15,9 @@ def forward(self, x): # Keep the dimension for broadcasting mean = x.mean(dim = -1, keepdim = True) # (batch, seq_len, 1) # Keep the dimension for broadcasting - std = x.std(dim = -1, keepdim = True) # (batch, seq_len, 1) + var = x.var(dim = -1, keepdim = True) # (batch, seq_len, 1) # eps is to prevent dividing by zero or when std is very small - return self.alpha * (x - mean) / (std + self.eps) + self.bias + return self.alpha * (x - mean) / torch.sqrt(var + self.eps) + self.bias class FeedForwardBlock(nn.Module):