From e7afe9ca5fe1a463d5e9cfb230b95be4983844a2 Mon Sep 17 00:00:00 2001 From: richardburleigh Date: Sat, 7 Dec 2019 11:14:58 +1100 Subject: [PATCH] Remove dropout from decoder cell state Fix FP16 stagnation at "OVERFLOW! Skipping step. Attempted loss scale.." --- model.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/model.py b/model.py index 571fe32c8..c77ba93b1 100755 --- a/model.py +++ b/model.py @@ -383,8 +383,6 @@ def decode(self, decoder_input, attention_weights=None): cell_input, (self.attention_hidden, self.attention_cell)) self.attention_hidden = F.dropout( self.attention_hidden, self.p_attention_dropout, self.training) - self.attention_cell = F.dropout( - self.attention_cell, self.p_attention_dropout, self.training) attention_weights_cat = torch.cat( (self.attention_weights.unsqueeze(1), @@ -400,8 +398,6 @@ def decode(self, decoder_input, attention_weights=None): decoder_input, (self.decoder_hidden, self.decoder_cell)) self.decoder_hidden = F.dropout( self.decoder_hidden, self.p_decoder_dropout, self.training) - self.decoder_cell = F.dropout( - self.decoder_cell, self.p_decoder_dropout, self.training) decoder_hidden_attention_context = torch.cat( (self.decoder_hidden, self.attention_context), dim=1)