From 97030066a3f0c678e77efde2ce1b8c89e20e2d9f Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Thu, 2 Dec 2021 11:28:03 +0100 Subject: [PATCH 1/3] added zoneout to rec --- nn/rec.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/nn/rec.py b/nn/rec.py index 9a87685d..176e019a 100644 --- a/nn/rec.py +++ b/nn/rec.py @@ -37,3 +37,33 @@ def make_layer_dict( # TODO specify per-step, how? this should also work without rec loop, when there is no time dim. # https://github.com/rwth-i6/returnn/issues/847 return super().make_layer_dict(source, state=state, axis=None) + + +class ZoneoutLSTM(_Rec): + """ + LSTM with zoneout operating on a sequence. returns (output, final_state) tuple, where final_state is (h,c). + """ + def __init__(self, n_out: int, **kwargs): + super().__init__(n_out=n_out, unit="zoneoutlstm", **kwargs) + + # noinspection PyMethodOverriding + def make_layer_dict( + self, source: nn.LayerRef, *, initial_state: Optional[nn.LayerState] = None) -> nn.LayerDictRaw: + """make layer""" + return super().make_layer_dict(source, initial_state=initial_state) + + +class ZoneoutLSTMStep(_Rec): + """ + LSTM with zoneout operating one step. returns (output, state) tuple, where state is (h,c). + """ + default_name = "zoneoutlstm" # make consistent to ZoneoutLSTM + + def __init__(self, n_out: int, **kwargs): + super().__init__(n_out=n_out, unit="zoneoutlstm", **kwargs) + + # noinspection PyMethodOverriding + def make_layer_dict( + self, source: nn.LayerRef, *, state: nn.LayerState) -> nn.LayerDictRaw: + """make layer""" + return super().make_layer_dict(source, state=state) From e936a99ea99d1f7cff6e8b23ebd3e1f0a2b1dbb6 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Wed, 8 Dec 2021 10:33:13 +0100 Subject: [PATCH 2/3] added explicit parameters --- nn/rec.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nn/rec.py b/nn/rec.py index 176e019a..37d4d93a 100644 --- a/nn/rec.py +++ b/nn/rec.py @@ -43,8 +43,10 @@ class ZoneoutLSTM(_Rec): """ LSTM with zoneout operating on a sequence. returns (output, final_state) tuple, where final_state is (h,c). """ - def __init__(self, n_out: int, **kwargs): - super().__init__(n_out=n_out, unit="zoneoutlstm", **kwargs) + def __init__(self, n_out: int, zoneout_factor_cell: int = 0., zoneout_factor_output: int = 0., **kwargs): + super().__init__( + unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, + n_out=n_out, unit="zoneoutlstm", **kwargs) # noinspection PyMethodOverriding def make_layer_dict( @@ -59,8 +61,10 @@ class ZoneoutLSTMStep(_Rec): """ default_name = "zoneoutlstm" # make consistent to ZoneoutLSTM - def __init__(self, n_out: int, **kwargs): - super().__init__(n_out=n_out, unit="zoneoutlstm", **kwargs) + def __init__(self, n_out: int, zoneout_factor_cell: int = 0., zoneout_factor_output: int = 0., **kwargs): + super().__init__( + unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, + n_out=n_out, unit="zoneoutlstm", **kwargs) # noinspection PyMethodOverriding def make_layer_dict( From 1816e3a329068ba0bde75f4a16379bc0dd2ed847 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Wed, 8 Dec 2021 10:35:37 +0100 Subject: [PATCH 3/3] changed order of params --- nn/rec.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nn/rec.py b/nn/rec.py index 37d4d93a..0a0d7d9a 100644 --- a/nn/rec.py +++ b/nn/rec.py @@ -45,12 +45,12 @@ class ZoneoutLSTM(_Rec): """ def __init__(self, n_out: int, zoneout_factor_cell: int = 0., zoneout_factor_output: int = 0., **kwargs): super().__init__( - unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, - n_out=n_out, unit="zoneoutlstm", **kwargs) + n_out=n_out, unit="zoneoutlstm", + unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, **kwargs) # noinspection PyMethodOverriding def make_layer_dict( - self, source: nn.LayerRef, *, initial_state: Optional[nn.LayerState] = None) -> nn.LayerDictRaw: + self, source: nn.LayerRef, *, initial_state: Optional[nn.LayerState] = None) -> nn.LayerDictRaw: """make layer""" return super().make_layer_dict(source, initial_state=initial_state) @@ -63,8 +63,8 @@ class ZoneoutLSTMStep(_Rec): def __init__(self, n_out: int, zoneout_factor_cell: int = 0., zoneout_factor_output: int = 0., **kwargs): super().__init__( - unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, - n_out=n_out, unit="zoneoutlstm", **kwargs) + n_out=n_out, unit="zoneoutlstm", + unit_opts={'zoneout_factor_cell': zoneout_factor_cell, 'zoneout_factor_output': zoneout_factor_output}, **kwargs) # noinspection PyMethodOverriding def make_layer_dict(