test_reclayer_optimize_out_cum_concat wip

Author: albertz

tests/test_TFNetworkRecLayer.py

@@ -3310,34 +3310,39 @@ def test_rec_subnet_simple_rnn():
     print("rnn_cell also fine.")
 
 
-def check_reclayer_optimize_out(subnet_layer_dict, other_subnet_layers=None, shared_base_net=None, rtol=1e-4):
+def check_reclayer_optimize_out(
+  subnet_layer_dict, other_subnet_layers=None, shared_base_net=None,
+  n_in=13, n_out=NotSpecified,
+  rtol=1e-4):
   """
   :param dict[str] subnet_layer_dict: opts for the output layer inside the rec-layer subnet
   :param dict[str,dict[str]] other_subnet_layers: other layers for the rec-layer subnet
   :param dict[str,dict[str]] shared_base_net:
+  :param int n_in:
+  :param int|NotSpecified|None n_out:
   :param float rtol: for the final comparison check
   """
   subnet_layer_dict = subnet_layer_dict.copy()
-  n_in = 13
-  n_out = subnet_layer_dict.get("n_out", 17)
+  if n_out is NotSpecified:
+    n_out = subnet_layer_dict.get("n_out", 17)
   n_batch = 5
   n_time = 7
-  subnet_layer_dict["n_out"] = n_out
   subnet_layer_dict.setdefault("from", ["data:source"])
   rec_layer_dict = {
     "class": "rec",
     "from": ["data"],
     "unit": {"output": subnet_layer_dict},
-    "n_out": n_out,
     "is_output_layer": True
   }
+  if n_out is not None:
+    subnet_layer_dict["n_out"] = n_out
+    rec_layer_dict["n_out"] = n_out
   if other_subnet_layers:
     assert "output" not in other_subnet_layers
     rec_layer_dict["unit"].update(other_subnet_layers)
   config = Config({
     "debug_print_layer_output_template": True,
-    "num_inputs": n_in,
-    "num_outputs": n_out
+    "extern_data": {"data": {"dim": n_in}},
   })
   from returnn.tf.layers.rec import _SubnetworkRecCell
   with make_scope() as session:
@@ -3414,6 +3419,11 @@ def test_reclayer_optimize_out_selfatt_left():
     "class": "self_attention", "attention_left_only": True, "num_heads": 2, "total_key_dim": 6, "n_out": 18})
 
 
+def test_reclayer_optimize_out_cum_concat():
+  new_dim = DimensionTag(kind=DimensionTag.Types.Spatial, description="cum_concat_new_dim")
+  check_reclayer_optimize_out({"class": "cum_concat", "new_dim": new_dim}, n_in=13, n_out=None)
+
+
 def test_reclayer_optimize_out_dot():
   # Used for multi-head dot-attention.
   AttNumHeads = 4