Update

Author: allenwind

README.md

@@ -1,6 +1,6 @@
 # text-globalmaxpool-visualization
 
-NLP中，我们常常使用AttentionPooling对词向量序列进行聚合作为句向量的表示，在[text-attentionpooling-visualization](https://github.com/allenwind/text-attentionpooling-visualization)分析过，有不错的效果。此外，还常常使用MaxPooling对词向量序列进行聚合（句向量），如Tensorflow中的`tf.keras.layers.GlobalMaxPool1D`，这里尝试可视化MaxPooling的效果。一个词向量序列可以用矩阵表示，其维度为(maxlen, hdims)，MaxPooling操作后，变为向量，可以看做句向量的表示，其维度为(1, hdims)。hdims每个值对应着某个词向量的最大值，把其作为该词在下游任务中重要性的权重，并通过可视化来观察是否具有预期的效果。
+NLP中，我们常常使用AttentionPooling对词向量序列进行聚合作为句向量的表示，在[text-attentionpooling-visualization](https://github.com/allenwind/text-attentionpooling-visualization)分析过，有不错的效果。此外，还常常使用MaxPooling和AveragePooling对词向量序列进行聚合（句向量），其中AveragePooling可以看做是AttentionPooling的特例。这里只探讨MaxPooling，如Tensorflow中的`tf.keras.layers.GlobalMaxPool1D`。这里尝试可视化MaxPooling的效果。一个词向量序列可以用矩阵表示，其维度为(maxlen, hdims)，MaxPooling操作后，变为向量，可以看做句向量的表示，其维度为(1, hdims)。hdims每个值对应着某个词向量的最大值，把其作为该词在下游任务中重要性的权重，并通过可视化来观察是否具有预期的效果。
 
 
 

dataset.py

@@ -7,14 +7,21 @@
 import numpy as np
 
 _THUCNews = "/home/zhiwen/workspace/dataset/THUCNews-title-label.txt"
-def load_THUCNews_title_label(file=_THUCNews):
+def load_THUCNews_title_label(file=_THUCNews, nobrackets=True):
     with open(file, encoding="utf-8") as fd:
         text = fd.read()
     lines = text.split("\n")[:-1]
+    np.random.shuffle(lines)
     titles = []
     labels = []
     for line in lines:
         title, label = line.split("\t")
+        if not title:
+            continue
+
+        # 去掉括号内容
+        if nobrackets:
+            title = re.sub("\(.+?\)", lambda x:"", title)
         titles.append(title)
         labels.append(label)
     categoricals = list(set(labels))

model.py

@@ -21,15 +21,15 @@
 from dataset import load_weibo_senti_100k
 from dataset import load_simplifyweibo_4_moods
 
-X, y, classes = load_weibo_senti_100k()
+X, y, classes = load_THUCNews_title_label()
 X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=7384672)
 
 num_classes = len(classes)
 tokenizer = SimpleTokenizer()
 tokenizer.fit(X_train)
 X_train = tokenizer.transform(X_train)
 
-# maxlen = 48
+maxlen = 48
 maxlen = find_best_maxlen(X_train)
 
 X_train = sequence.pad_sequences(
@@ -38,7 +38,7 @@
     dtype="int32",
     padding="post",
     truncating="post",
-    value=0
+    value=0.0
 )
 y_train = tf.keras.utils.to_categorical(y_train)
 
@@ -48,8 +48,9 @@
 inputs = Input(shape=(maxlen,))
 mask = Lambda(lambda x: tf.not_equal(x, 0))(inputs)
 x = Embedding(num_words, embedding_dims,
-    embeddings_initializer="glorot_normal",
-    input_length=maxlen)(inputs)
+    embeddings_initializer="normal",
+    input_length=maxlen,
+    mask_zero=True)(inputs)
 x = Dropout(0.2)(x)
 x = Conv1D(filters=128,
            kernel_size=3,
@@ -71,7 +72,7 @@
 model_w_outputs = Model(inputs, w)
 
 batch_size = 32
-epochs = 10
+epochs = 2
 callbacks = []
 model.fit(X_train, y_train,
           batch_size=batch_size,

pooling.py

@@ -1,6 +1,5 @@
 import tensorflow as tf
 
-
 class MaskGlobalMaxPooling1D(tf.keras.layers.Layer):
 
     def __init__(self, **kwargs):
@@ -11,11 +10,54 @@ def call(self, inputs, mask=None):
             mask = 1
         else:
             # 扩展维度便于广播
-            mask = tf.expand_dims(tf.cast(mask, "float32"), -1)
+            mask = tf.expand_dims(tf.cast(mask, tf.float32), -1)
         x = inputs
         x = x - (1 - mask) * 1e12 # 用一个大的负数mask
         x = tf.reduce_max(x, axis=1, keepdims=True)
         ws = tf.where(inputs == x, x, 0.0)
         ws = tf.reduce_sum(ws, axis=2)
         x = tf.squeeze(x, axis=1)
         return x, ws
+
+class MaskGlobalAveragePooling1D(tf.keras.layers.Layer):
+    
+    def __init__(self, **kwargs):
+        super(MaskGlobalAveragePooling1D, self).__init__(**kwargs)
+
+    def call(self, inputs, mask=None):
+        if mask is None:
+            mask = 1
+        else:
+            mask = tf.expand_dims(tf.cast(mask, tf.float32), -1)
+        x = inputs
+        x = x * mask
+        x = tf.reduce_sum(x, axis=1)
+        x =  x / tf.reduce_sum(mask, axis=1)
+        ws = tf.square(inputs - tf.expand_dims(x, axis=1))
+        ws = tf.reduce_mean(ws, axis=2)
+        ws = ws + (1 - mask) * 1e12
+        ws = 1 / ws
+        return x, ws
+
+class MinVariancePooling(tf.keras.layers.Layer):
+    """最小方差加权平均，Inverse-variance weighting
+    等价于正太分布的最小熵加权平均"""
+
+    def __init__(self, **kwargs):
+        super(MinVariancePooling, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        d = tf.cast(input_shape[2], tf.float32)
+        self.alpha = 1 / (d - 1)
+
+    def call(self, inputs, mask=None):
+        if mask is None:
+            mask = 1
+        else:
+            mask = tf.expand_dims(tf.cast(mask, tf.float32), -1)
+        mu = tf.reduce_mean(inputs, axis=2, keepdims=True) # 均值
+        var = self.alpha * tf.reduce_sum(tf.square(inputs - mu), axis=2, keepdims=True) # 方差的无偏估计
+        var = var + (1 - mask) * 1e12 # 倒数的mask处理
+        ivar = 1 / var
+        ws = ivar / tf.reduce_sum(ivar, axis=1, keepdims=True)
+        return tf.reduce_sum(inputs * ws * mask, axis=1), ws