At least executable now. ORZ

Yan · Yan · commit 06e9a21ba8f6 · 2017-06-15T16:44:49.000+08:00
diff --git a/README.md b/README.md
@@ -1,3 +1,3 @@
 # zhihu_cup
 draft code for zhihu cup using MxNet</br>
-Code under construction
+Barely executable now.
diff --git a/concise_data.py b/concise_data.py
@@ -1,4 +1,3 @@
-import mxnet as mx
 from read_embed import read_embed
 char_raw = open('sorted_char_count.txt').readlines()
 word_raw = open('sorted_word_count.txt').readlines()
diff --git a/iter.py b/iter.py
@@ -1,4 +1,6 @@
-import os
+import os,sys
+curr_path = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(curr_path, "../mxnet/python"))
 import numpy as np
 import mxnet as mx
 from read_embed import read_embed
@@ -49,14 +51,26 @@ def __init__(self, question_set_path,
                 for v in self.buckets:
                     f.write(v+'\n')
         self.reset()
+        keys = self.max_bucket_key.split(',')
+        self.provide_data = []
+
         if embed_mode %2 == 0:
             self.char_dict, self.char_dict_size, self.char_dict_dim = read_embed(char_embed_path)
+            self.provide_data +=[('tc_array',(self.batch_size,int(keys[0]),self.char_dict_dim)),\
+                                 ('cc_array',(self.batch_size,int(keys[1]),self.char_dict_dim))] 
         if embed_mode > 0:
             self.word_dict, self.word_dict_size, self.word_dict_dim = read_embed(word_embed_path)
-        self.provide_data = [('data', (self.batch_size, ))]
+            if embed_mode %2 == 0:
+                self.provide_data +=[('tw_array',(self.batch_size,int(keys[2]),self.word_dict_dim)),\
+                                     ('cw_array',(self.batch_size,int(keys[3]),self.word_dict_dim))] 
+            else:
+                self.provide_data +=[('tw_array',(self.batch_size,int(keys[0]),self.word_dict_dim)),\
+                                     ('cw_array',(self.batch_size,int(keys[1]),self.word_dict_dim))] 
+
         self.provide_label = [('label', (self.batch_size, len(self.topic_info) + 1))]
 
 
+
     def create_buckets(self, buckets=None):
         if buckets is None:
             self.buckets = self.default_buckets()
@@ -178,7 +192,6 @@ def __iter__(self):
             bucket_key = self.buckets[idx]
             inds= self.bucket_samples_inds[bucket_key] \
                     [self.bucket_offset[idx]:self.bucket_offset[idx]+self.batch_size]
-
             shapes= [(self.batch_size, int(v)) for v in bucket_key.split(',')]
             if len(shapes) == 4:
                 tc_array = np.zeros(shapes[0]+(self.char_dict_dim,))
@@ -191,6 +204,8 @@ def __iter__(self):
             else:
                 tw_array = np.zeros(shapes[0]+(self.word_dict_dim,))
                 cw_array = np.zeros(shapes[1]+(self.word_dict_dim,))
+            #print '*'*20
+            #print shapes,bucket_key,tw_array.shape, cw_array.shape
             
             label = np.zeros((self.batch_size, len(self.topic_encode)+1))
             for i,ind in enumerate(inds):
@@ -209,33 +224,36 @@ def __iter__(self):
                     cc = tc
                     cw = tw
 
-                data_name = []
-                data = []
                 if self.embed_mode  %2 == 0:
                     for j, v in enumerate(tc.split(',')):
                         tc_array[i,j] = self.char_dict[v]
                     for j, v in enumerate(cc.split(',')):
                         cc_array[i,j] = self.char_dict[v]
-                    data_name += ['tc_array', 'cc_array']
-                    data += [mx.nd.array(tc_array), mx.nd.array(cc_array)]
                 if self.embed_mode > 0:
                     for j, v in enumerate(tw.split(',')):
                         tw_array[i,j] = self.word_dict[v]
                     for j, v in enumerate(cw.split(',')):
                         cw_array[i,j] = self.word_dict[v]
-                    data_name += ['tw_array', 'cw_array']
-                    data += [mx.nd.array(tw_array), mx.nd.array(cw_array)]
 
                 top = self.question_topic[ind].split()[1].split(',')
                 for t in top:
                     label[i,self.topic_encode[t]] = 1
+            data_name = []
+            data = []
+            if self.embed_mode  %2 == 0:
+                data_name += ['tc_array', 'cc_array']
+                data += [mx.nd.array(tc_array), mx.nd.array(cc_array)]
+            if self.embed_mode > 0:
+                data_name += ['tw_array', 'cw_array']
+                data += [mx.nd.array(tw_array), mx.nd.array(cw_array)]
             label = [mx.nd.array(label)]
             label_name = ['label']
+            #print bucket_key, data
             yield SimpleBatch(data_name, data, label_name, label, bucket_key)
         raise StopIteration
 
 if __name__ == '__main__':
     ziter = zhihu_iter('tidy_question_train_set.txt','tidy_question_topic_train_set.txt',embed_mode=1)
     #ziter.reset()
     for i in ziter:
-        print i
+        print i.provide_data
diff --git a/purge_data.py b/purge_data.py
@@ -1,4 +1,3 @@
-import mxnet as mx
 from read_embed import read_embed
 char_embed_path='./char_embedding.txt'
 word_embed_path='./word_embedding.txt'
diff --git a/sym.py b/sym.py
@@ -1,4 +1,6 @@
-
+import os,sys
+curr_path = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(curr_path, "../mxnet/python"))
 import mxnet as mx
 import numpy as np
 
@@ -43,6 +45,9 @@ def fc_module(data, prefix, num_hidden=256):
         return relu_fc1
 
 def sym_gen_char(bucket_key):
+    num_layers = 1
+    num_class = 2000
+    num_hidden = 512
     key = bucket_key.split(',')
     tc_length = int(key[0])
     cc_length = int(key[1])
@@ -55,17 +60,55 @@ def sym_gen_char(bucket_key):
     cc_slices = list(mx.symbol.SliceChannel(data=cc_data, axis=1, num_outputs=cc_length, squeeze_axis=True, name='cc_slice'))
     tc_concat, _ = tc_cell.unroll(tc_length, inputs = tc_slices, merge_outputs=True, layout='TNC')
     cc_concat, _ = cc_cell.unroll(cc_length, inputs = cc_slices, merge_outputs=True, layout='TNC')
-    tc_concat = mx.sym.swapaxes(tc_concat, 0, 1)
-    cc_concat = mx.sym.swapaxes(cc_concat, 0, 1)
+    tc_concat = mx.sym.transpose(tc_concat, (1, 2, 0))
+    cc_concat = mx.sym.transpose(cc_concat, (1, 2, 0))
+    tc_concat = mx.sym.Pooling(tc_concat, kernel=(1,), global_pool = True, pool_type='max')
+    cc_concat = mx.sym.Pooling(cc_concat, kernel=(1,), global_pool = True, pool_type='max')
+    feature = mx.sym.Concat(*[tc_concat, cc_concat], name= 'concat')
+    feature = mx.sym.Dropout(feature, p=0.5)
+    feature = fc_module(feature, 'feature', num_hidden=2000)
+    loss = mx.sym.LogisticRegressionOutput(feature, label=label, name='regression')
+    return loss
 
 
 def sym_gen_word(bucket_key):
+    num_layers = 1
+    num_class = 2000
+    num_hidden = 512
+    key = bucket_key.split(',')
+    tw_length = int(key[0])
+    cw_length = int(key[1])
+    tw_data = mx.sym.Variable('tw_array')
+    cw_data = mx.sym.Variable('cw_array')
+    label   = mx.sym.Variable('label')
+    tw_cell = mx.rnn.FusedRNNCell(num_hidden, num_layers=num_layers, bidirectional=True, mode='lstm', prefix ='tw_')
+    cw_cell = mx.rnn.FusedRNNCell(num_hidden, num_layers=num_layers, bidirectional=True, mode='lstm', prefix ='cw_')
+    tw_slices = list(mx.symbol.SliceChannel(data=tw_data, axis=1, num_outputs=tw_length, squeeze_axis=True, name='tw_slice'))
+    cw_slices = list(mx.symbol.SliceChannel(data=cw_data, axis=1, num_outputs=cw_length, squeeze_axis=True, name='cw_slice'))
+    tw_concat, _ = tw_cell.unroll(tw_length, inputs = tw_slices, merge_outputs=True, layout='TNC')
+    cw_concat, _ = cw_cell.unroll(cw_length, inputs = cw_slices, merge_outputs=True, layout='TNC')
+    tw_concat = mx.sym.transpose(tw_concat, (1, 2, 0))
+    cw_concat = mx.sym.transpose(cw_concat, (1, 2, 0))
+    tw_concat = mx.sym.Pooling(tw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    cw_concat = mx.sym.Pooling(cw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    feature = mx.sym.Concat(*[tw_concat, cw_concat], name= 'concat')
+    feature = mx.sym.Dropout(feature, p=0.5)
+    feature = fc_module(feature, 'feature', num_hidden=2000)
+    loss = mx.sym.LogisticRegressionOutput(feature, label=label, name='regression')
+    data_name = ['tw_array', 'cw_array']
+    label_name = ['label']
+    return loss, data_name, label_name
 
 
 def sym_gen_both(bucket_key):
-
-
+    num_layers = 1
+    num_class = 2000
+    num_hidden = 512
     key = bucket_key.split(',')
+    tc_length = int(key[0])
+    cc_length = int(key[1])
+    tw_length = int(key[2])
+    cw_length = int(key[3])
     tc_data = mx.sym.Variable('tc_array')
     cc_data = mx.sym.Variable('cc_array')
     tw_data = mx.sym.Variable('tw_array')
@@ -83,34 +126,29 @@ def sym_gen_both(bucket_key):
     cc_concat, _ = cc_cell.unroll(cc_length, inputs = cc_slices, merge_outputs=True, layout='TNC')
     tw_concat, _ = tw_cell.unroll(tw_length, inputs = tw_slices, merge_outputs=True, layout='TNC')
     cw_concat, _ = cw_cell.unroll(cw_length, inputs = cw_slices, merge_outputs=True, layout='TNC')
-    tc_concat = mx.sym.swapaxes(tc_concat, 0, 1)
-    cc_concat = mx.sym.swapaxes(cc_concat, 0, 1)
-    tw_concat = mx.sym.swapaxes(tw_concat, 0, 1)
-    cw_concat = mx.sym.swapaxes(cw_concat, 0, 1)
-    #ch_outputs = mx.sym.Concat(*[tc_concat, cc_concat])
-    #wd_outputs = mx.sym.Concat(*[tw_concat, cw_concat])
-    #title_outputs= mx.sym.Concat(*[tc_concat, tw_concat])
-    #content_outputs= mx.sym.Concat(*[cc_concat, cw_concat])
-    #ch_outputs = fc_module(ch_outputs, 'ch_', num_hidden = 2000)
-    #wd_outputs = fc_module(wd_outputs, 'wd_', num_hidden = 2000)
-    #title_outputs = fc_module(title_outputs, 'title_', num_hidden = 2000)
-    #content_outputs = fc_module(content_outputs, 'content_', num_hidden = 2000)
-    #feature = mx.sym.Concat(*[ch_outputs, wd_outputs, title_outputs, content_outputs])
-    feature = mx.sym.Concat(*[tc_concat, cc_concat, tw_concat, cw_concat])
-    feature = fc_module(feature, 'feature', num_hidden=4000)
-    feature = mx.sym.FullyConnected(data=feature, num_hidden=num_class, name='fc1')
-    loss = mx.sym.LogisticRegressionOutput(feature, label, name='regression')
+    tc_concat = mx.sym.transpose(tc_concat, (1, 2, 0))
+    cc_concat = mx.sym.transpose(cc_concat, (1, 2, 0))
+    tw_concat = mx.sym.transpose(tw_concat, (1, 2, 0))
+    cw_concat = mx.sym.transpose(cw_concat, (1, 2, 0))
+    tc_concat = mx.sym.Pooling(tc_concat, kernel=(1,), global_pool = True, pool_type='max')
+    cc_concat = mx.sym.Pooling(cc_concat, kernel=(1,), global_pool = True, pool_type='max')
+    tw_concat = mx.sym.Pooling(tw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    cw_concat = mx.sym.Pooling(cw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    feature = mx.sym.Concat(*[tc_concat, cc_concat, tw_concat, cw_concat], name= 'concat')
+    feature = mx.sym.Dropout(feature, p=0.5)
+    feature = fc_module(feature, 'feature', num_hidden=2000)
+    loss = mx.sym.LogisticRegressionOutput(feature, label=label, name='regression')
     return loss
 
 if __name__ == '__main__':
-    sym = sym_gen(100,100, 100, 100)
+    sym = sym_gen_both('100,33,11,21')
     batch_size = 32
     dim = 256
     length = 100
-    shapes = sym.infer_shape_partial(tc_array=(batch_size,length,dim),
-                                     cc_array=(batch_size,length,dim),
-                                     tw_array=(batch_size,length,dim),
-                                     cw_array=(batch_size,length,dim),
+    shapes = sym.infer_shape_partial(tc_array=(batch_size,100,dim),
+                                     cc_array=(batch_size,33,dim),
+                                     tw_array=(batch_size,11,dim),
+                                     cw_array=(batch_size,21,dim),
                                      label=(batch_size,2000))
     names = sym.list_arguments()
     for name, shape in zip(names, shapes[0]):
diff --git a/train.py b/train.py
@@ -0,0 +1,69 @@
+import os,sys
+curr_path = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(curr_path, "../mxnet/python"))
+import numpy as np
+import mxnet as mx
+from iter import zhihu_iter
+batch_size =4
+ziter = zhihu_iter('tiny_train.txt','tiny_topic.txt',batch_size=batch_size,embed_mode=1)
+
+
+num_layers = 1
+num_class = 2000
+num_hidden = 512
+tw_cell = mx.rnn.FusedRNNCell(num_hidden, num_layers=num_layers, bidirectional=True, mode='lstm', prefix ='tw_')
+cw_cell = mx.rnn.FusedRNNCell(num_hidden, num_layers=num_layers, bidirectional=True, mode='lstm', prefix ='cw_')
+
+def fc_module(data, prefix, num_hidden=256):
+    with mx.name.Prefix(prefix):
+        fc1  = mx.sym.FullyConnected(data=data, num_hidden=num_hidden, name='fc1')
+        relu_fc1 = mx.sym.Activation(data=fc1, act_type='relu', name='relu_fc1')
+        return relu_fc1
+
+data_name = [i[0] for i in ziter.provide_data]
+label_name = [i[0] for i in ziter.provide_label]
+def sym_gen_word(bucket_key):
+    key = bucket_key.split(',')
+    tw_length = int(key[0])
+    cw_length = int(key[1])
+    tw_data = mx.sym.Variable('tw_array')
+    cw_data = mx.sym.Variable('cw_array')
+    label   = mx.sym.Variable('label')
+    tw_slices = list(mx.symbol.SliceChannel(data=tw_data, axis=1, num_outputs=tw_length, squeeze_axis=True, name='tw_slice'))
+    cw_slices = list(mx.symbol.SliceChannel(data=cw_data, axis=1, num_outputs=cw_length, squeeze_axis=True, name='cw_slice'))
+    tw_concat, _ = tw_cell.unroll(tw_length, inputs = tw_slices, merge_outputs=True, layout='TNC')
+    cw_concat, _ = cw_cell.unroll(cw_length, inputs = cw_slices, merge_outputs=True, layout='TNC')
+    tw_concat = mx.sym.transpose(tw_concat, (1, 2, 0))
+    cw_concat = mx.sym.transpose(cw_concat, (1, 2, 0))
+    tw_concat = mx.sym.Pooling(tw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    cw_concat = mx.sym.Pooling(cw_concat, kernel=(1,), global_pool = True, pool_type='max')
+    feature = mx.sym.Concat(*[tw_concat, cw_concat], name= 'concat')
+    feature = mx.sym.Dropout(feature, p=0.5)
+    feature = fc_module(feature, 'feature', num_hidden=2000)
+    loss = mx.sym.LogisticRegressionOutput(feature, label=label, name='regression')
+    return loss, data_name, label_name
+
+#mod = mx.module.BucketingModule(sym_gen_word, default_bucket_key=ziter.max_bucket_key,context=mx.gpu(1),data_names=data_name, label_names=label_name)
+mod = mx.module.BucketingModule(sym_gen_word, default_bucket_key=ziter.max_bucket_key,context=mx.context.gpu(1))
+import logging
+head = '%(asctime)-15s %(message)s'
+logging.basicConfig(level=logging.DEBUG, format=head)
+prefix='model/textline'
+learning_rate = 0.01
+optimizer_params={'learning_rate': learning_rate,
+                'clip_gradient': 10 }
+monitor=mx.mon.Monitor(200, pattern='.*')
+
+
+
+num_epoch = 10
+print 'fit begin'
+mod.fit(train_data=ziter, eval_data=ziter,
+          optimizer='adadelta',
+          optimizer_params = optimizer_params,
+          eval_metric = mx.metric.MSE(),
+          num_epoch=num_epoch,
+          initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
+          batch_end_callback=mx.callback.Speedometer(batch_size, 50),
+          epoch_end_callback = mx.rnn.do_rnn_checkpoint([tw_cell, cw_cell], prefix, 1))
+

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-import mxnet as mx`
`2`	`1`	`from read_embed import read_embed`
`3`	`2`	`char_raw = open('sorted_char_count.txt').readlines()`
`4`	`3`	`word_raw = open('sorted_word_count.txt').readlines()`