multiple models

Author: wasimusu

config.py

@@ -17,14 +17,14 @@ def __init__(self):
         self.use_cuda = False
 
         # Global dimension params
-        self.embedding_dim = 50
+        self.embedding_dim = 200
         self.hidden_size = self.embedding_dim
         self.context_len = 600
         self.question_len = 30
 
         # Training params
         self.num_epochs = 10
-        self.learning_rate = 0.00001
+        self.learning_rate = 0.001
         self.batch_size = 32
         self.l2_norm = 0.1
         self.max_grad_norm = 5
@@ -52,6 +52,7 @@ def __init__(self):
         # Logs
         self.print_every = 5
         self.save_every = 100
+        self.evaluate_every = 20
 
         # Vectors
         self.glove_base_url = "http://nlp.stanford.edu/data/"

data_util/vocab.py

@@ -45,7 +45,7 @@ def get_glove(glove_path, glove_dim):
     """
 
     print("Loading GLoVE vectors from file: %s" % glove_path)
-    vocab_size = int(4e5) # this is the vocab size of the corpus we've downloaded
+    vocab_size = int(4e5)  # this is the vocab size of the corpus we've downloaded
 
     emb_matrix = np.zeros((vocab_size + len(_START_VOCAB), glove_dim))
     word2id = {}
@@ -70,7 +70,9 @@ def get_glove(glove_path, glove_dim):
             word = line[0]
             vector = list(map(float, line[1:]))
             if glove_dim != len(vector):
-                raise Exception("You set --glove_path=%s but --embedding_size=%i. If you set --glove_path yourself then make sure that --embedding_size matches!" % (glove_path, glove_dim))
+                raise Exception(
+                    "You set --glove_path=%s but --embedding_size=%i. If you set --glove_path yourself then make sure that --embedding_size matches!" % (
+                    glove_path, glove_dim))
             emb_matrix[idx, :] = vector
             word2id[word] = idx
             id2word[idx] = word

networks.py

@@ -33,9 +33,14 @@ def __init__(self, emb_matrix,
         self.batch_size = batch_size
         self.hidden_size = hidden_size
 
-        self.encoder = nn.GRU(input_size=hidden_size, hidden_size=hidden_size, bidirectional=bidirectional,
-                              num_layers=num_layers,
-                              batch_first=True)
+        # self.encoder = nn.GRU(input_size=hidden_size, hidden_size=hidden_size, bidirectional=bidirectional,
+        #                       num_layers=num_layers,
+        #                       batch_first=True)
+
+        self.encoder = nn.LSTM(input_size=hidden_size, hidden_size=hidden_size, bidirectional=bidirectional,
+                               num_layers=num_layers,
+                               batch_first=True)
+
         self.hidden = self.init_hidden()
         self.sentinel = nn.Parameter(torch.rand(hidden_size, ))
 
@@ -70,7 +75,10 @@ def forward(self, inputs, mask):
         return output
 
     def init_hidden(self):
-        return torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size)
+        # return torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size)
+
+        return (torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size),
+                torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size))
 
 
 # TODO : Takes input and produces out of same dimension our reference implementation
@@ -86,9 +94,14 @@ def __init__(self, dropout_rate,
         self.batch_size = batch_size
         self.hidden_size = hidden_size
 
-        self.fusion_bilstm = nn.GRU(num_layers=num_layers, input_size=hidden_size * 3, hidden_size=hidden_size,
-                                    batch_first=True,
-                                    bidirectional=True)
+        # self.fusion_bilstm = nn.GRU(num_layers=num_layers, input_size=hidden_size * 3, hidden_size=hidden_size,
+        #                             batch_first=True,
+        #                             bidirectional=True)
+
+        self.fusion_bilstm = nn.LSTM(num_layers=num_layers, input_size=hidden_size * 3, hidden_size=hidden_size,
+                                     batch_first=True,
+                                     bidirectional=True)
+
         self.hidden = self.init_hidden()
         self.dropout = nn.Dropout(p=dropout_rate)
 
@@ -122,7 +135,10 @@ def forward(self, inputs, mask):
         return output
 
     def init_hidden(self):
-        return torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size)
+        # return torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size)
+
+        return (torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size),
+                torch.zeros(self.num_directions * self.num_layers, self.batch_size, self.hidden_size))
 
 
 class DynamicDecoder(nn.Module):
@@ -194,10 +210,10 @@ def forward(self, U, d_mask, target_span):
 
             # Get hidden state
             # TODO : There could be problem with the dimension
-            h_i = self.gru(u_cat.unsqueeze(1), h_i)[1]
+            output, h_i = self.gru(u_cat.unsqueeze(1), h_i)
 
             # Get new start estimate and start loss
-            s_i, _, start_loss_i = self.start_hmn(h_i, U, None, s_i, u_cat, None, s_target)
+            s_i, _, start_loss_i = self.start_hmn(output, U, None, s_i, u_cat, None, s_target)
             # s_i, start_loss_i = self.start_hmn(h_i, U, u_cat, s_target)
 
             # Update embedding at start estimate
@@ -207,7 +223,7 @@ def forward(self, U, d_mask, target_span):
             u_cat = torch.cat((u_s_i, u_e_i), 1)  # batch_size x 4l
 
             # Get new end estimate and end loss
-            e_i, _, end_loss_i = self.end_hmn(h_i, U, None, e_i, u_cat, None, e_target)
+            e_i, _, end_loss_i = self.end_hmn(output, U, None, e_i, u_cat, None, e_target)
             # e_i, end_loss_i = self.end_hmn(h_i, U, u_cat, e_target)
 
             # Update cumulative loss if computing loss
@@ -220,6 +236,7 @@ def forward(self, U, d_mask, target_span):
             loss = cumulative_loss / self.max_dec_steps
         return loss, s_i, e_i
 
+
 class CoattentionNetwork(nn.Module):
     def __init__(self, device,
                  hidden_size,