Merge pull request #34 from torch/measure

Measure score functions added

Author: nicholas-leonard

CMakeLists.txt

@@ -84,6 +84,7 @@ SET(luasrc
   TotalDropout.lua
   VRClassReward.lua
   ReverseUnreverse.lua
+  measure.lua
   deprecated/SeqLSTMP.lua
   deprecated/SeqReverseSequence.lua
   deprecated/BiSequencerLM.lua

init.lua

@@ -27,7 +27,6 @@ paths.require 'librnn'
 unpack = unpack or table.unpack
 
 require('rnn.utils')
-
 -- extensions to existing nn.Module
 require('rnn.Module')
 require('rnn.Container')
@@ -112,6 +111,9 @@ require('rnn.SeqLSTMP')
 require('rnn.SeqReverseSequence')
 require('rnn.BiSequencerLM')
 
+
+require('rnn.measure')
+
 -- prevent likely name conflicts
 nn.rnn = rnn
 

measure.lua

@@ -0,0 +1,94 @@
+function nn.get_bleu(cand, ref, n)
+   n = n or 4
+   local smooth = 1
+   if type(cand) ~= 'table' then
+      cand = cand:totable()
+   end
+   if type(ref) ~= 'table' then
+      ref = ref:totable()
+   end
+   local res = nn.utils.get_ngram_prec(cand, ref, n)
+   local brevPen = math.exp(1-math.max(1, #ref/#cand))
+   local correct = 0
+   local total = 0
+   local bleu = 1
+   for i = 1, n do
+      if res[i][1] > 0 then
+         if res[i][2] == 0 then
+            smooth = smooth*0.5
+            res[i][2] = smooth
+         end
+         local prec = res[i][2]/res[i][1]
+         bleu = bleu * prec
+      end
+   end
+   bleu = bleu^(1/n)
+   return bleu*brevPen
+end
+
+function nn.get_rougeN(cand, ref, n, weight)
+   n = n or 4
+   weight = weight or {}
+   if #weight == 0 then
+      for i=1, n do
+	 weight[i] = 0
+      end
+      weight[n] = 1
+   end
+   if type(cand) ~= 'table' then
+      cand = cand:totable()
+   end
+   if type(ref) ~= 'table' then
+      ref = ref:totable()
+   end
+   local res = nn.utils.get_ngram_recall(cand, ref, n)
+   local correct = 0
+   local total = 0
+   local rouge = 0
+   weight_sum = 0
+   
+   for i = 1, n do
+      local recall = res[i][2]/res[i][1]
+      rouge = rouge + recall*weight[i]
+      weight_sum = weight_sum + weight[i]
+   end
+   rouge = rouge/weight_sum
+   return rouge
+end
+
+function nn.get_rougeS(cand, ref, beta, dskip)
+   local beta = beta or 1
+   beta = beta * beta
+
+   local dskip = dskip or (#cand)
+   dskip = math.min(dskip, #cand)
+   if type(cand) ~= 'table' then
+      cand = cand:totable()
+   end
+   if type(ref) ~= 'table' then
+      ref = ref:totable()
+   end
+   local cand_unigrams = nn.utils.get_ngrams(cand, 1)
+   local ref_unigrams = nn.utils.get_ngrams(ref, 1)
+
+   local cand_skip_bigrams = nn.utils.get_skip_bigrams(cand, ref_unigrams, 1, dskip)
+   local ref_skip_bigrams = nn.utils.get_skip_bigrams(ref, cand_unigrams, 1, dskip)
+   local correct = 0
+   
+   for bigram, freq in pairs(ref_skip_bigrams) do
+      local actual
+      if cand_skip_bigrams[bigram] == nil then
+         actual = 0
+      else
+         actual = cand_skip_bigrams[bigram]
+      end
+     correct = correct + math.min(actual, freq)
+   end
+   local total_skip_bigrams_ref = (dskip - 1)*(2 * #ref - dskip)/2
+   local total_skip_bigrams_cand = (dskip - 1)*(2 * #cand - dskip)/2
+   local rskip2 = correct/total_skip_bigrams_cand
+   local pskip2 = correct/total_skip_bigrams_ref
+   local rouge = (1 + beta)*rskip2*pskip2/(rskip2 + beta*pskip2)
+   return rouge
+end
+

scripts/evaluate-rnnlm.lua

@@ -42,77 +42,6 @@ local validerr = xplog.valnceloss or xplog.valppl
 
 print(string.format("Error (epoch=%d): training=%f; validation=%f", xplog.epoch, trainerr[#trainerr], validerr[#validerr]))
 
-
-local function get_ngrams(sent, n, count)
-   local ngrams = {}
-   for beg = 1, #sent do
-      for  last= beg, math.min(beg+n-1, #sent) do
-         local ngram = table.concat(sent, ' ', beg, last)
-         local len = last-beg+1 -- keep track of ngram length
-         if not count then
-            table.insert(ngrams, ngram)
-         else
-            if ngrams[ngram] == nil then
-               ngrams[ngram] = {1, len}
-            else
-               ngrams[ngram][1] = ngrams[ngram][1] + 1
-            end
-         end
-      end
-   end
-   return ngrams
-end
-
-local function get_ngram_prec(cand, ref, n)
-   local results = {}
-   for i = 1, n do
-      results[i] = {0, 0}
-   end
-   local cand_ngrams = get_ngrams(cand, n, 1)
-   local ref_ngrams = get_ngrams(ref, n, 1)
-   for ngram, dist in pairs(cand_ngrams) do
-      local freq = dist[1]
-      local length = dist[2]
-      results[length][1] = results[length][1] + freq
-      local actual
-      if ref_ngrams[ngram] == nil then
-         actual = 0
-      else
-         actual = ref_ngrams[ngram][1]
-      end
-      results[length][2] = results[length][2] + math.min(actual, freq)
-   end
-   return results
-end
-
-function get_bleu(cand, ref, n)
-   n = n or 4
-   local smooth = 1
-   if type(cand) ~= 'table' then
-      cand = cand:totable()
-   end
-   if type(ref) ~= 'table' then
-      ref = ref:totable()
-   end
-   local res = get_ngram_prec(cand, ref, n)
-   local brevPen = math.exp(1-math.max(1, #ref/#cand))
-   local correct = 0
-   local total = 0
-   local bleu = 1
-   for i = 1, n do
-      if res[i][1] > 0 then
-         if res[i][2] == 0 then
-            smooth = smooth*0.5
-            res[i][2] = smooth
-         end
-         local prec = res[i][2]/res[i][1]
-         bleu = bleu * prec
-      end
-   end
-   bleu = bleu^(1/n)
-   return bleu*brevPen
-end
-
 if opt.dumpcsv then
    local csvfile = opt.xplogpath:match('([^/]+)[.]t7$')..'.csv'
    paths.mkdir('learningcurves')
@@ -220,7 +149,7 @@ else
       if opt.bleu then
          max_ind = torch.multinomial(torch.exp(outputs:view(targets:nElement(), -1)), 1):view(targets:size(1),targets:size(2))
             for batchIdx=1, targets:size(2) do
-               sum_bleu = sum_bleu + get_bleu(max_ind:select(2, batchIdx),
+               sum_bleu = sum_bleu + nn.get_bleu(max_ind:select(2, batchIdx),
                                               targets:select(2, batchIdx),
                                               opt.blueN)
                num_sent = num_sent + 1

test/test.lua

@@ -6464,6 +6464,39 @@ function rnntest.NCE_multicuda()
    mytester:assertTensorEq(nce2.gradWeight[{{},{1+(hiddensize/2), hiddensize}}]:float(), nce.gradWeight.tensors[2]:float(), 0.000001)
 end
 
+function rnntest.bleu()
+   local cand = {1, 2, 3, 2, 3, 2, 3, 4, 5, 2, 1, 3 ,2 ,3}
+   local ref = {3, 2, 3, 2, 1, 3, 2, 3 ,2 , 3, 4, 5, 2, 1, 3}
+   local bleu = nn.get_bleu(cand, ref, 4)
+   mytester:assert(math.abs(bleu - 0.83101069788036) < 0.000001)
+   
+end
+
+function rnntest.get_rougeN()
+   
+   local cand = {1, 2, 3, 2, 3, 2, 3, 4, 5, 2, 1, 3 ,2 ,3}
+   local ref = {3, 2, 3, 2, 1, 3, 2, 3 ,2 , 3, 4, 5, 2, 1, 3}
+   local rouge = nn.get_rougeN(cand, ref, 4)
+   mytester:assert(math.abs(rouge - 0.75)< 0.000000000001)
+end
+
+
+function rnntest.get_rougeS()
+   local cand_tbl = {
+      'police kill the gunman',
+      'the gunman kill police',
+      'the gunman police killed'
+   }
+   local ref_str = "police killed the gunman"
+   local ref = ref_str:split(" ")
+   local rouge = nn.get_rougeS(cand_tbl[1]:split(" "), ref, 1, 4)
+   mytester:assert(math.abs(rouge - 1/2)< 0.000000000001)
+   local rouge = nn.get_rougeS(cand_tbl[2]:split(" "), ref, 1, 4)
+   mytester:assert(math.abs(rouge - 1/6)< 0.000000000001)
+   local rouge = nn.get_rougeS(cand_tbl[3]:split(" "), ref, 1, 4)
+   mytester:assert(math.abs(rouge - 1/3)< 0.000000000001)
+end
+
 function rnn.test(tests, exclude, benchmark_)
    benchmark = benchmark_
    mytester = torch.Tester()

utils.lua

@@ -290,4 +290,89 @@ function nn.utils.setZeroMask(modules, zeroMask, cuda)
    for i,module in ipairs(torch.type(modules) == 'table' and modules or {modules}) do
       module:setZeroMask(zeroMask)
    end
-end
+end
+function nn.utils.get_ngrams(sent, n, count)
+   local ngrams = {}
+   for beg = 1, #sent do
+      for  last= beg, math.min(beg+n-1, #sent) do
+         local ngram = table.concat(sent, ' ', beg, last)
+	 local len = last-beg+1 -- keep track of ngram length
+         if not count then
+            ngrams[ngram] = 1
+         else
+            if ngrams[ngram] == nil then
+               ngrams[ngram] = {1, len}
+            else
+               ngrams[ngram][1] = ngrams[ngram][1] + 1
+            end
+         end
+      end
+   end
+   return ngrams
+end
+
+function nn.utils.get_skip_bigrams(sent, ref, count, dskip)
+   local skip_bigrams = {}
+   ref = ref or sent
+   for beg = 1, #sent do
+      if ref[sent[beg]] then
+	 local temp_token = sent[beg]
+	 for  last= beg+1, math.min(beg + dskip-1, #sent) do
+	    if ref[sent[last]] then
+	       skip_bigram = temp_token..sent[last]
+	       if not count then
+		  skip_bigrams[skip_bigram] = 1
+	       else
+		  skip_bigrams[skip_bigram] = (skip_bigram[bigram] or 0) + 1
+	       end
+	    end
+	 end
+      end
+   end
+   return skip_bigrams
+end
+
+
+function nn.utils.get_ngram_prec(cand, ref, n)
+   local results = {}
+   for i = 1, n do
+      results[i] = {0, 0}
+   end
+   local cand_ngrams = nn.utils.get_ngrams(cand, n, 1)
+   local ref_ngrams = nn.utils.get_ngrams(ref, n, 1)
+   for ngram, dist in pairs(cand_ngrams) do
+      local freq = dist[1]
+      local length = dist[2]
+      results[length][1] = results[length][1] + freq
+      local actual
+      if ref_ngrams[ngram] == nil then
+         actual = 0
+      else
+         actual = ref_ngrams[ngram][1]
+      end
+      results[length][2] = results[length][2] + math.min(actual, freq)
+   end
+   return results
+end
+
+function nn.utils.get_ngram_recall(cand, ref, n)
+   local results = {}
+   for i = 1, n do
+      results[i] = {0, 0}
+   end
+   local cand_ngrams = nn.utils.get_ngrams(cand, n, 1)
+   local ref_ngrams = nn.utils.get_ngrams(ref, n, 1)
+   for ngram, dist in pairs(ref_ngrams) do
+      local freq = dist[1]
+      local length = dist[2]
+      results[length][1] = results[length][1] + freq
+      local actual
+      if cand_ngrams[ngram] == nil then
+         actual = 0
+      else
+         actual = cand_ngrams[ngram][1]
+      end
+      results[length][2] = results[length][2] + math.min(actual, freq)
+   end
+   return results
+end