diff --git a/NCEModule.lua b/NCEModule.lua index ab03800..632a350 100644 --- a/NCEModule.lua +++ b/NCEModule.lua @@ -255,4 +255,57 @@ function NCEModule:clearState() self.gradInput[2]:set() end --- TODO : speedup unigram sampling using frequency bins... +-- NOT IN USE : the following is experimental and not currently in use. +-- ref.: https://hips.seas.harvard.edu/blog/2013/03/03/the-alias-method-efficient-sampling-with-many-discrete-outcomes/ +function NCEModule:aliasDraw(J, q) + local K = J:nElement() + + -- Draw from the overall uniform mixture. + local kk = math.random(1,K) + + -- Draw from the binary mixture, either keeping the + -- small one, or choosing the associated larger one. + if math.random() < q[kk] then + return kk + else + return J[kk] + end +end + +function NCEModule:aliasSetup(probs) + assert(probs:dim() == 1) + local K = probs:nElement() + local q = probs.new(K) + local J = torch.LongTensor(K):zero() + + -- Sort the data into the outcomes with probabilities + -- that are larger and smaller than 1/K. + local smaller, larger = {}, {} + for kk = 1,K do + local prob = probs[kk] + q[kk] = K*prob + if q[kk] < 1 then + table.insert(smaller, kk) + else + table.insert(larger, kk) + end + end + + -- Loop though and create little binary mixtures that + -- appropriately allocate the larger outcomes over the + -- overall uniform mixture. + while #smaller > 0 and #larger > 0 do + local small = table.remove(smaller) + local large = table.remove(larger) + + J[small] = large + q[large] = q[large] - (1.0 - q[small]) + + if q[large] < 1.0 then + table.insert(smaller,large) + else + table.insert(larger,large) + end + end + return J, q +end diff --git a/Sequential.lua b/Sequential.lua index 998403e..a33c47d 100644 --- a/Sequential.lua +++ b/Sequential.lua @@ -5,10 +5,10 @@ function Sequential:profile() function Sequential:updateOutput(input) local currentOutput = input for i=1,#self.modules do - local start = sys.clock() + local start = torch.Timer() currentOutput = self.modules[i]:updateOutput(currentOutput) if cutorch then cutorch.synchronize() end - print(torch.type(self.modules[i])..' updateOutput: '..sys.clock() - start.." s") + print(torch.type(self.modules[i])..' updateOutput: '..start:time().real.." s") end self.output = currentOutput return currentOutput @@ -19,16 +19,16 @@ function Sequential:profile() local currentModule = self.modules[#self.modules] for i=#self.modules-1,1,-1 do local previousModule = self.modules[i] - local start = sys.clock() + local start = torch.Timer() currentGradOutput = currentModule:updateGradInput(previousModule.output, currentGradOutput) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' updateGradInput: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' updateGradInput: '..start:time().real.." s") currentModule = previousModule end - local start = sys.clock() + local start = torch.Timer() currentGradOutput = currentModule:updateGradInput(input, currentGradOutput) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' updateGradInput: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' updateGradInput: '..start:time().real.." s") self.gradInput = currentGradOutput return currentGradOutput end @@ -40,18 +40,18 @@ function Sequential:profile() local currentModule = self.modules[#self.modules] for i=#self.modules-1,1,-1 do local previousModule = self.modules[i] - local start = sys.clock() + local start = torch.Timer() currentModule:accGradParameters(previousModule.output, currentGradOutput, scale) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' accGradParameters: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' accGradParameters: '..start:time().real.." s") currentGradOutput = currentModule.gradInput currentModule = previousModule end - local start = sys.clock() + local start = torch.Timer() currentModule:accGradParameters(input, currentGradOutput, scale) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' accGradParameters: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' accGradParameters: '..start:time().real.." s") end function Sequential:backward(input, gradOutput, scale) @@ -60,17 +60,17 @@ function Sequential:profile() local currentModule = self.modules[#self.modules] for i=#self.modules-1,1,-1 do local previousModule = self.modules[i] - local start = sys.clock() + local start = torch.Timer() currentGradOutput = currentModule:backward(previousModule.output, currentGradOutput, scale) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' backward: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' backward: '..start:time().real.." s") currentModule.gradInput = currentGradOutput currentModule = previousModule end - local start = sys.clock() + local start = torch.Timer() currentGradOutput = currentModule:backward(input, currentGradOutput, scale) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' backward: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' backward: '..start:time().real.." s") self.gradInput = currentGradOutput return currentGradOutput end @@ -80,18 +80,18 @@ function Sequential:profile() local currentModule = self.modules[#self.modules] for i=#self.modules-1,1,-1 do local previousModule = self.modules[i] - local start = sys.clock() + local start = torch.Timer() currentModule:accUpdateGradParameters(previousModule.output, currentGradOutput, lr) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' accUpdateGradParameters: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' accUpdateGradParameters: '..start:time().real.." s") currentGradOutput = currentModule.gradInput currentModule = previousModule end - local start = sys.clock() + local start = torch.Timer() currentModule:accUpdateGradParameters(input, currentGradOutput, lr) if cutorch then cutorch.synchronize() end - print(torch.type(currentModule)..' accUpdateGradParameters: '..sys.clock() - start.." s") + print(torch.type(currentModule)..' accUpdateGradParameters: '..start:time().real.." s") end parent.profile(self) diff --git a/test/test.lua b/test/test.lua index 068420f..e370b59 100644 --- a/test/test.lua +++ b/test/test.lua @@ -282,36 +282,37 @@ function dpnntest.Module_type() mytester:assertTensorEq(gradParams[i], gradParams2[i], 0.00001, " gradParams err "..i) end + local input = torch.randn(3,32,32) + local cnn = nn.Sequential() + cnn:add(nn.SpatialConvolution(3,8,5,5)) + cnn:add(nn.ReLU()) + cnn:add(nn.SpatialAveragePooling(2,2,2,2)) + cnn:add(nn.SpatialConvolution(8,12,5,5)) + cnn:add(nn.ReLU()) + cnn:add(nn.SpatialAveragePooling(2,2,2,2)) + local outsize = cnn:outside{1,3,32,32} + cnn:add(nn.Collapse(3)) + cnn:add(nn.Linear(outsize[2]*outsize[3]*outsize[4],20)) + cnn:add(nn.ReLU()) + cnn:add(nn.Linear(20,10)) + local output = cnn:forward(input):clone() + local gradOutput = output:clone() + local gradInput = cnn:backward(input, gradOutput):clone() + cnn:float() + local input3 = input:float() + local output3 = cnn:forward(input3):clone() + local gradOutput3 = output3:clone() + local gradInput3 = cnn:backward(input3, gradOutput3):clone() + local o1, o2 = output3:float(), output:float() + mytester:assertTensorEq(o1, o2, 0.000001) + mytester:assertTensorEq(gradInput3:float(), gradInput:float(), 0.00001, "type float bwd err") if pcall(function() require 'cunn' end) then - local input = torch.randn(3,32,32) - local cnn = nn.Sequential() - cnn:add(nn.SpatialConvolutionMM(3,8,5,5)) - cnn:add(nn.ReLU()) - cnn:add(nn.SpatialAveragePooling(2,2,2,2)) - cnn:add(nn.SpatialConvolutionMM(8,12,5,5)) - cnn:add(nn.ReLU()) - cnn:add(nn.SpatialAveragePooling(2,2,2,2)) - local outsize = cnn:outside{1,3,32,32} - cnn:add(nn.Collapse(3)) - cnn:add(nn.Linear(outsize[2]*outsize[3]*outsize[4],20)) - cnn:add(nn.ReLU()) - cnn:add(nn.Linear(20,10)) - local output = cnn:forward(input):clone() - local gradOutput = output:clone() - local gradInput = cnn:backward(input, gradOutput):clone() - cnn:float() - local input3 = input:float() - local output3 = cnn:forward(input3):clone() - local gradOutput3 = output3:clone() - local gradInput3 = cnn:backward(input3, gradOutput3):clone() - mytester:assertTensorEq(output3:float(), output:float(), 0.000001, "type float fwd err") - mytester:assertTensorEq(gradInput3:float(), gradInput:float(), 0.00001, "type float bwd err") cnn:cuda() local input2 = input3:cuda() local gradOutput2 = gradOutput3:cuda() local output2 = cnn:forward(input2) local gradInput2 = cnn:backward(input2, gradOutput2) - mytester:assertTensorEq(output2:float(), output3, 0.000001, "type cuda fwd err") + mytester:assertTensorEq(output2:float(), output3, 0.000001) mytester:assertTensorEq(gradInput2:float(), gradInput3, 0.00001, "type cuda bwd err") end end