Merge pull request torch#849 from jonathantompson/relu6

Added ReLU6 layer, test and doc.

Author: soumith

ReLU6.lua

@@ -0,0 +1,35 @@
+local ReLU6, parent = torch.class('nn.ReLU6', 'nn.Module')
+
+function ReLU6:__init(inplace)
+   parent.__init(self)
+   
+   if inplace == nil then
+      self.inplace = false
+   else
+      self.inplace = inplace
+   end
+
+   if (inplace and type(inplace) ~= 'boolean') then
+      error('in-place flag must be boolean')
+   end
+end
+
+function ReLU6:updateOutput(input)
+   input.THNN.ReLU6_updateOutput(
+      input:cdata(),
+      self.output:cdata(),
+      self.inplace
+   )
+   return self.output
+end
+
+function ReLU6:updateGradInput(input, gradOutput)
+   input.THNN.ReLU6_updateGradInput(
+      input:cdata(),
+      gradOutput:cdata(),
+      self.gradInput:cdata(),
+      self.inplace
+   )
+   return self.gradInput
+end
+

doc/image/relu6.png

@@ -261,6 +261,29 @@ gnuplot.grid(true)
 ```
 ![](image/relu.png)
 
+<a name="nn.ReLU6"></a>
+## ReLU6 ##
+
+Same as `ReLU` except that the rectifying function `f(x)` saturates at `x = 6`. This layer is useful for training networks that do not loose precision (due to FP saturation) when implemented as FP16.
+
+`ReLU6` is defined as `f(x)` = `min(max(0, x), 6)`
+
+Can optionally do its operation in-place without using extra state memory:
+```lua
+m=nn.ReLU6(true) -- true = in-place, false = keeping separate state.
+```
+
+```lua
+ii=torch.linspace(-3, 9)
+m=nn.ReLU6() 
+oo=m:forward(ii)
+go=torch.ones(100)
+gi=m:backward(ii,go)
+gnuplot.plot({'f(x)',ii,oo,'+-'},{'df/dx',ii,gi,'+-'})
+gnuplot.grid(true)
+```
+![](image/relu6.png)
+
 <a name="nn.PReLU"></a>
 ## PReLU ##
 

doc/transfer.md

@@ -83,6 +83,7 @@ require('nn.HardShrink')
 require('nn.SoftShrink')
 require('nn.Threshold')
 require('nn.ReLU')
+require('nn.ReLU6')
 require('nn.PReLU')
 require('nn.LeakyReLU')
 require('nn.SpatialSoftMax')

init.lua

@@ -0,0 +1,58 @@
+#ifndef TH_GENERIC_FILE
+#define TH_GENERIC_FILE "generic/ReLU6.c"
+#else
+
+void THNN_(ReLU6_updateOutput)(
+          THNNState *state,
+          THTensor *input,
+          THTensor *output,
+          bool inplace)
+{
+  if (inplace)
+  {
+    TH_TENSOR_APPLY(real, input,
+      if (*input_data <= 0)
+        *input_data = 0;
+      else if (*input_data >= 6)
+        *input_data = 6;
+    );
+    THTensor_(set)(output, input);
+  }
+  else
+  {
+    THTensor_(resizeAs)(output, input);
+    TH_TENSOR_APPLY2(real, output, real, input,
+      *output_data =
+         (*input_data > 0) ? ((*input_data < 6) ? *input_data : 6) : 0;
+    );
+  }
+}
+
+void THNN_(ReLU6_updateGradInput)(
+          THNNState *state,
+          THTensor *input,
+          THTensor *gradOutput,
+          THTensor *gradInput,
+          bool inplace)
+{
+  if (inplace)
+  {
+    TH_TENSOR_APPLY2(real, gradOutput, real, input,
+      if ((*input_data) <= 0 || (*input_data) >= 6)
+        *gradOutput_data = 0;
+    );
+    THTensor_(set)(gradInput, gradOutput);
+  }
+  else
+  {
+    THTensor_(resizeAs)(gradInput, input);
+    TH_TENSOR_APPLY3(real, gradInput, real, gradOutput, real, input,
+      if ((*input_data) > 0 && (*input_data) < 6)
+        *gradInput_data = *gradOutput_data;
+      else
+        *gradInput_data = 0;
+    );
+  }
+}
+
+#endif

lib/THNN/generic/ReLU6.c

@@ -472,6 +472,18 @@ TH_API void THNN_(Threshold_updateGradInput)(
           real threshold,
           bool inplace);
 
+TH_API void THNN_(ReLU6_updateOutput)(
+          THNNState *state,
+          THTensor *input,
+          THTensor *output,
+          bool inplace);
+TH_API void THNN_(ReLU6_updateGradInput)(
+          THNNState *state,
+          THTensor *input,
+          THTensor *gradOutput,
+          THTensor *gradInput,
+          bool inplace);
+
 TH_API void THNN_(TemporalConvolution_updateOutput)(
           THNNState *state,
           THTensor *input,

lib/THNN/generic/THNN.h

@@ -94,6 +94,9 @@
 #include "generic/Threshold.c"
 #include "THGenerateFloatTypes.h"
 
+#include "generic/ReLU6.c"
+#include "THGenerateFloatTypes.h"
+
 #include "generic/TemporalConvolution.c"
 #include "THGenerateFloatTypes.h"
 

lib/THNN/init.c

@@ -263,6 +263,23 @@ function nntest.ReLU()
    mytester:assertTensorEq(gradInput, gradInput2, 0.000001, 'ReLU gradInput')
 end
 
+function nntest.ReLU6()
+   for inplace = 0, 1 do
+      local input = torch.randn(3, 4):mul(6)
+      local gradOutput = torch.randn(3,4)
+      local module = nn.ReLU6(inplace == 1)
+      local output = module:forward(input:clone())
+      local gt = input:clone():gt(input, 0)
+      local lt = input:clone():lt(input, 6)
+      local output2 = gt:clone():cmul(lt):cmul(input)
+      output2:add(6, input:clone():gt(input, 6))
+      mytester:assertTensorEq(output, output2, 0.000001, 'ReLU6 output')
+      local gradInput = module:backward(input, gradOutput:clone())
+      local gradInput2 = gt:clone():cmul(lt):cmul(gradOutput)
+      mytester:assertTensorEq(gradInput, gradInput2, 0.000001, 'ReLU gradInput')
+   end
+end
+
 function nntest.Exp()
    local ini = math.random(3,5)
    local inj = math.random(3,5)