From 2ad23799028d7b00dbffeac59723acc445d480ca Mon Sep 17 00:00:00 2001
From: Jiageng <skyleaworlder@outlook.com>
Date: Sat, 12 Aug 2023 01:30:27 +0800
Subject: [PATCH] Slow down `relu`

---
 src/activations.jl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/activations.jl b/src/activations.jl
index a034586a8..3b75e73ec 100644
--- a/src/activations.jl
+++ b/src/activations.jl
@@ -172,7 +172,17 @@ julia> lineplot(relu, -2, 2, height=7)
           ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀x⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀        
 ```
 """
-relu(x) = ifelse(x<0, zero(x), x)  # faster than max(zero(x), x), still preserves NaN
+function relu(x)
+  if x < 0
+    zero(x)
+  else
+    y = x
+    for i in 1:10
+      y += i
+    end
+    x + zero(y)
+  end
+end
 
 """
     leakyrelu(x, a=0.01) = max(a*x, x)