From bb44d6696e71db56ecf9ba896bca39e5c4473bd1 Mon Sep 17 00:00:00 2001
From: Connor Olding <cloningdonor@gmail.com>
Date: Mon, 26 Mar 2018 16:33:23 +0200
Subject: [PATCH] remove backprop code

---
 nn.lua | 69 ----------------------------------------------------------
 1 file changed, 69 deletions(-)

diff --git a/nn.lua b/nn.lua
index 3887684..8fcb092 100644
--- a/nn.lua
+++ b/nn.lua
@@ -355,10 +355,6 @@ function Layer:forward_deterministic(...)
     return self:forward(...)
 end
 
-function Layer:backward()
-    error("Unimplemented.")
-end
-
 function Layer:_new_weights(init)
     local w = Weights(init)
     insert(self.weights, w)
@@ -419,11 +415,6 @@ function Input:forward(X)
     return X
 end
 
-function Input:backward(dY)
-    checkshape(dY, self.shape_out)
-    return zeros(#dY)
-end
-
 function Merge:init()
     Layer.init(self, "Merge")
     self.size = 0
@@ -483,17 +474,6 @@ function Relu:forward(X)
     return Y
 end
 
-function Relu:backward(dY)
-    local bs = checkshape(dY, self.shape_out)
-    local Y = self.cache
-    local dX = self.dcache
-
-    for i = 1, #dY do dX[i] = Y[i] >= 0 and dY[i] or 0 end
-
-    checkshape(dX, self.shape_in)
-    return dX
-end
-
 function Gelu:init()
     Layer.init(self, "Gelu")
 end
@@ -525,21 +505,6 @@ function Gelu:forward(X)
     return Y
 end
 
-function Gelu:backward(dY)
-    checkshape(dY, self.shape_out)
-    local Y = self.cache
-    local a = self.cache_a
-    local sig = self.cache_sig
-    local dX = self.dcache
-
-    for i = 1, #dY do
-        dX[i] = dY[i] * sig[i] * (1 + a[i] * (1 - sig[i]))
-    end
-
-    checkshape(dX, self.shape_in)
-    return dX
-end
-
 function Dense:init(dim)
     Layer.init(self, "Dense")
     assert(type(dim) == "number")
@@ -568,11 +533,6 @@ function Dense:forward(X)
     if self.bs ~= bs then self:reset_cache(bs) end
     local Y = self.cache
 
-    for i = 1, #X do
-        -- only needed for backwards pass.
-        self.cache_x[i] = X[i]
-    end
-
     --dot_1aab(X, self.coeffs, Y)
     dot(X, self.coeffs, 2, 1, Y)
 
@@ -584,26 +544,6 @@ function Dense:forward(X)
     return Y
 end
 
-function Dense:backward(dY)
-    local X = self.cache_x
-    local dX = self.dcache
-
-    --dot_ta(X, dY, self.coeffs.g)
-    dot(X, dY, 1, 1, self.coeffs.g)
-
-    for b = 1, X.shape[1] do
-        local l = X.shape[2]
-        local j = (b - 1) * l
-        for i = 1, l do self.biases.g[i] = self.biases.g[i] + dY[j-1+i] end
-    end
-
-    --dot_tb(dY, self.coeffs, dX)
-    dot(dY, self.coeffs, 2, 2, dX)
-
-    checkshape(dX, self.shape_in)
-    return dX
-end
-
 function Softmax:init()
     Layer.init(self, "Softmax")
 end
@@ -636,10 +576,6 @@ function Softmax:forward(X)
     return Y
 end
 
---function Softmax:backward(dY)
-    --return (dY - np.sum(dY * self.sm, axis=-1, keepdims=True)) * self.cache
---end
-
 function Embed:init(vocab, dim)
     Layer.init(self, "Embed")
     assert(type(vocab) == "number")
@@ -667,11 +603,6 @@ function Embed:forward(X)
     if self.bs ~= bs then self:reset_cache(bs) end
     local Y = self.cache
 
-    for i = 1, #X do
-        -- only needed for backwards pass.
-        self.cache_x[i] = X[i]
-    end
-
     local yi = 0
     for i, x in ipairs(X) do
         local xi = x * self.dim