notwa/smbot
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use local in LayerNorm

Browse Source
This commit is contained in:
Connor Olding 2018-06-12 05:36:57 +02:00
parent 50a7ba78f9
commit 6c554e0f49
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
nn.lua
View File

@ -674,6 +674,7 @@ end
function LayerNorm:forward(X)
local bs = checkshape(X, self.shape_in)
if self.bs ~= bs then self:reset_cache(bs) end
local Y = self.cache
local mean = 0
for i, v in ipairs(X) do
@ -683,16 +684,16 @@ function LayerNorm:forward(X)
local var = 0
for i, v in ipairs(X) do
local delta = v - mean
self.cache[i] = delta
Y[i] = delta
var = var + delta * delta / #X
end
local std = sqrt(var + self.eps)
for i, v in ipairs(self.cache) do
self.cache[i] = v / std
for i, v in ipairs(Y) do
Y[i] = v / std
end
return self.cache
return Y
end
function Model:init(nodes_in, nodes_out)