use local in LayerNorm
This commit is contained in:
parent
50a7ba78f9
commit
6c554e0f49
9
nn.lua
9
nn.lua
|
@ -674,6 +674,7 @@ end
|
||||||
function LayerNorm:forward(X)
|
function LayerNorm:forward(X)
|
||||||
local bs = checkshape(X, self.shape_in)
|
local bs = checkshape(X, self.shape_in)
|
||||||
if self.bs ~= bs then self:reset_cache(bs) end
|
if self.bs ~= bs then self:reset_cache(bs) end
|
||||||
|
local Y = self.cache
|
||||||
|
|
||||||
local mean = 0
|
local mean = 0
|
||||||
for i, v in ipairs(X) do
|
for i, v in ipairs(X) do
|
||||||
|
@ -683,16 +684,16 @@ function LayerNorm:forward(X)
|
||||||
local var = 0
|
local var = 0
|
||||||
for i, v in ipairs(X) do
|
for i, v in ipairs(X) do
|
||||||
local delta = v - mean
|
local delta = v - mean
|
||||||
self.cache[i] = delta
|
Y[i] = delta
|
||||||
var = var + delta * delta / #X
|
var = var + delta * delta / #X
|
||||||
end
|
end
|
||||||
|
|
||||||
local std = sqrt(var + self.eps)
|
local std = sqrt(var + self.eps)
|
||||||
for i, v in ipairs(self.cache) do
|
for i, v in ipairs(Y) do
|
||||||
self.cache[i] = v / std
|
Y[i] = v / std
|
||||||
end
|
end
|
||||||
|
|
||||||
return self.cache
|
return Y
|
||||||
end
|
end
|
||||||
|
|
||||||
function Model:init(nodes_in, nodes_out)
|
function Model:init(nodes_in, nodes_out)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user