notwa/smbot
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

add cosine activation

Browse source
This commit is contained in:
Connor Olding 2018-05-12 21:51:00 +02:00
parent 7db43038ac
commit 7f34de8e7c
1 changed files with 24 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
nn.lua
View file

@ -308,6 +308,7 @@ local Input = Layer:extend()
local Merge = Layer:extend()
local Relu = Layer:extend()
local Gelu = Layer:extend()
local Cos = Layer:extend()
local Dense = Layer:extend()
local Softmax = Layer:extend()
local Embed = Layer:extend()
@ -508,6 +509,28 @@ function Gelu:forward(X)
return Y
end
function Cos:init()
Layer.init(self, "Cos")
end
function Cos:reset_cache(bs)
self.bs = bs
self.cache = cache(bs, self.shape_out)
self.dcache = cache(bs, self.shape_in)
end
function Cos:forward(X)
local bs = checkshape(X, self.shape_in)
if bs ~= self.bs then self:reset_cache(bs) end
local Y = self.cache
for i = 1, #X do Y[i] = cos(X[i]) end
checkshape(Y, self.shape_out)
return Y
end
function Dense:init(dim)
Layer.init(self, "Dense")
assert(type(dim) == "number")
@ -808,6 +831,7 @@ return {
Merge = Merge,
Relu = Relu,
Gelu = Gelu,
Cos = Cos,
Dense = Dense,
Softmax = Softmax,
Embed = Embed,