From 7f34de8e7c546dbcdf9d04aa51471a51df77949c Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sat, 12 May 2018 21:51:00 +0200 Subject: [PATCH] add cosine activation --- nn.lua | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nn.lua b/nn.lua index 16e5c88..b2d0b55 100644 --- a/nn.lua +++ b/nn.lua @@ -308,6 +308,7 @@ local Input = Layer:extend() local Merge = Layer:extend() local Relu = Layer:extend() local Gelu = Layer:extend() +local Cos = Layer:extend() local Dense = Layer:extend() local Softmax = Layer:extend() local Embed = Layer:extend() @@ -508,6 +509,28 @@ function Gelu:forward(X) return Y end +function Cos:init() + Layer.init(self, "Cos") +end + +function Cos:reset_cache(bs) + self.bs = bs + + self.cache = cache(bs, self.shape_out) + self.dcache = cache(bs, self.shape_in) +end + +function Cos:forward(X) + local bs = checkshape(X, self.shape_in) + if bs ~= self.bs then self:reset_cache(bs) end + local Y = self.cache + + for i = 1, #X do Y[i] = cos(X[i]) end + + checkshape(Y, self.shape_out) + return Y +end + function Dense:init(dim) Layer.init(self, "Dense") assert(type(dim) == "number") @@ -808,6 +831,7 @@ return { Merge = Merge, Relu = Relu, Gelu = Gelu, + Cos = Cos, Dense = Dense, Softmax = Softmax, Embed = Embed,