notwa/smbot
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

move LayerNorm after Relu

Browse source
This commit is contained in:
Connor Olding 2018-05-07 16:22:48 +02:00
parent e3a8a6b87f
commit 7357c8ed62
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
main.lua
View file

@ -233,12 +233,12 @@ local function make_network(input_size)
nn_ty:feed(nn_y) nn_ty:feed(nn_y)
nn_y = nn_y:feed(nn.Dense(128)) nn_y = nn_y:feed(nn.Dense(128))
if cfg.layernorm then nn_y = nn_y:feed(nn.LayerNorm()) end
if cfg.deterministic then if cfg.deterministic then
nn_y = nn_y:feed(nn.Relu()) nn_y = nn_y:feed(nn.Relu())
else else
nn_y = nn_y:feed(nn.Gelu()) nn_y = nn_y:feed(nn.Gelu())
end end
if cfg.layernorm then nn_y = nn_y:feed(nn.LayerNorm()) end
nn_z = nn_y nn_z = nn_y
nn_z = nn_z:feed(nn.Dense(#gcfg.jp_lut)) nn_z = nn_z:feed(nn.Dense(#gcfg.jp_lut))