From d3c23912c18196b16df6d9df38aaa9f812b4cf43 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sun, 23 Apr 2017 17:40:47 +0000 Subject: [PATCH] add Softplus and LeCunTanh; use LeCunTanh --- optim_nn.py | 16 +++++++++------- optim_nn_core.py | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/optim_nn.py b/optim_nn.py index 0858d09..6ae5219 100755 --- a/optim_nn.py +++ b/optim_nn.py @@ -468,8 +468,10 @@ def multiresnet(x, width, depth, block=2, multi=1, # Toy Data {{{1 -inits = dict(he_normal=init_he_normal, he_uniform=init_he_uniform) -activations = dict(sigmoid=Sigmoid, tanh=Tanh, relu=Relu, elu=Elu, gelu=GeluApprox) +inits = dict(he_normal=init_he_normal, he_uniform=init_he_uniform, + glorot_normal=init_glorot_normal, glorot_uniform=init_glorot_uniform) +activations = dict(sigmoid=Sigmoid, tanh=Tanh, lecun=LeCunTanh, + relu=Relu, elu=Elu, gelu=GeluApprox, softplus=Softplus) def prettyize(data): if isinstance(data, np.ndarray): @@ -692,10 +694,10 @@ def run(program, args=None): # style of resnet (order of layers, which layers, etc.) parallel_style = 'onelesssum', - activation = 'gelu', + activation = 'lecun', optim = 'adam', # note: most features only implemented for Adam - optim_decay1 = 2, # first momentum given in epochs (optional) + optim_decay1 = 24, # first momentum given in epochs (optional) optim_decay2 = 100, # second momentum given in epochs (optional) nesterov = True, batch_size = 64, @@ -705,13 +707,13 @@ def run(program, args=None): learn = 1e-2, epochs = 24, learn_halve_every = 16, # only used with anneal/dumb - restarts = 8, + restarts = 5, restart_decay = 0.25, # only used with SGDR expando = lambda i: 24 * i, # misc - init = 'he_normal', - loss = 'msee', + init = 'glorot_uniform', + loss = 'mse', mloss = 'mse', ritual = 'default', restart_optim = False, # restarts also reset internal state of optimizer diff --git a/optim_nn_core.py b/optim_nn_core.py index 86c1f69..feba346 100644 --- a/optim_nn_core.py +++ b/optim_nn_core.py @@ -570,7 +570,7 @@ class Dropout(Layer): # Activation Layers {{{2 -class Sigmoid(Layer): # aka Logistic +class Sigmoid(Layer): # aka Logistic, Expit (inverse of Logit) def forward(self, X): self.sig = sigmoid(X) return self.sig @@ -578,6 +578,16 @@ class Sigmoid(Layer): # aka Logistic def backward(self, dY): return dY * self.sig * (1 - self.sig) +class Softplus(Layer): + # integral of Sigmoid. + + def forward(self, X): + self.X = X + return np.log(1 + np.exp(X)) + + def backward(self, dY): + return sigmoid(self.X) + class Tanh(Layer): def forward(self, X): self.sig = np.tanh(X) @@ -586,6 +596,20 @@ class Tanh(Layer): def backward(self, dY): return dY * (1 - self.sig * self.sig) +class LeCunTanh(Layer): + # paper: http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf + # paper: http://yann.lecun.com/exdb/publis/pdf/lecun-89.pdf + # scaled such that f([-1, 1]) = [-1, 1]. + # helps preserve an input variance of 1. + # second derivative peaks around an input of ±1. + + def forward(self, X): + self.sig = np.tanh(2 / 3 * X) + return 1.7159 * self.sig + + def backward(self, dY): + return dY * (2 / 3 * 1.7159) * (1 - self.sig * self.sig) + class Relu(Layer): def forward(self, X): self.cond = X >= 0