From 9c0cbba3fff50c96bcc09afd89083220e1004ce1 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Wed, 21 Jun 2017 00:13:53 +0000 Subject: [PATCH] move epsilon out of square-root (big improvement) --- optim_nn_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optim_nn_core.py b/optim_nn_core.py index 38f1eda..d7e2dc5 100644 --- a/optim_nn_core.py +++ b/optim_nn_core.py @@ -258,7 +258,7 @@ class RMSprop(Optimizer): #self.g += (dW * dW - self.g) * (1 - self.mu) # finally sqrt it to complete the running root-mean-square approximation - return -self.alpha * dW / np.sqrt(self.g + self.eps) + return -self.alpha * dW / (np.sqrt(self.g) + self.eps) class Adam(Optimizer): # paper: https://arxiv.org/abs/1412.6980 @@ -301,7 +301,7 @@ class Adam(Optimizer): self.vt[:] = self.b2 * self.vt + (1 - self.b2) * dW * dW return -self.alpha * (self.mt / (1 - self.b1_t)) \ - / np.sqrt((self.vt / (1 - self.b2_t)) + self.eps) + / (np.sqrt(self.vt / (1 - self.b2_t)) + self.eps) class Nadam(Optimizer): # paper: https://arxiv.org/abs/1412.6980