notwa/optim
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

move epsilon out of square-root (big improvement)

Browse source
This commit is contained in:
Connor Olding 2017-06-21 00:13:53 +00:00
parent 9b85b49ee5
commit 9c0cbba3ff
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
optim_nn_core.py
View file

@ -258,7 +258,7 @@ class RMSprop(Optimizer):
#self.g += (dW * dW - self.g) * (1 - self.mu) #self.g += (dW * dW - self.g) * (1 - self.mu)
# finally sqrt it to complete the running root-mean-square approximation # finally sqrt it to complete the running root-mean-square approximation
return -self.alpha * dW / np.sqrt(self.g + self.eps) return -self.alpha * dW / (np.sqrt(self.g) + self.eps)
class Adam(Optimizer): class Adam(Optimizer):
# paper: https://arxiv.org/abs/1412.6980 # paper: https://arxiv.org/abs/1412.6980
@ -301,7 +301,7 @@ class Adam(Optimizer):
self.vt[:] = self.b2 * self.vt + (1 - self.b2) * dW * dW self.vt[:] = self.b2 * self.vt + (1 - self.b2) * dW * dW
return -self.alpha * (self.mt / (1 - self.b1_t)) \ return -self.alpha * (self.mt / (1 - self.b1_t)) \
/ np.sqrt((self.vt / (1 - self.b2_t)) + self.eps) / (np.sqrt(self.vt / (1 - self.b2_t)) + self.eps)
class Nadam(Optimizer): class Nadam(Optimizer):
# paper: https://arxiv.org/abs/1412.6980 # paper: https://arxiv.org/abs/1412.6980