notwa/optim
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

add debug mode to MomentumClip to print norms

Browse source
This commit is contained in:
Connor Olding 2017-07-23 03:54:37 +00:00
parent b20a34c2de
commit ee83ffa88e
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
onn.py
View file

@ -155,10 +155,11 @@ class FTML(Optimizer):
return -self.zt / self.dt - W return -self.zt / self.dt - W
class MomentumClip(Optimizer): class MomentumClip(Optimizer):
def __init__(self, lr=0.01, mu=0.9, nesterov=False, clip=1.0): def __init__(self, lr=0.01, mu=0.9, nesterov=False, clip=1.0, debug=False):
self.mu = _f(mu) self.mu = _f(mu)
self.clip = _f(clip) self.clip = _f(clip)
self.nesterov = bool(nesterov) self.nesterov = bool(nesterov)
self.debug = bool(debug)
super().__init__(lr) super().__init__(lr)
@ -172,7 +173,8 @@ class MomentumClip(Optimizer):
total_norm = np.linalg.norm(dW) total_norm = np.linalg.norm(dW)
clip_scale = self.clip / (total_norm + 1e-6) clip_scale = self.clip / (total_norm + 1e-6)
if clip_scale < 1: if clip_scale < 1:
#print("clipping gradients; norm: {:10.5f}".format(total_norm)) if self.debug:
lament("clipping gradients; norm: {:10.5f}".format(total_norm))
dW *= clip_scale dW *= clip_scale
self.accum[:] = self.accum * self.mu + dW self.accum[:] = self.accum * self.mu + dW