From dcbaef303289ebfb926eb7ff0520012256e5405a Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sat, 16 Sep 2017 17:30:52 +0000 Subject: [PATCH] use MomentumClip in warmup for stability --- onn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onn.py b/onn.py index 4fc73fa..16d345a 100755 --- a/onn.py +++ b/onn.py @@ -1273,7 +1273,7 @@ def run(program, args=None): # use plain SGD in warmup to prevent (or possibly cause?) numeric issues temp_optim = learner.optim temp_loss = model.loss - learner.optim = Optimizer(lr=0.001) + learner.optim = MomentumClip(lr=0.01, mu=0) ritual.loss = Absolute() # less likely to blow up; more general # NOTE: experiment: trying const batches and batch_size