From 604ffb9fa1dc8b8770819f57660a2fd38755f2f6 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Wed, 7 Mar 2018 01:53:40 +0100 Subject: [PATCH] add variant of L1L2 regularization using averages --- onn/experimental.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/onn/experimental.py b/onn/experimental.py index 27f1a5a..8da2e4a 100644 --- a/onn/experimental.py +++ b/onn/experimental.py @@ -74,3 +74,25 @@ class PowerSignClip(Optimizer): inter /= np.log(1 + np.exp(total_norm / self.clip - 1)) + 1 return -self.lr * inter + + +class L1L2avg(Regularizer): + def __init__(self, l1=0.0, l2=0.0): + self.l1 = _f(l1) + self.l2 = _f(l2) + + def forward(self, X): + f = _0 + if self.l1: + f += np.average(self.l1 * np.abs(X)) + if self.l2: + f += np.average(self.l2 * np.square(X)) + return f + + def backward(self, X): + df = np.zeros_like(X) + if self.l1: + df += self.l1 / len(X) * np.sign(X) + if self.l2: + df += self.l2 / len(X) * 2 * X + return df