reorganize some code
This commit is contained in:
parent
87ffa014ca
commit
9b730b0516
138
optim_nn_core.py
138
optim_nn_core.py
|
@ -50,6 +50,60 @@ def init_glorot_uniform(size, ins, outs):
|
||||||
s = np.sqrt(6 / (ins + outs))
|
s = np.sqrt(6 / (ins + outs))
|
||||||
return np.random.uniform(-s, s, size=size)
|
return np.random.uniform(-s, s, size=size)
|
||||||
|
|
||||||
|
# Weight container {{{1
|
||||||
|
|
||||||
|
class Weights:
|
||||||
|
# we may or may not contain weights -- or any information, for that matter.
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
self.f = None # forward weights
|
||||||
|
self.g = None # backward weights (gradients)
|
||||||
|
self.shape = None
|
||||||
|
self.init = None
|
||||||
|
self.allocator = None
|
||||||
|
self.regularizer = None
|
||||||
|
|
||||||
|
self.configure(**kwargs)
|
||||||
|
|
||||||
|
def configure(self, **kwargs):
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
getattr(self, k) # ensures the key already exists
|
||||||
|
setattr(self, k, v)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def size(self):
|
||||||
|
assert self.shape is not None
|
||||||
|
return np.prod(self.shape)
|
||||||
|
|
||||||
|
def allocate(self, *args, **kwargs):
|
||||||
|
self.configure(**kwargs)
|
||||||
|
|
||||||
|
# intentionally not using isinstance
|
||||||
|
assert type(self.shape) == tuple, self.shape
|
||||||
|
|
||||||
|
f, g = self.allocator(self.size)
|
||||||
|
assert len(f) == self.size, "{} != {}".format(f.shape, self.size)
|
||||||
|
assert len(g) == self.size, "{} != {}".format(g.shape, self.size)
|
||||||
|
f[:] = self.init(self.size, *args)
|
||||||
|
g[:] = self.init(self.size, *args)
|
||||||
|
self.f = f.reshape(self.shape)
|
||||||
|
self.g = g.reshape(self.shape)
|
||||||
|
|
||||||
|
def forward(self):
|
||||||
|
if self.regularizer is None:
|
||||||
|
return 0.0
|
||||||
|
return self.regularizer.forward(self.f)
|
||||||
|
|
||||||
|
def backward(self):
|
||||||
|
if self.regularizer is None:
|
||||||
|
return 0.0
|
||||||
|
return self.regularizer.backward(self.f)
|
||||||
|
|
||||||
|
def update(self):
|
||||||
|
if self.regularizer is None:
|
||||||
|
return
|
||||||
|
self.g += self.regularizer.backward(self.f)
|
||||||
|
|
||||||
# Loss functions {{{1
|
# Loss functions {{{1
|
||||||
|
|
||||||
class Loss:
|
class Loss:
|
||||||
|
@ -296,60 +350,6 @@ class Nadam(Optimizer):
|
||||||
|
|
||||||
return -self.alpha * mt_bar / (np.sqrt(vtp) + self.eps)
|
return -self.alpha * mt_bar / (np.sqrt(vtp) + self.eps)
|
||||||
|
|
||||||
# Weight container {{{1
|
|
||||||
|
|
||||||
class Weights:
|
|
||||||
# we may or may not contain weights -- or any information, for that matter.
|
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
self.f = None # forward weights
|
|
||||||
self.g = None # backward weights (gradients)
|
|
||||||
self.shape = None
|
|
||||||
self.init = None
|
|
||||||
self.allocator = None
|
|
||||||
self.regularizer = None
|
|
||||||
|
|
||||||
self.configure(**kwargs)
|
|
||||||
|
|
||||||
def configure(self, **kwargs):
|
|
||||||
for k, v in kwargs.items():
|
|
||||||
getattr(self, k) # ensures the key already exists
|
|
||||||
setattr(self, k, v)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def size(self):
|
|
||||||
assert self.shape is not None
|
|
||||||
return np.prod(self.shape)
|
|
||||||
|
|
||||||
def allocate(self, *args, **kwargs):
|
|
||||||
self.configure(**kwargs)
|
|
||||||
|
|
||||||
# intentionally not using isinstance
|
|
||||||
assert type(self.shape) == tuple, self.shape
|
|
||||||
|
|
||||||
f, g = self.allocator(self.size)
|
|
||||||
assert len(f) == self.size, "{} != {}".format(f.shape, self.size)
|
|
||||||
assert len(g) == self.size, "{} != {}".format(g.shape, self.size)
|
|
||||||
f[:] = self.init(self.size, *args)
|
|
||||||
g[:] = self.init(self.size, *args)
|
|
||||||
self.f = f.reshape(self.shape)
|
|
||||||
self.g = g.reshape(self.shape)
|
|
||||||
|
|
||||||
def forward(self):
|
|
||||||
if self.regularizer is None:
|
|
||||||
return 0.0
|
|
||||||
return self.regularizer.forward(self.f)
|
|
||||||
|
|
||||||
def backward(self):
|
|
||||||
if self.regularizer is None:
|
|
||||||
return 0.0
|
|
||||||
return self.regularizer.backward(self.f)
|
|
||||||
|
|
||||||
def update(self):
|
|
||||||
if self.regularizer is None:
|
|
||||||
return
|
|
||||||
self.g += self.regularizer.backward(self.f)
|
|
||||||
|
|
||||||
# Abstract Layers {{{1
|
# Abstract Layers {{{1
|
||||||
|
|
||||||
class Layer:
|
class Layer:
|
||||||
|
@ -532,6 +532,22 @@ class Sum(Layer):
|
||||||
#assert len(edges) == 1, "unimplemented"
|
#assert len(edges) == 1, "unimplemented"
|
||||||
return edges[0] # TODO: does this always work?
|
return edges[0] # TODO: does this always work?
|
||||||
|
|
||||||
|
class ActivityRegularizer(Layer):
|
||||||
|
def __init__(self, reg):
|
||||||
|
super().__init__()
|
||||||
|
assert isinstance(reg, Regularizer), reg
|
||||||
|
self.reg = reg
|
||||||
|
|
||||||
|
def forward(self, X):
|
||||||
|
self.X = X
|
||||||
|
self.loss = np.sum(self.reg.forward(X))
|
||||||
|
return X
|
||||||
|
|
||||||
|
def backward(self, dY):
|
||||||
|
return dY + self.reg.backward(self.X)
|
||||||
|
|
||||||
|
# Activation Layers {{{2
|
||||||
|
|
||||||
class Sigmoid(Layer): # aka Logistic
|
class Sigmoid(Layer): # aka Logistic
|
||||||
def forward(self, X):
|
def forward(self, X):
|
||||||
self.sig = sigmoid(X)
|
self.sig = sigmoid(X)
|
||||||
|
@ -610,20 +626,6 @@ class LogSoftmax(Softmax):
|
||||||
def backward(self, dY):
|
def backward(self, dY):
|
||||||
return dY - np.sum(dY, axis=-1, keepdims=True) * self.sm
|
return dY - np.sum(dY, axis=-1, keepdims=True) * self.sm
|
||||||
|
|
||||||
class ActivityRegularizer(Layer):
|
|
||||||
def __init__(self, reg):
|
|
||||||
super().__init__()
|
|
||||||
assert isinstance(reg, Regularizer), reg
|
|
||||||
self.reg = reg
|
|
||||||
|
|
||||||
def forward(self, X):
|
|
||||||
self.X = X
|
|
||||||
self.loss = np.sum(self.reg.forward(X))
|
|
||||||
return X
|
|
||||||
|
|
||||||
def backward(self, dY):
|
|
||||||
return dY + self.reg.backward(self.X)
|
|
||||||
|
|
||||||
# Parametric Layers {{{1
|
# Parametric Layers {{{1
|
||||||
|
|
||||||
class Dense(Layer):
|
class Dense(Layer):
|
||||||
|
|
Loading…
Reference in New Issue
Block a user