notwa/optim
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

.

Browse source
This commit is contained in:
Connor Olding 2017-02-15 10:43:57 -08:00
parent 9cba495ce4
commit d232d81f5a
2 changed files with 42 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
optim_nn.py
View file

@ -249,29 +249,35 @@ def normalize_data(data, mean=None, std=None):
mean = np.mean(data, axis=0) mean = np.mean(data, axis=0)
std = np.std(data, axis=0) std = np.std(data, axis=0)
# TODO: construct function call string for copy-paste convenience # TODO: construct function call string for copy-paste convenience
print('mean:', mean) lament('mean:', mean)
print('std: ', std) lament('std: ', std)
import sys
sys.exit(1) sys.exit(1)
data -= mean data -= _f(mean)
data /= std data /= _f(std)
def toy_data(train_samples, valid_samples, problem=2): def toy_data(train_samples, valid_samples, problem=2):
total_samples = train_samples + valid_samples total_samples = train_samples + valid_samples
if problem == 1: if problem == 0:
from ml.cie_mlp_data import rgbcompare, input_samples, output_samples, \
inputs, outputs, valid_inputs, valid_outputs, \
x_scale, y_scale
inputs, outputs = _f(inputs), _f(outputs)
valid_inputs, valid_outputs = _f(valid_inputs), _f(valid_outputs)
normalize_data(inputs, 127.5, 73.9)
normalize_data(outputs, 44.8, 21.7)
normalize_data(valid_inputs, 127.5, 73.9)
normalize_data(valid_outputs, 44.8, 21.7)
elif problem == 1:
from sklearn.datasets import make_friedman1 from sklearn.datasets import make_friedman1
inputs, outputs = make_friedman1(total_samples) inputs, outputs = make_friedman1(total_samples)
inputs, outputs = _f(inputs), _f(outputs) inputs, outputs = _f(inputs), _f(outputs)
outputs = np.expand_dims(outputs, -1) outputs = np.expand_dims(outputs, -1)
normalize_data(inputs, normalize_data(inputs, 0.5, 1/np.sqrt(12))
0.5, normalize_data(outputs, 14.4, 4.9)
1/np.sqrt(12))
normalize_data(outputs,
14.4,
4.9)
elif problem == 2: elif problem == 2:
from sklearn.datasets import make_friedman2 from sklearn.datasets import make_friedman2
@ -283,9 +289,7 @@ def toy_data(train_samples, valid_samples, problem=2):
[5.00e+01, 9.45e+02, 5.01e-01, 5.98e+00], [5.00e+01, 9.45e+02, 5.01e-01, 5.98e+00],
[2.89e+01, 4.72e+02, 2.89e-01, 2.87e+00]) [2.89e+01, 4.72e+02, 2.89e-01, 2.87e+00])
normalize_data(outputs, normalize_data(outputs, [482], [380])
[482],
[380])
elif problem == 3: elif problem == 3:
from sklearn.datasets import make_friedman3 from sklearn.datasets import make_friedman3
@ -297,20 +301,19 @@ def toy_data(train_samples, valid_samples, problem=2):
[4.98e+01, 9.45e+02, 4.99e-01, 6.02e+00], [4.98e+01, 9.45e+02, 4.99e-01, 6.02e+00],
[2.88e+01, 4.73e+02, 2.90e-01, 2.87e+00]) [2.88e+01, 4.73e+02, 2.90e-01, 2.87e+00])
normalize_data(outputs, normalize_data(outputs, [1.32327931], [0.31776295])
[1.32327931],
[0.31776295])
else: else:
raise Exception("unknown toy data set", problem) raise Exception("unknown toy data set", problem)
# split off a validation set if problem != 0:
indices = np.arange(inputs.shape[0]) # split off a validation set
np.random.shuffle(indices) indices = np.arange(inputs.shape[0])
valid_inputs = inputs[indices][-valid_samples:] np.random.shuffle(indices)
valid_outputs = outputs[indices][-valid_samples:] valid_inputs = inputs[indices][-valid_samples:]
inputs = inputs[indices][:-valid_samples] valid_outputs = outputs[indices][-valid_samples:]
outputs = outputs[indices][:-valid_samples] inputs = inputs[indices][:-valid_samples]
outputs = outputs[indices][:-valid_samples]
return (inputs, outputs), (valid_inputs, valid_outputs) return (inputs, outputs), (valid_inputs, valid_outputs)
@ -451,15 +454,13 @@ def run(program, args=[]):
optim = 'adam', optim = 'adam',
optim_decay1 = 2, # given in epochs (optional) optim_decay1 = 2, # given in epochs (optional)
optim_decay2 = 100, # given in epochs (optional) optim_decay2 = 100, # given in epochs (optional)
nesterov = False, # only used with SGD or Adam
momentum = 0.50, # only used with SGD momentum = 0.50, # only used with SGD
nesterov = False, # only used with SGD or Adam
batch_size = 64, batch_size = 64,
# learning parameters # learning parameters
learner = 'sgdr', learner = 'sgdr',
learn = 1e-2, learn = 1e-2,
learn_halve_every = 16, # unused with SGDR
learn_restart_advance = 16, # unused with SGDR
epochs = 24, epochs = 24,
restarts = 2, restarts = 2,
restart_decay = 0.25, # only used with SGDR restart_decay = 0.25, # only used with SGDR
@ -473,10 +474,15 @@ def run(program, args=[]):
restart_optim = False, # restarts also reset internal state of optimizer restart_optim = False, # restarts also reset internal state of optimizer
problem = 3, problem = 3,
# best results for ~10,000 parameters compare = (
# (keep these paired; update both at the same time!) # best results for ~10,000 parameters
train_compare = 1.854613e-05, # training/validation pairs for each problem (starting from problem 0):
valid_compare = 1.623881e-05, #(5.08e-05, 6.78e-05),
(7.577717e-04, 1.255284e-03),
(3.032806e-06, 3.963775e-06),
(3.676451e-07, 4.495362e-07),
(1.854613e-05, 1.623881e-05)
),
unsafe = True, # aka gotta go fast mode unsafe = True, # aka gotta go fast mode
) )
@ -532,10 +538,10 @@ def run(program, args=[]):
train_err = print_error("train", train_err = print_error("train",
inputs, outputs, inputs, outputs,
config.train_compare) config.compare[config.problem][0])
valid_err = print_error("valid", valid_err = print_error("valid",
valid_inputs, valid_outputs, valid_inputs, valid_outputs,
config.valid_compare) config.compare[config.problem][1])
train_losses.append(train_err) train_losses.append(train_err)
valid_losses.append(valid_err) valid_losses.append(valid_err)

1
optim_nn_core.py
View file

@ -105,6 +105,7 @@ class Momentum(Optimizer):
class RMSprop(Optimizer): class RMSprop(Optimizer):
# RMSprop generalizes* Adagrad, etc. # RMSprop generalizes* Adagrad, etc.
# TODO: verify this is correct:
# * RMSprop == Adagrad when # * RMSprop == Adagrad when
# RMSprop.mu == 1 # RMSprop.mu == 1