smbot/presets.lua

-- these are all still very experimental,
-- but they should at least give you a head-start.
local presets = {}

local function make_preset(cfg)
    local name = cfg.name
    if name ~= nil then
        cfg.log_fn = cfg.log_fn or 'logs-'..name..'.csv'
        cfg.params_fn = cfg.params_fn or 'params-'..name..'.txt'
    end
    presets[name] = cfg
    return cfg
end

make_preset{
    name = 'big-scroll',

    starting_world = 1,
    starting_level = 1,
    start_big = true,
    starting_lives = 0,
    min_time = 300,
    max_time = 300,
    timer_loser = 1.0,

    decrement_reward = false,
    score_multiplier = 0,
}

make_preset{
    name = 'big-scroll-reduced',
    parent = 'big-scroll',

    init_zeros = true,

    reduce_tiles = 5,
    bias_out = false,

    deterministic = false,
}

make_preset{
    name = 'big-scroll-hidden',
    parent = 'big-scroll',

    init_zeros = true,

    hidden = true,
    hidden_size = 64, --128,
    layernorm = true,
    reduce_tiles = false,
    bias_out = false,

    deterministic = false,
}

make_preset{
    name = 'snes',
    parent = 'big-scroll-hidden',

    es = 'snes',

    epoch_trials = 100,

    deviation = 0.5,
    negate_trials = false,
    min_refresh = 0.25,

    param_rate = 0.50,
    sigma_rate = 0.100, --0.125,
    param_decay = 0.006,
    sigma_decay = 0.008,
}

make_preset{
    name = 'snes2',
    parent = 'big-scroll-reduced',

    es = 'snes',
    deterministic = true,
    deviation = 0.01,
    negate_trials = false,
    epoch_trials = 60,
    min_refresh = 2/3,
    param_rate = 0.368,
    param_decay = 0.0138,
    sigma_rate = 0.100,
    sigma_decay = 0.0051,
}

make_preset{
    name = 'snes3',
    parent = 'snes2',
    min_refresh = 1/3,
}

make_preset{
    name = 'xnes',
    parent = 'big-scroll-reduced',

    es = 'xnes',

    deviation = 0.5,
    negate_trials = false,

    epoch_trials = 50,

    param_rate = 1.0,
    sigma_rate = 0.01,
    covar_rate = 0.01,
}

make_preset{
    name = 'xnes2',
    parent = 'big-scroll-reduced',

    es = 'xnes',

    deviation = 0.1207,
    negate_trials = true,

    epoch_trials = 10,

    param_rate = 0.5,
    sigma_rate = 0.16,
    covar_rate = 0.04,
    param_decay = 0.004,
    sigma_decay = 0.00128,
}

make_preset{
    name = 'ars',
    parent = 'big-scroll-reduced',

    es = 'ars',
    epoch_top_trials = 20,
    deterministic = true,
    deviation = 0.01,
    epoch_trials = 20,
    param_rate = 0.368,
    param_decay = 0.0138,
    momentum = 0.5,
}

make_preset{
    name = 'ars-vanilla',
    parent = 'ars',
}

make_preset{
    name = 'ars-lips',
    parent = 'ars',

    ars_lips = true,
--  momentum = 0.5, -- this is default.
    param_rate = 1.0,
}

make_preset{
    name = 'ars-skip',
    parent = 'ars',

    frameskip = 1,
    prob_frameskip = 0.25,
}

make_preset{
    name = 'ars-big',
    parent = 'ars',

    epoch_top_trials = 75,
    epoch_trials = 100,
    momentum = 0.5,
    param_rate = 1.0,

    --graycode = true,
}

make_preset{
    name = 'ars-huge',
    parent = 'big-scroll-hidden',

    deterministic = true,
    deviation = 0.01,
    epoch_top_trials = 75,
    epoch_trials = 100,
    es = 'ars',
    momentum = 0.5,
    param_decay = 0.0138,
    param_rate = 0.5,
}

make_preset{
    name = 'ars-stupid',
    parent = 'big-scroll-reduced',

    es = 'ars',
    epoch_top_trials = 4,
    deterministic = false,
    deviation = 0.2,
    epoch_trials = 4,
    param_rate = 0.1,
    param_decay = 0.003,
    momentum = 0.99,
}

-- new stuff for 2019:

make_preset{
    name = 'ars-skip-more',
    parent = 'ars',

    -- old:
    --frameskip = 2,
    --prob_frameskip = 0.5,
    --max_frameskip = 60,
    -- new:
    frameskip = 3,
    prob_frameskip = 0.5,
    max_frameskip = 5,
}

make_preset{
    name = 'ars-skip-more-3',
    parent = 'ars-skip-more',

    attempts = 3, -- per trial. score = mean(scores) - stdev(scores)
}

make_preset{
    name = 'snes-skip-more-3',
    parent = 'snes3',

    frameskip = 3,
    prob_frameskip = 0.5,
    max_frameskip = 5,
    attempts = 3,
}

make_preset{
    name = 'guided',
    parent = 'big-scroll-reduced',

    es = 'guided',
    epoch_top_trials = 20,
    deterministic = true,
    deviation = 0.1,
    epoch_trials = 20,
    param_rate = 0.00368,
    param_decay = 0.0,
}

make_preset{
    name = 'guided2',
    parent = 'guided',

    past_grads = 2,
    -- after epoch 50, trying this:
    --param_rate = 0.05,
    -- after epoch 50+20, stepping back to this:
    param_rate = 0.01,
}

make_preset{
    name = 'guided10',
    parent = 'guided',

    past_grads = 10,
}

make_preset{
    name = 'guided69', -- the nice one
    parent = 'guided',

    deviation = 0.05,
    epoch_top_trials = 10,
    epoch_trials = 20,
    param_rate = 0.006,

    past_grads = 4,
    alpha = 0.25,
}

-- TODO: yet another preset. try building up from 1 trial ARS to something good.
make_preset{
    name = 'redux',

    min_time = 300,
    max_time = 300,
    timer_loser = 1/1,

    score_multiplier = 1,

    init_zeros = true,

    deterministic = true,

    es = 'guided',
    past_grads = 2, -- for Guided.
    alpha = 0.25, -- for Guided.

    ars_lips = false, -- for ARS.
    beta = 1.0, -- fix the default.

    epoch_top_trials = 4, -- for ARS, Guided.
    epoch_trials = 5,
    attempts = 1, -- TODO: document.

    deviation = 1.0, -- 0.1
    base_rate = 1.0,
    param_decay = 0.01,

    graycode = false, -- for ARS.
    min_refresh = 0.1, -- for SNES.
    sigma_decay = 0.0, -- for SNES, xNES.
    momentum = 0.0, -- for ARS.
}

make_preset{
    name = 'redux_big',
    parent = 'redux',

    time_inputs = true, -- insert binary inputs of a frame counter.
    hidden = true, -- use a hidden layer with ReLU/GELU activation.
    hidden_size = 64,
    layernorm = true, -- use a LayerNorm layer after said activation.
    reduce_tiles = false,
    bias_out = false,

    -- gets stuck pretty quick, so tweak some stuff...
    epoch_top_trials = 8,
    epoch_trials = 10,
    deviation = 1.0,
    base_rate = 0.15,
    param_decay = 0.05,
    past_grads = 4,
    alpha = 0.25,
    -- well it doesn't get stuck anymore, but regular redux works much better.
}

make_preset{
    name = 'guided-skip-more-3',
    parent = 'guided',

    --param_rate = 0.00368, -- should probably be this instead...
    param_rate = 0.01,

    frameskip = 3,
    prob_frameskip = 0.5,
    max_frameskip = 5,
    attempts = 3,
}

make_preset{
    name = 'guided-skip-more-3-again',
    parent = 'guided-skip-more-3',

    param_rate = 0.08, --0.0316,
    deviation = 0.5,
    alpha = 0.1, --0.5,
}

make_preset{
    name = 'crazy',
    parent = 'big-scroll-reduced',

    es = 'guided',
    epoch_top_trials = 15,
    deterministic = false,
    deviation = 1.0,
    epoch_trials = 15,
    param_rate = 1.0,
    param_decay = 0.0,
    alpha = 0.0316,
    --attempts = 3,
}

make_preset{
    name = 'ars-lips2',
    parent = 'ars',

    ars_lips = true,
    --epoch_trials = 10,
    param_rate = 0.147,
}

make_preset{
    name = 'ars-lips3',
    parent = 'ars',

    ars_lips = true,
    param_rate = 0.5,
    deviation = 0.02, -- added after like 272 epochs
    param_decay = 0.0276, -- added after like 62 epochs
}

make_preset{
    name = 'hard-embed',
    parent = 'big-scroll-hidden',

    embed = false,
    reduce_tiles = 5,
    hidden_size = 54,

    epoch_top_trials = 20,
    deterministic = true,
    deviation = 0.01,
    epoch_trials = 20,
    param_rate = 0.368,
    param_decay = 0.0138,
    momentum = 0.5,
    beta = 1.0,
}

-- end of new stuff

make_preset{
    name = 'play',

    playable_mode = true,
}

return presets