flesh out patch.lua and use the new features

.gitignore

@@ -7,3 +7,4 @@
 __pycache__/*
 cm mm save.lua
 cm oot save.lua
+z64yaz0

patch/.gitignore

@@ -2,3 +2,4 @@ patchme
 lips
 entrances.asm
 crc32.asm
+labels.lua

patch/code.asm

@@ -24,35 +24,14 @@
     li      a0, @start          ; 1 (just make sure @start can be a LUI!)
 
 .org 0xC4808 ; 0x8016A2C8
-/*
-    ; if we've already loaded once, don't load again
-    lbu     t0, @start          ; 2
-    bnez    t0, +               ; 1
-    nop                         ; 1
-    push    4, a0, a1, a2, ra   ; 5
-    li      a0, @start          ; 1
-    li      a1, @vstart         ; 2
-    li      a2, @size           ; 2
-    jal     @DMARomToRam        ; 1
-    nop                         ; 1
-    pop     4, a0, a1, a2, ra   ; 5
-+:
-    j       @dma_hook           ; 1
-    nop                         ; 1
-; total overwriten instructions: 23
-; the original function is in setup.asm,
-; and is moved into our extra file.
-; we have (0x944 / 4 - 23) = 570 words of space here, should we need it.
-    .word    0xDEADBEEF
-*/
-    j       @dma_hook           ; 1
+    j       dma_hook            ; 1
     nop                         ; 1
 
 .org 0x9F9A4 ; JR of starting_exit's function
-    j       @load_hook ; tail call
+    j       load_hook ; tail call
 
 .org 0x80710
-    j       @tunic_color_hook
+    j       tunic_color_hook
     lhu     t1, 0x1DB0(t1); original code
 
 .org @starting_exit

patch/common.asm

@@ -22,11 +22,3 @@
 ;[link_object_ptr]: 0x244
 
 [scene_record_size]: 0x14
-
-; TODO: use arithmetic to do this (add that to lips)
-; TODO: better yet, add proper label exports to lips
-;       so you can write the routines anywhere
-[dma_hook]:         0x807DA000
-[load_hook]:        0x807DA010
-[setup_hook]:       0x807DA020
-[tunic_color_hook]: 0x807DA030

patch/extra.asm

@@ -349,29 +349,3 @@ setup_hook:
     jpop    4, a0, ra
 
 .word 0xDEADBEEF
-
-.org @dma_hook
-    j       dma_hook
-    nop
-.word dma_hook
-    nop
-
-.org @setup_hook
-    j       setup_hook
-    nop
-.word setup_hook
-    nop
-
-.org @load_hook
-    j       load_hook
-    nop
-.word load_hook
-    nop
-
-.org @tunic_color_hook
-    j       tunic_color_hook
-    nop
-.word tunic_color_hook
-    nop
-
-.align 0x10

patch/extra.lua

@@ -0,0 +1,62 @@
+local function strpad(num, count, pad)
+    num = tostring(num)
+    return (pad:rep(count)..num):sub(#num)
+end
+
+local function add_zeros(num, count)
+    return strpad(num, count - 1, '0')
+end
+
+local function mixed_sorter(a, b)
+    a = type(a) == 'number' and add_zeros(a, 16) or tostring(a)
+    b = type(b) == 'number' and add_zeros(b, 16) or tostring(b)
+    return a < b
+end
+
+-- loosely based on http://lua-users.org/wiki/SortedIteration
+-- the original didn't make use of closures for who knows why
+local function order_keys(t)
+    local oi = {}
+    for key in pairs(t) do
+        table.insert(oi, key)
+    end
+    table.sort(oi, mixed_sorter)
+    return oi
+end
+
+local function opairs(t, cache)
+    local oi = cache and cache[t] or order_keys(t)
+    if cache then
+        cache[t] = oi
+    end
+    local i = 0
+    return function()
+        i = i + 1
+        local key = oi[i]
+        if key then return key, t[key] end
+    end
+end
+
+local function traverse(path)
+    if not path then return end
+    local parent = _G
+    local key
+    for w in path:gfind("[%w_]+") do
+        if key then
+            parent = rawget(parent, key)
+            if type(parent) ~= 'table' then return end
+        end
+        key = w
+    end
+    if not key then return end
+    return {parent=parent, key=key}
+end
+
+return {
+    strpad = strpad,
+    add_zeros = add_zeros,
+    mixed_sorter = mixed_sorter,
+    order_keys = order_keys,
+    opairs = opairs,
+    traverse = traverse,
+}

patch/mm-bq

@@ -37,6 +37,8 @@ comp() {
     rm patchme/"$1"
 }
 
+[ ! -s ../z64yaz0 ] && cc -O3 ../z64yaz0.c -o ../z64yaz0
+
 if [ $fast -eq 0 ] || [ ! -d patchme ]; then
     [ -d patchme ] && rm -r patchme
     (cd ..; ./z64dump.py -c "$rom")
@@ -50,11 +52,12 @@ mkdir -p lips
 cp "$lips"/* lips
 cp "$inject/"{crc32,entrances}.asm .
 
-touch "extra"
+dd if=/dev/zero of=extra bs=370688 count=1 2>/dev/null
 
-luajit patch.lua code.asm patchme/"$code" 0
-luajit patch.lua extra.asm extra 0x80780000
+luajit patch.lua -e labels.lua -o 0x80780000 extra.asm extra
+luajit patch.lua -i labels.lua code.asm patchme/"$code"
 
+# ensure the file is the proper size (Lua seems to expand it?)
 dd if=extra of=patchme/"$extra" bs=370688 count=1 2>/dev/null
 rm extra
 

patch/oot-widescreen

@@ -10,6 +10,6 @@ lips=../Lua/lib/lips
 mkdir -p lips
 cp "$lips"/* lips
 
-luajit patch.lua widescreen-inline.asm oot-widescreen.z64 0 0x035D0000 0x80700000
+luajit patch.lua -o 0 --extra-rom 0x035D0000 --extra-ram 0x80700000 widescreen-inline.asm oot-widescreen.z64
 
 (cd ..; ./z64dump.py -f patch/oot-widescreen.z64)

patch/patch.lua

@@ -1,22 +1,32 @@
 package.path = package.path..";./?/init.lua"
 
 local assemble = require "lips"
+local cereal = require "serialize"
+local argparse = require "argparse"
 
-local function inject(patch, target, offset, erom, eram)
-    offset = offset or 0
+local function inject(args)
+    args.offset = args.offset or 0
 
-    local f = io.open(target, 'r+b')
+    local f = io.open(args.output, 'r+b')
     if not f then
-        print("file not found:", target)
+        print("file not found:", args.output)
         return
     end
 
+    local state = {}
+    for _, import in ipairs(args.import) do
+        local new_state = cereal.deserialize(import)
+        for k, v in pairs(new_state) do
+            state[k] = v
+        end
+    end
+
     local function write(pos, line)
         assert(#line == 2, "that ain't const")
-        if erom and eram and pos >= eram then
-            pos = pos - eram + erom
-        elseif pos >= offset then
-            pos = pos - offset
+        if args.extra_rom and args.extra_ram and pos >= args.extra_ram then
+            pos = pos - args.extra_ram + args.extra_rom
+        elseif pos >= args.offset then
+            pos = pos - args.offset
         end
         if pos >= 1024*1024*1024 then
             print("you probably don't want to do this:")
@@ -26,30 +36,41 @@ local function inject(patch, target, offset, erom, eram)
         f:seek('set', pos)
 
         -- TODO: write hex dump format of written bytes
-        print(("%08X    %s"):format(pos, line))
+        --print(("%08X    %s"):format(pos, line))
 
         f:write(string.char(tonumber(line, 16)))
     end
 
-    -- offset assembly labels so they work properly, and assemble!
-    assemble(patch, write, {unsafe=true, offset=offset})
+    assemble(args.input, write, {unsafe=true, offset=args.offset, labels=state})
+
+    if args.export then
+        cereal.serialize(args.export, state)
+    end
 
     f:close()
 end
 
 local function parsenum(s)
-    if s:sub(2) == '0x' then
-        s = tonumber(s, 16)
+    if s:sub(1, 2) == '0x' then
+        return tonumber(s, 16)
+    elseif s:sub(1, 1) == '0' then
+        return tonumber(s, 8)
     else
-        s = tonumber(s)
+        return tonumber(s)
     end
-    return s
 end
 
-local offset = arg[3]
-local extra_rom = arg[4]
-local extra_ram = arg[5]
-if offset then offset = parsenum(offset) end
-if extra_rom then extra_rom = parsenum(extra_rom) end
-if extra_ram then extra_ram = parsenum(extra_ram) end
-inject(arg[1], arg[2], offset, extra_rom, extra_ram)
+local ap = argparse("patch", "patch a binary file with assembly")
+
+ap:argument("input", "input assembly file")
+ap:argument("output", "output binary file")
+ap:option("-o --offset", "offset to pass to lips", "0"):convert(parsenum)
+ap:option("-i --import", "import state file(s) containing labels"):count("*")
+ap:option("-e --export", "export state file containing labels")
+--ap:option("-s --state", "--import and --export to this file")
+ap:option("--extra-rom", "dumb stuff"):convert(parsenum)
+ap:option("--extra-ram", "dumb stuff"):convert(parsenum)
+
+local inject_args = ap:parse()
+
+inject(inject_args)

patch/serialize.lua

@@ -0,0 +1,76 @@
+-- it's simple, dumb, unsafe, incomplete, and it gets the damn job done
+
+local type = type
+local extra = require "extra"
+local opairs = extra.opairs
+local tostring = tostring
+local open = io.open
+local strfmt = string.format
+local strrep = string.rep
+
+local function kill_bom(s)
+    if #s >= 3 and s:byte(1)==0xEF and s:byte(2)==0xBB and s:byte(3)==0xBF then
+        return s:sub(4)
+    end
+    return s
+end
+
+local function sanitize(v)
+    local force = type(v) == 'string' and v:sub(1, 1):match('%d')
+    force = force and true or false
+    return type(v) == 'string' and strfmt('%q', v) or tostring(v), force
+end
+
+local function _serialize(value, writer, level)
+    level = level or 1
+    if type(value) == 'table' then
+        local indent = strrep('\t', level)
+        writer('{\n')
+        for key,value in opairs(value) do
+            local sane, force = sanitize(key)
+            local keyval = (sane == '"'..key..'"' and not force) and key or '['..sane..']'
+            writer(indent..keyval..' = ')
+            _serialize(value, writer, level + 1)
+            writer(',\n')
+        end
+        writer(strrep('\t', level - 1)..'}')
+    else
+        local sane, force = sanitize(value)
+        writer(sane)
+    end
+end
+
+local function _deserialize(script)
+    local f = loadstring(kill_bom(script))
+    if f ~= nil then
+        return f()
+    else
+        print('WARNING: no function to deserialize with')
+        return nil
+    end
+end
+
+local function serialize(path, value)
+    local file = open(path, 'w')
+    if not file then return end
+    file:write("return ")
+    _serialize(value, function(...)
+        file:write(...)
+    end)
+    file:write("\n")
+    file:close()
+end
+
+local function deserialize(path)
+    local file = open(path, 'r')
+    if not file then return end
+    local script = file:read('*a')
+    local value = _deserialize(script)
+    file:close()
+    return value
+end
+
+return {
+    serialize = serialize,
+    deserialize = deserialize,
+}