add ROM dumping utility

Yaz0.py

@@ -0,0 +1,56 @@
+# decoder ripped from: http://www.amnoid.de/gc/yaz0.txt
+# encoder ripped from:
+# https://bitbucket.org/ottehr/z64-fm/src/9fdc704ca42ff15c8e01b1566d4692d986920c6a/yaz0.c
+
+def decode(comp):
+    src = 16 # skip header
+    dst = 0
+    valid = 0 # bit count
+    curr = 0 # code byte
+
+    assert(comp[:4] == b'Yaz0')
+    assert(comp[8:12] == b'\x00\x00\x00\x00')
+    assert(comp[12:16] == b'\x00\x00\x00\x00')
+
+    # we could use struct but eh we only need it once
+    size = comp[4]*0x1000000 + comp[5]*0x10000 + comp[6]*0x100 + comp[7]
+    uncomp = bytearray(size)
+
+    while dst < size:
+        if not valid:
+            curr = comp[src]
+            src += 1
+            valid = 8
+
+        if curr & 0x80:
+            uncomp[dst] = comp[src]
+            dst += 1
+            src += 1
+        else:
+            byte1 = comp[src]
+            byte2 = comp[src + 1]
+            src += 2
+
+            dist = ((byte1 & 0xF) << 8) | byte2
+            copy = dst - (dist + 1)
+            assert(copy >= 0)
+
+            n = byte1 >> 4
+            if n:
+                n += 2
+            else:
+                n = comp[src] + 0x12
+                src += 1
+
+            for i in range(n):
+                uncomp[dst] = uncomp[copy]
+                copy += 1
+                dst += 1
+
+        curr <<= 1
+        valid -= 1
+
+    return uncomp
+
+def encode(uncomp):
+    raise Exception('Yaz0_encode: unimplemented')

n64.py

@@ -0,0 +1,69 @@
+# Based on uCON64's N64 checksum algorithm by Andreas Sterbenz
+
+crc_seeds = {
+    6101: 0xF8CA4DDC,
+    6102: 0xF8CA4DDC,
+    6103: 0xA3886759,
+    6105: 0xDF26F436,
+    6106: 0x1FEA617A,
+}
+
+MAX32 = 0xFFFFFFFF
+
+def ROL(i, b):
+    return ((i << b) | (i >> (32 - b))) & MAX32
+
+def R4(b):
+    return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3]
+
+def crc(f, bootcode=6105):
+    seed = crc_seeds[bootcode]
+    t1 = t2 = t3 = t4 = t5 = t6 = seed
+
+    if bootcode == 6105:
+        f.seek(0x0710 + 0x40)
+        lookup = f.read(0x100)
+
+    f.seek(0x1000)
+    for i in range(0x1000, 0x101000, 4):
+        d = R4(f.read(4))
+
+        if ((t6 + d) & MAX32) < t6:
+            t4 += 1
+            t4 &= MAX32
+
+        t6 += d
+        t6 &= MAX32
+
+        t3 ^= d
+
+        b = d & 0x1F
+        r = (d << b) | (d >> (32 - b))
+        r &= MAX32
+
+        t5 += r
+        t5 &= MAX32
+
+        if t2 > d:
+            t2 ^= r
+        else:
+            t2 ^= t6 ^ d
+
+        if bootcode == 6105:
+            o = i & 0xFF
+            temp = R4(lookup[o:o + 4])
+        else:
+            temp = t5
+        t1 += temp ^ d
+        t1 &= MAX32
+
+    if bootcode == 6103:
+        crc1 = (t6 ^ t4) + t3
+        crc2 = (t5 ^ t2) + t1
+    elif bootcode == 6106:
+        crc1 = t6*t4 + t3
+        crc2 = t5*t2 + t1
+    else:
+        crc1 = t6 ^ t4 ^ t3
+        crc2 = t5 ^ t2 ^ t1
+    return crc1 & MAX32, crc2 & MAX32

z64dump

@@ -0,0 +1,220 @@
+#!/bin/python
+# shoutouts to spinout182
+
+import os, os.path
+import sys
+import struct
+import hashlib
+
+import n64
+import Yaz0
+
+lament = lambda *args, **kwargs: print(*args, file=sys.stderr, **kwargs)
+
+R1 = lambda data: struct.unpack('>B', data)[0]
+R2 = lambda data: struct.unpack('>H', data)[0]
+R4 = lambda data: struct.unpack('>I', data)[0]
+W1 = lambda data: struct.pack('>B', data)
+W2 = lambda data: struct.pack('>H', data)
+W4 = lambda data: struct.pack('>I', data)
+
+# assume first entry is makerom (0x1060), and second entry begins from makerom
+fs_sig = b"\x00\x00\x00\x00\x00\x00\x10\x60\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x60"
+
+def dump_as(b, fn):
+    with open(fn, 'w+b') as f:
+        f.write(b)
+
+def z_dump_file(f, prefix=None):
+    vs = R4(f.read(4)) # virtual start
+    ve = R4(f.read(4)) # virtual end
+    ps = R4(f.read(4)) # physical start
+    pe = R4(f.read(4)) # physical end
+    here = f.tell()
+
+    if vs == ve == ps == pe == 0:
+        return False
+
+    fn = 'V{:08X}-{:08X},P{:08X}-{:08X}'.format(vs, ve, ps, pe)
+    if prefix is not None:
+        fn = str(prefix) + fn
+
+    size = ve - vs
+
+    if ps == 0xFFFFFFFF or pe == 0xFFFFFFFF:
+        #lament('file does not exist')
+        dump_as(b'', fn)
+    elif pe == 0:
+        #lament('file is uncompressed')
+        pe = ps + size
+        f.seek(ps)
+        data = f.read(pe - ps)
+        dump_as(data, fn)
+    else:
+        #lament('file is compressed')
+        f.seek(ps)
+        compressed = f.read(pe - ps)
+        if compressed[:4] == b'Yaz0':
+            data = Yaz0.decode(compressed)
+            dump_as(data, fn)
+        else:
+            lament('unknown compression; skipping:', fn)
+            lament(compressed[:4])
+
+    f.seek(here)
+    return True, fn, vs, ve, ps, pe
+
+def z_find_fs(f):
+    while True:
+        # assume row alignment
+        data = f.read(16)
+        if len(data) == 0: # EOF
+            break
+        if data == fs_sig[:16]:
+            rest = fs_sig[16:]
+            if f.read(len(rest)) == rest:
+                return f.tell() - len(rest) - 16
+            else:
+                f.seek(len(rest), 1)
+
+def z_dump(f):
+    f.seek(0x1060) # skip header when finding fs
+    addr = z_find_fs(f)
+    if addr == None:
+        raise Exception("couldn't find file offset table")
+
+    f.seek(addr - 0x30)
+    build = f.read(0x30).strip(b'\x00').replace(b'\x00', b'\n')
+    lament(str(build, 'utf-8'))
+
+    f.seek(addr)
+    i = 0
+    while z_dump_file(f, '{:05} '.format(i)):
+        i += 1
+
+def dump_rom(fn):
+    with open(fn, 'rb') as f:
+        data = f.read()
+
+        if data[:4] != b'\x80\x37\x12\x40':
+            # TODO: check if it's a .n64 (2 byte swap) and convert
+            lament('not a .z64:', fn)
+            return
+
+        outdir = hashlib.sha1(data).hexdigest()
+        del data
+
+        # TODO: a `with` would be suitable here for handling cwd
+        try:
+            os.mkdir(outdir)
+        except FileExistsError:
+            pass
+        os.chdir(outdir)
+
+        f.seek(0)
+        z_dump(f)
+
+def z_read_file(path, fn=None):
+    if fn == None:
+        # TODO: infer from path
+        return False
+    if len(fn) < 37:
+        return False
+
+    fn = str(fn[-37:])
+
+    if fn[0] != 'V' or fn[9] != '-' or fn[18:20] != ',P' or fn[28] != '-':
+        return False
+
+    try:
+        vs = int(fn[ 1: 9], 16)
+        ve = int(fn[10:18], 16)
+        ps = int(fn[20:28], 16)
+        pe = int(fn[29:37], 16)
+    except ValueError:
+        return False
+
+    with open(path, 'rb') as f:
+        data = f.read()
+
+    return True, data, vs, ve, ps, pe
+
+def create_rom(d):
+    walker = os.walk(d)
+    if not walker:
+        return
+
+    root, _, files = next(walker)
+    del walker
+    #files = [os.path.join(root, fn) for fn in files]
+
+    rom_size = 64*1024*1024
+    with open(d+'.z64', 'w+b') as f:
+        fs = []
+        f.write(bytearray(rom_size))
+        f.seek(0)
+
+        for fn in files:
+            path = os.path.join(root, fn)
+            success, data, vs, ve, ps, pe = z_read_file(path, fn)
+            if not success:
+                lament('skipping:', fn)
+                continue
+
+            assert(vs < rom_size)
+            assert(ve <= rom_size)
+            if ps == 0xFFFFFFFF or pe == 0xFFFFFFFF:
+                ps = 0xFFFFFFFF
+                pe = 0xFFFFFFFF
+            else:
+                ps = vs
+                pe = 0
+                f.seek(vs)
+                f.write(data)
+
+            fs.append([vs, ve, ps, pe])
+
+        # fix filesystem
+        fs.sort(key=lambda vf: vf[0]) # sort by vs
+        assert(len(fs) > 2)
+        fs_entry = fs[2] # assumption
+        vs, ve, ps, pe = fs_entry
+        fs_size = ve - vs
+        f.seek(ps)
+        f.write(bytearray(fs_size))
+        f.seek(ps)
+        for vf in fs:
+            vs, ve, ps, pe = vf
+            #lament('{:08X} {:08X} {:08X} {:08X}'.format(vs, ve, ps, pe))
+            f.write(W4(vs))
+            f.write(W4(ve))
+            f.write(W4(ps))
+            f.write(W4(pe))
+        assert(f.tell() <= (pe or ve))
+
+        # fix makerom (n64 header)
+        # TODO: don't assume bootcode is 6105
+        crc1, crc2 = n64.crc(f)
+        lament('crcs: {:08X} {:08X}'.format(crc1, crc2))
+        f.seek(0x10)
+        f.write(W4(crc1))
+        f.write(W4(crc2))
+
+def run(args):
+    cwd = os.getcwd()
+    for path in args:
+        if os.path.isdir(path):
+            create_rom(path)
+        elif os.path.isfile(path):
+            dump_rom(path)
+        else:
+            lament('no-op:', path)
+        os.chdir(cwd)
+
+if __name__ == '__main__':
+    ret = 0
+    try:
+        ret = run(sys.argv)
+    except KeyboardInterrupt:
+        sys.exit(1)
+    sys.exit(ret)