From a2fe369827ab0549f9d1b455009084abbee3e90d Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sun, 1 Mar 2015 07:58:42 -0800 Subject: [PATCH] add ROM dumping utility --- Yaz0.py | 56 +++++++++++++++ n64.py | 69 ++++++++++++++++++ z64dump | 220 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 345 insertions(+) create mode 100644 Yaz0.py create mode 100644 n64.py create mode 100755 z64dump diff --git a/Yaz0.py b/Yaz0.py new file mode 100644 index 0000000..1e9ec4a --- /dev/null +++ b/Yaz0.py @@ -0,0 +1,56 @@ +# decoder ripped from: http://www.amnoid.de/gc/yaz0.txt +# encoder ripped from: +# https://bitbucket.org/ottehr/z64-fm/src/9fdc704ca42ff15c8e01b1566d4692d986920c6a/yaz0.c + +def decode(comp): + src = 16 # skip header + dst = 0 + valid = 0 # bit count + curr = 0 # code byte + + assert(comp[:4] == b'Yaz0') + assert(comp[8:12] == b'\x00\x00\x00\x00') + assert(comp[12:16] == b'\x00\x00\x00\x00') + + # we could use struct but eh we only need it once + size = comp[4]*0x1000000 + comp[5]*0x10000 + comp[6]*0x100 + comp[7] + uncomp = bytearray(size) + + while dst < size: + if not valid: + curr = comp[src] + src += 1 + valid = 8 + + if curr & 0x80: + uncomp[dst] = comp[src] + dst += 1 + src += 1 + else: + byte1 = comp[src] + byte2 = comp[src + 1] + src += 2 + + dist = ((byte1 & 0xF) << 8) | byte2 + copy = dst - (dist + 1) + assert(copy >= 0) + + n = byte1 >> 4 + if n: + n += 2 + else: + n = comp[src] + 0x12 + src += 1 + + for i in range(n): + uncomp[dst] = uncomp[copy] + copy += 1 + dst += 1 + + curr <<= 1 + valid -= 1 + + return uncomp + +def encode(uncomp): + raise Exception('Yaz0_encode: unimplemented') diff --git a/n64.py b/n64.py new file mode 100644 index 0000000..244b399 --- /dev/null +++ b/n64.py @@ -0,0 +1,69 @@ +# Based on uCON64's N64 checksum algorithm by Andreas Sterbenz + +crc_seeds = { + 6101: 0xF8CA4DDC, + 6102: 0xF8CA4DDC, + 6103: 0xA3886759, + 6105: 0xDF26F436, + 6106: 0x1FEA617A, +} + +MAX32 = 0xFFFFFFFF + +def ROL(i, b): + return ((i << b) | (i >> (32 - b))) & MAX32 + +def R4(b): + return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3] + +def crc(f, bootcode=6105): + seed = crc_seeds[bootcode] + t1 = t2 = t3 = t4 = t5 = t6 = seed + + if bootcode == 6105: + f.seek(0x0710 + 0x40) + lookup = f.read(0x100) + + f.seek(0x1000) + for i in range(0x1000, 0x101000, 4): + d = R4(f.read(4)) + + if ((t6 + d) & MAX32) < t6: + t4 += 1 + t4 &= MAX32 + + t6 += d + t6 &= MAX32 + + t3 ^= d + + b = d & 0x1F + r = (d << b) | (d >> (32 - b)) + r &= MAX32 + + t5 += r + t5 &= MAX32 + + if t2 > d: + t2 ^= r + else: + t2 ^= t6 ^ d + + if bootcode == 6105: + o = i & 0xFF + temp = R4(lookup[o:o + 4]) + else: + temp = t5 + t1 += temp ^ d + t1 &= MAX32 + + if bootcode == 6103: + crc1 = (t6 ^ t4) + t3 + crc2 = (t5 ^ t2) + t1 + elif bootcode == 6106: + crc1 = t6*t4 + t3 + crc2 = t5*t2 + t1 + else: + crc1 = t6 ^ t4 ^ t3 + crc2 = t5 ^ t2 ^ t1 + return crc1 & MAX32, crc2 & MAX32 diff --git a/z64dump b/z64dump new file mode 100755 index 0000000..12cdcf5 --- /dev/null +++ b/z64dump @@ -0,0 +1,220 @@ +#!/bin/python +# shoutouts to spinout182 + +import os, os.path +import sys +import struct +import hashlib + +import n64 +import Yaz0 + +lament = lambda *args, **kwargs: print(*args, file=sys.stderr, **kwargs) + +R1 = lambda data: struct.unpack('>B', data)[0] +R2 = lambda data: struct.unpack('>H', data)[0] +R4 = lambda data: struct.unpack('>I', data)[0] +W1 = lambda data: struct.pack('>B', data) +W2 = lambda data: struct.pack('>H', data) +W4 = lambda data: struct.pack('>I', data) + +# assume first entry is makerom (0x1060), and second entry begins from makerom +fs_sig = b"\x00\x00\x00\x00\x00\x00\x10\x60\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x60" + +def dump_as(b, fn): + with open(fn, 'w+b') as f: + f.write(b) + +def z_dump_file(f, prefix=None): + vs = R4(f.read(4)) # virtual start + ve = R4(f.read(4)) # virtual end + ps = R4(f.read(4)) # physical start + pe = R4(f.read(4)) # physical end + here = f.tell() + + if vs == ve == ps == pe == 0: + return False + + fn = 'V{:08X}-{:08X},P{:08X}-{:08X}'.format(vs, ve, ps, pe) + if prefix is not None: + fn = str(prefix) + fn + + size = ve - vs + + if ps == 0xFFFFFFFF or pe == 0xFFFFFFFF: + #lament('file does not exist') + dump_as(b'', fn) + elif pe == 0: + #lament('file is uncompressed') + pe = ps + size + f.seek(ps) + data = f.read(pe - ps) + dump_as(data, fn) + else: + #lament('file is compressed') + f.seek(ps) + compressed = f.read(pe - ps) + if compressed[:4] == b'Yaz0': + data = Yaz0.decode(compressed) + dump_as(data, fn) + else: + lament('unknown compression; skipping:', fn) + lament(compressed[:4]) + + f.seek(here) + return True, fn, vs, ve, ps, pe + +def z_find_fs(f): + while True: + # assume row alignment + data = f.read(16) + if len(data) == 0: # EOF + break + if data == fs_sig[:16]: + rest = fs_sig[16:] + if f.read(len(rest)) == rest: + return f.tell() - len(rest) - 16 + else: + f.seek(len(rest), 1) + +def z_dump(f): + f.seek(0x1060) # skip header when finding fs + addr = z_find_fs(f) + if addr == None: + raise Exception("couldn't find file offset table") + + f.seek(addr - 0x30) + build = f.read(0x30).strip(b'\x00').replace(b'\x00', b'\n') + lament(str(build, 'utf-8')) + + f.seek(addr) + i = 0 + while z_dump_file(f, '{:05} '.format(i)): + i += 1 + +def dump_rom(fn): + with open(fn, 'rb') as f: + data = f.read() + + if data[:4] != b'\x80\x37\x12\x40': + # TODO: check if it's a .n64 (2 byte swap) and convert + lament('not a .z64:', fn) + return + + outdir = hashlib.sha1(data).hexdigest() + del data + + # TODO: a `with` would be suitable here for handling cwd + try: + os.mkdir(outdir) + except FileExistsError: + pass + os.chdir(outdir) + + f.seek(0) + z_dump(f) + +def z_read_file(path, fn=None): + if fn == None: + # TODO: infer from path + return False + if len(fn) < 37: + return False + + fn = str(fn[-37:]) + + if fn[0] != 'V' or fn[9] != '-' or fn[18:20] != ',P' or fn[28] != '-': + return False + + try: + vs = int(fn[ 1: 9], 16) + ve = int(fn[10:18], 16) + ps = int(fn[20:28], 16) + pe = int(fn[29:37], 16) + except ValueError: + return False + + with open(path, 'rb') as f: + data = f.read() + + return True, data, vs, ve, ps, pe + +def create_rom(d): + walker = os.walk(d) + if not walker: + return + + root, _, files = next(walker) + del walker + #files = [os.path.join(root, fn) for fn in files] + + rom_size = 64*1024*1024 + with open(d+'.z64', 'w+b') as f: + fs = [] + f.write(bytearray(rom_size)) + f.seek(0) + + for fn in files: + path = os.path.join(root, fn) + success, data, vs, ve, ps, pe = z_read_file(path, fn) + if not success: + lament('skipping:', fn) + continue + + assert(vs < rom_size) + assert(ve <= rom_size) + if ps == 0xFFFFFFFF or pe == 0xFFFFFFFF: + ps = 0xFFFFFFFF + pe = 0xFFFFFFFF + else: + ps = vs + pe = 0 + f.seek(vs) + f.write(data) + + fs.append([vs, ve, ps, pe]) + + # fix filesystem + fs.sort(key=lambda vf: vf[0]) # sort by vs + assert(len(fs) > 2) + fs_entry = fs[2] # assumption + vs, ve, ps, pe = fs_entry + fs_size = ve - vs + f.seek(ps) + f.write(bytearray(fs_size)) + f.seek(ps) + for vf in fs: + vs, ve, ps, pe = vf + #lament('{:08X} {:08X} {:08X} {:08X}'.format(vs, ve, ps, pe)) + f.write(W4(vs)) + f.write(W4(ve)) + f.write(W4(ps)) + f.write(W4(pe)) + assert(f.tell() <= (pe or ve)) + + # fix makerom (n64 header) + # TODO: don't assume bootcode is 6105 + crc1, crc2 = n64.crc(f) + lament('crcs: {:08X} {:08X}'.format(crc1, crc2)) + f.seek(0x10) + f.write(W4(crc1)) + f.write(W4(crc2)) + +def run(args): + cwd = os.getcwd() + for path in args: + if os.path.isdir(path): + create_rom(path) + elif os.path.isfile(path): + dump_rom(path) + else: + lament('no-op:', path) + os.chdir(cwd) + +if __name__ == '__main__': + ret = 0 + try: + ret = run(sys.argv) + except KeyboardInterrupt: + sys.exit(1) + sys.exit(ret)