From b87651c549eefb5e72a7797a520bdaa3d282701e Mon Sep 17 00:00:00 2001
From: Connor <cloningdonor@gmail.com>
Date: Wed, 5 Dec 2018 03:49:48 -0800
Subject: [PATCH 1/2]

---
 extract_params.py | 224 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100644 extract_params.py

diff --git a/extract_params.py b/extract_params.py
new file mode 100644
index 0000000..077cdfc
--- /dev/null
+++ b/extract_params.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+import struct
+import csv
+import sys
+
+from collections import namedtuple
+
+Def = namedtuple("Def", "name desc notes typedef ctype pytype length bits default min max step".split(" "))
+
+typemap = dict(
+    s8="b",
+    u8="B",
+    s16="h",
+    u16="H",
+    s32="i",
+    u32="I",
+    s64="q",
+    u64="Q",
+    f32="f",
+    f64="d",
+
+    dummy8="B",  # note the array length typically alongside them.
+)
+
+
+def U(fmt, *args, **kwargs):
+    return struct.unpack(E + fmt, *args, **kwargs)
+
+
+def trunc(s):
+    if b"\0" in s:
+        return s[:s.find(b"\0")]
+    return s
+
+
+def readcstr(f, offset=None):
+    if offset is not None:
+        here = f.tell()
+        f.seek(offset)
+
+    raw = b""
+    while True:
+        buf = f.read(16)
+        if len(buf) == 0:
+            break
+
+        if b"\0" in buf:
+            raw += buf[:buf.find(b"\0")]
+            break
+        else:
+            raw += buf
+
+    if offset is not None:
+        f.seek(here)
+
+    return raw
+
+
+def read_paramdef(f):
+    defs = []
+
+    filesize, unk1, unk2, count, unk3 = U("IHHHH", f.read(12))
+    paramdef_title = f.read(32)
+    unk4, unk5 = U("HH", f.read(4))
+
+    for i in range(count):
+        # TODO: rename a lot of the variables here.
+
+        desc = f.read(64)
+        typename = f.read(8)
+        printformat = f.read(8)
+        default, min_, max_, step = U("ffff", f.read(16))
+        unk6, unk7, notes_offset = U("IIi", f.read(12))
+        full_typename = f.read(32)
+        name = f.read(32)
+        # ID? it seems to increase by 100 sometimes.
+        (unk8,) = U("I", f.read(4))
+
+        desc_str = trunc(desc).decode("shift-jis", errors="replace")
+        type_str = trunc(full_typename).decode()
+        name_str = trunc(name).decode()
+
+        length = None
+        if "[" in name_str and "]" in name_str:
+            length = int(name_str.split("[")[1].split("]")[0])
+
+        bits = None
+        if ":" in name_str:
+            bits = int(name_str.split(":")[1])
+
+        if type_str in typemap:
+            type_ = typemap[type_str]
+        else:
+            underlying_type = trunc(typename).decode()
+            type_ = typemap[underlying_type]
+
+        if notes_offset in (0, -1):
+            notes_str = ""
+        else:
+            notes = readcstr(f, notes_offset)
+            notes_str = notes.decode("shift-jis", errors="replace")
+
+        d = Def(name_str, desc_str, notes_str,
+                type_str, trunc(typename).decode(), type_, length, bits,
+                default, min_, max_, step)
+        defs.append(d)
+
+    return paramdef_title, defs
+
+
+def read_param(f, paramdef_title=None):
+    entries = []
+
+    filesize, unk1, unk2, unk3, count = U("IHHHH", f.read(12))
+    param_title = f.read(32)
+    if paramdef_title is not None:
+        if trunc(paramdef_title) != trunc(param_title):
+            raise Exception(
+                "that's the wrong paramdef for this param file!" +
+                f"\nexpected: {paramedef_title}\nretrieved: {param_title}")
+
+    unk4, unk5 = U("HH", f.read(4))
+    here = f.tell()
+
+    for i in range(count):
+        f.seek(here)
+        entry_id, param_offset, notes_offset = U("iii", f.read(12))
+        here = f.tell()
+        f.seek(param_offset)
+
+        entry = [entry_id]
+        prev_type = None
+        for d in defs:
+            is_simple = d.length is None and d.bits is None
+            if d.pytype != prev_type or d.bits is None:
+                buf, bufbits = 0, 0
+
+            # print(f"{d.pytype:24} {f.tell():X}")
+
+            size = struct.calcsize(d.pytype)
+            if is_simple:
+                (datum,) = U(d.pytype, f.read(size))
+
+            elif d.length is not None:
+                # this only seems to be used for padding, so we can skip it.
+                assert d.ctype == "dummy8"  # let's assert that though.
+                datum = f.read(d.length * size)
+
+            elif d.bits is not None:
+                if bufbits == 0 or bufbits < d.bits:
+                    assert d.pytype not in ("f", "d")
+                    (buf,) = U(d.pytype.upper(), f.read(size))
+                    bufbits = size * 8
+
+                mask = (1 << d.bits) - 1
+                if big_endian:
+                    datum = (buf >> (size * 8 - d.bits)) & mask
+                    buf <<= d.bits
+                else:
+                    datum = buf & mask
+                    buf >>= d.bits
+                bufbits -= d.bits
+
+            else:
+                raise Exception("unhandled definition: " + name)
+
+            if d.ctype != "dummy8":
+                entry.append(datum)
+
+            prev_type = d.pytype
+
+        if notes_offset in (0, -1):
+            notes_str = ""
+        else:
+            notes = readcstr(f, notes_offset)
+            notes_str = notes.decode("shift-jis", errors="replace")
+        entry.append(notes_str)
+
+        entries.append(entry)
+
+    return param_title, entries
+
+
+if __name__ == "__main__":
+    fp1 = sys.argv[1]
+    fp2 = sys.argv[2]
+    fpo = sys.argv[3]
+    fph = sys.argv[4]
+    # ew, nasty global:
+    big_endian = sys.argv[5] == "big" if len(sys.argv) > 5 else False
+
+    # ew, nasty global:
+    E = ">" if big_endian else "<"
+
+    with open(fp1, "rb") as f:
+        paramdef_title, defs = read_paramdef(f)
+
+    header = ["entryId"]
+    for d in defs:
+        name = d.name
+        if ":" in name:
+            name = name.split(":")[0]
+        if "[" in name:
+            name = name.split("[")[0]
+        if d.ctype == "dummy8":
+            # print("skipping", name)
+            continue
+        header.append(name)
+    header.append("notes")
+
+    with open(fp2, "rb") as f:
+        param_title, entries = read_param(f, paramdef_title)
+
+    with open(fpo, "w", newline="", encoding="utf-8") as f:
+        cw = csv.writer(f, dialect="excel-tab")
+        cw.writerow(header)
+        for entry in entries:
+            cw.writerow(entry)
+
+    with open(fph, "w", newline="", encoding="utf-8") as f:
+        cw = csv.writer(f, dialect="excel-tab")
+        cw.writerow(Def._fields)
+        for d in defs:
+            cw.writerow(d)

From 591b49647910b395cb714a23e2a7764b39c5bdbe Mon Sep 17 00:00:00 2001
From: Connor <cloningdonor@gmail.com>
Date: Sun, 10 Mar 2019 22:15:22 -0700
Subject: [PATCH 2/2]

---
 extract_params.py |  68 ++++++++++++++++-----------
 fmg_flatten.py    | 115 ++++++++++++++++++++++++++++++++++++++++++++++
 param_notes.py    |  27 +++++++++++
 3 files changed, 183 insertions(+), 27 deletions(-)
 create mode 100644 fmg_flatten.py
 create mode 100644 param_notes.py

diff --git a/extract_params.py b/extract_params.py
index 077cdfc..4ee1fdb 100644
--- a/extract_params.py
+++ b/extract_params.py
@@ -77,8 +77,8 @@ def read_paramdef(f):
         (unk8,) = U("I", f.read(4))
 
         desc_str = trunc(desc).decode("shift-jis", errors="replace")
-        type_str = trunc(full_typename).decode()
-        name_str = trunc(name).decode()
+        type_str = trunc(full_typename).decode("shift-jis")
+        name_str = trunc(name).decode("shift-jis")
 
         length = None
         if "[" in name_str and "]" in name_str:
@@ -182,12 +182,25 @@ def read_param(f, paramdef_title=None):
 
 
 if __name__ == "__main__":
-    fp1 = sys.argv[1]
-    fp2 = sys.argv[2]
-    fpo = sys.argv[3]
-    fph = sys.argv[4]
-    # ew, nasty global:
-    big_endian = sys.argv[5] == "big" if len(sys.argv) > 5 else False
+    if len(sys.argv) == 6:
+        fp1 = sys.argv[1]
+        fp2 = sys.argv[2]
+        fpo = sys.argv[3]
+        fph = sys.argv[4]
+        # ew, nasty global:
+        big_endian = sys.argv[5] == "big"
+    elif len(sys.argv) == 4:
+        fp1 = sys.argv[1]
+        fp2 = None
+        fpo = None
+        fph = sys.argv[2]
+        # ew, nasty global:
+        big_endian = sys.argv[3] == "big"
+    else:
+        print("usage:")
+        print(" python3 extract_params.py {paramdef in} {param in} {param out} {paramdef out} [big]")
+        print(" python3 extract_params.py {paramdef in} {paramdef out} [big]")
+        sys.exit(1)
 
     # ew, nasty global:
     E = ">" if big_endian else "<"
@@ -195,27 +208,28 @@ if __name__ == "__main__":
     with open(fp1, "rb") as f:
         paramdef_title, defs = read_paramdef(f)
 
-    header = ["entryId"]
-    for d in defs:
-        name = d.name
-        if ":" in name:
-            name = name.split(":")[0]
-        if "[" in name:
-            name = name.split("[")[0]
-        if d.ctype == "dummy8":
-            # print("skipping", name)
-            continue
-        header.append(name)
-    header.append("notes")
+    if fp2 is not None and fph is not None:
+        header = ["entryId"]
+        for d in defs:
+            name = d.name
+            if ":" in name:
+                name = name.split(":")[0]
+            if "[" in name:
+                name = name.split("[")[0]
+            if d.ctype == "dummy8":
+                # print("skipping", name)
+                continue
+            header.append(name)
+        header.append("notes")
 
-    with open(fp2, "rb") as f:
-        param_title, entries = read_param(f, paramdef_title)
+        with open(fp2, "rb") as f:
+            param_title, entries = read_param(f, paramdef_title)
 
-    with open(fpo, "w", newline="", encoding="utf-8") as f:
-        cw = csv.writer(f, dialect="excel-tab")
-        cw.writerow(header)
-        for entry in entries:
-            cw.writerow(entry)
+        with open(fpo, "w", newline="", encoding="utf-8") as f:
+            cw = csv.writer(f, dialect="excel-tab")
+            cw.writerow(header)
+            for entry in entries:
+                cw.writerow(entry)
 
     with open(fph, "w", newline="", encoding="utf-8") as f:
         cw = csv.writer(f, dialect="excel-tab")
diff --git a/fmg_flatten.py b/fmg_flatten.py
new file mode 100644
index 0000000..14cd5c9
--- /dev/null
+++ b/fmg_flatten.py
@@ -0,0 +1,115 @@
+from struct import unpack as U
+import csv
+import sys
+
+big_endian = False
+
+def readint(f):
+    if big_endian:
+        return U(">i", f.read(4))[0]
+    else:
+        return U("<i", f.read(4))[0]
+
+def dumpy(f, mapping):
+    f.seek(0, 2)
+    fsize = f.tell()
+    f.seek(0, 0)
+
+    something = readint(f)
+    assert something == 0x10000, something
+
+    size = readint(f)
+    assert size == fsize, size
+
+    unk = readint(f)
+    if big_endian:
+        assert unk == 0x01FF0000, unk
+    else:
+        assert unk == 1, unk
+
+    count = readint(f)
+
+    offset_count = readint(f)
+
+    somecount1 = readint(f)  # still unknown!
+    something = readint(f)  # still unknown!
+
+    starts = {}
+    lengths = {}
+    ids = []
+    cumulative_length = 0
+    previous_end = None
+
+    for i in range(count):
+        if big_endian:
+            a, b, c = U(">iii", f.read(4 * 3))
+        else:
+            a, b, c = U("<iii", f.read(4 * 3))
+        #print(f"{a:10}: {b:10} to {c:10}")
+        length = c - b + 1
+
+        assert a not in starts
+        if previous_end is not None:
+            assert a == previous_end
+
+        starts[a] = b
+        lengths[a] = length
+
+        for i in range(length):
+            ids.append(b + i)
+
+        cumulative_length += length
+        previous_end = a + length
+
+    assert offset_count == cumulative_length
+
+    offsets = []
+    for i in range(offset_count):
+        offsets.append(readint(f))
+
+    for id, offset in zip(ids, offsets):
+        if offset == 0:
+            #mapping[id] = ""
+            continue
+
+        f.seek(offset)
+        string = ""
+        while True:
+            char = f.read(2)
+            if char == b"\0\0":
+                break
+            if big_endian:
+                string += char.decode("utf-16be")
+            else:
+                string += char.decode("utf-16le")
+        mapping[id] = string
+
+fp = sys.argv[1]
+fpo = sys.argv[2]
+
+if len(sys.argv) > 3:
+    big_endian = sys.argv[3] == "big"
+
+en_mapping = {}
+jp_mapping = {}
+
+with open(fp, "rb") as f:
+    dumpy(f, en_mapping)
+
+with open(fp.replace("ENGLISH", "JAPANESE"), "rb") as f:
+    dumpy(f, jp_mapping)
+
+from collections import defaultdict
+mappings = defaultdict(lambda: ["", ""])
+
+for k, v in en_mapping.items():
+    mappings[k][0] = v
+
+for k, v in jp_mapping.items():
+    mappings[k][1] = v
+
+with open(fpo, "w", newline="", encoding="utf-8") as f:
+    cw = csv.writer(f, dialect="excel-tab")
+    for k in sorted(mappings.keys()):
+        en_v, jp_v = mappings[k]
+        cw.writerow([k, en_v, jp_v])
diff --git a/param_notes.py b/param_notes.py
new file mode 100644
index 0000000..32fc584
--- /dev/null
+++ b/param_notes.py
@@ -0,0 +1,27 @@
+from sys import argv
+from struct import unpack as U
+
+big_endian = False
+if len(argv) > 2:
+    big_endian = argv[2] == "big"
+
+with open(argv[1], "rb") as f:
+    f.seek(0xA)
+    if big_endian:
+        count = U(">h", f.read(2))[0]
+    else:
+        count = U("<h", f.read(2))[0]
+
+    for i in range(count):
+        f.seek(0x30 + i * 3 * 4)
+
+        if big_endian:
+            entryID, paramAddr, infoAddr = U(">iii", f.read(3 * 4))
+        else:
+            entryID, paramAddr, infoAddr = U("<iii", f.read(3 * 4))
+
+        if infoAddr not in (0, -1):
+            f.seek(infoAddr)
+            string = f.read()
+            string = string[:string.index(b"\0")]
+            print(entryID, string.decode("shift-jis", errors="replace"), sep="\t")