Initial JAP REV0 extraction for function exploring (#241)

* progress so far

* no v

* root yamls ignored

* gitignore jap version src extraction for now

* wipe this

* separate yamls

.gitignore

@@ -4,7 +4,7 @@ __pycache__
 .splat/
 starfox64.ld
 starfox64.uncompressed.ld
-starfox64.us.rev1.yaml
+starfox64.*.*.yaml
 ctx.c
 ctx.c.m2c
 *.d
@@ -24,4 +24,5 @@ debug/
 .venv/
 m2cfiles/
 TempComp.bin.mio0
-torch.hash.yml
+torch.hash.yml
+srcjp

Makefile

@@ -122,8 +122,12 @@ BUILD_DEFINES ?=
 
 ifeq ($(VERSION),us)
     BUILD_DEFINES   += -DVERSION_US=1
-else
-$(error Invalid VERSION variable detected. Please use 'us')
+# else
+# $(error Invalid VERSION variable detected. Please use 'us')
+endif
+
+ifeq ($(VERSION),jap)
+    BUILD_DEFINES   += -DVERSION_JAP=1
 endif
 
 ifeq ($(NON_MATCHING),1)

tools/comptool.py

@@ -106,8 +106,9 @@ def find_file_table(ROM):
 
     return file_table_start
 
+decomp_inds = [0, 1, 2, 3, 4, 5, 15, 16, 21, 22, 23, 24, 48]
+
 def compress(baserom, comprom, mio0, extract_dest=None):
-    decomp_inds = [0, 1, 2, 3, 4, 5, 15, 16, 21, 22, 23, 24, 48]
     file_table = find_file_table(baserom)
     
     # comp_const = 0xFFFEFFFFFE1E7FC0
@@ -185,22 +186,14 @@ def compress(baserom, comprom, mio0, extract_dest=None):
 
 def decompress(baserom, decomprom, mio0, extract_dest=None, print_inds=False):
     file_table = find_file_table(baserom)
-    
-    if file_table_dict.get(file_table) == "US1.0":
-        ext = "rev0.z64"
-    elif file_table_dict.get(file_table) == "US1.1":
-        ext = "z64"
-    else:
-        ext = ('%X' % file_table) + ".z64"
-    
+
     print("File table found at 0x%X" % file_table)
     print("Detected ROM version is " + file_table_dict.get(file_table, "Unknown"))
         
-    outrom = decomprom.replace("z64", ext)
     
-    with open(outrom, 'w+b') as decompfile, open(baserom, 'rb') as basefile:
+    with open(decomprom, 'w+b') as decompfile, open(baserom, 'rb') as basefile:
         file_count = 0
-        decomp_inds = []
+        decomp_file_inds = []
         
         while True:
             file_entry = file_table + 0x10 * file_count
@@ -226,7 +219,7 @@ def decompress(baserom, decomprom, mio0, extract_dest=None, print_inds=False):
 
             if comp_flag == 0:
                 v_file_size = p_file_size
-                decomp_inds += [file_count]
+                decomp_file_inds += [file_count]
                 dec_msg = 'uncompressed'
             elif comp_flag == 1:
                 file_bytes = mio0_dec_bytes(file_bytes, mio0)
@@ -242,15 +235,18 @@ def decompress(baserom, decomprom, mio0, extract_dest=None, print_inds=False):
 
             v_file_end = v_file_begin + v_file_size
 
+            file_name = file_names[file_count] + '.bin'
+            
+            print("name: " + file_name)
+            print("start: 0x%X" % v_file_begin)
+
             if extract_dest is not None:
                 if not os.path.exists(extract_dest):
                     os.mkdir(extract_dest)
 
-                file_name = file_names[file_count] + '.bin'
-
                 with open(extract_dest + os.sep + file_name, 'wb') as extract_file:
                     extract_file.write(file_bytes)
-
+            
             decompfile.seek(file_entry + 4)
             decompfile.write(v_file_begin.to_bytes(4,'big'))
             decompfile.write(v_file_end.to_bytes(4,'big'))
@@ -265,10 +261,12 @@ def decompress(baserom, decomprom, mio0, extract_dest=None, print_inds=False):
         decompfile.write(crc1.to_bytes(4, 'big'))
         decompfile.write(crc2.to_bytes(4, 'big'))
         print("Decompressed %d files." % file_count)
-        if(print_inds) :
+        if print_inds:
             print("These file numbers were not compressed:")
-            print(decomp_inds)
-        
+            print(decomp_file_inds)
+        elif decomp_file_inds != decomp_inds:
+            print("Warning: Unusual compression scheme. These files were uncompressed:")
+            print(decomp_file_inds)
     return
 
 parser = argparse.ArgumentParser(description='Compress or decompress a Star Fox 64 ROM')
@@ -280,6 +278,7 @@ parser.add_argument('-d', action='store_true',help='decompress provided ROM')
 parser.add_argument('-m', metavar='mio0',dest='mio0',help='Path to mio0 tool if not in same directory')
 parser.add_argument('-r', action="store_true",help='Fix crc without compressing or decompressing')
 parser.add_argument('-i', action='store_true',help='Print indices of uncompressed files during decompression.')
+parser.add_argument('-v', action='store_true',help='Print file names and offsets of decompressed files.')
 # parser.add_argument('-v', action='store_true',help='show what changes are made')
 
 if __name__ == '__main__':
@@ -299,4 +298,3 @@ if __name__ == '__main__':
     else:
         print("Something went wrong.")
 
-

yamls/jp/rev0/assets.yaml

@@ -0,0 +1,3 @@
+  - name: audio_seq
+    type: bin
+    start: 0xE9950

yamls/jp/rev0/header.yaml

@@ -0,0 +1,78 @@
+name: Starfox64 JAP REV 0 (v1.0)
+sha1: d064229a32cc05ab85e2381ce07744eb3ffaf530
+options:
+  basename: starfox64
+  target_path: baserom.jp.rev0.uncompressed.z64
+  elf_path: build/starfox64.jp.rev0.elf
+  ld_script_path: linker_scripts/jp/rev0/starfox64.ld
+  base_path: .
+  compiler: IDO
+  find_file_boundaries: True
+  header_encoding: ASCII
+  platform: n64
+  undefined_funcs_auto_path: linker_scripts/jp/rev0/auto/undefined_funcs_auto.ld
+  undefined_syms_auto_path: linker_scripts/jp/rev0/auto/undefined_syms_auto.ld
+
+  symbol_addrs_path: 
+  - linker_scripts/jp/rev0/symbol_addrs.txt
+  - linker_scripts/jp/rev0/symbol_addrs_engine.txt
+  - linker_scripts/jp/rev0/symbol_addrs_libultra.txt
+  - linker_scripts/jp/rev0/symbol_addrs_nlib_funcs.txt
+  - linker_scripts/jp/rev0/symbol_addrs_nlib_vars.txt
+  - linker_scripts/jp/rev0/symbol_addrs_assets.txt
+  - linker_scripts/jp/rev0/symbol_addrs_mesg.txt
+  - linker_scripts/jp/rev0/symbol_addrs_audio.txt
+  - linker_scripts/jp/rev0/symbol_addrs_overlays.txt
+  - linker_scripts/jp/rev0/symbol_addrs_EBFBE0.txt
+  - linker_scripts/jp/rev0/ignored_addresses.txt
+
+  asm_path: asm/jp/rev0
+  src_path: srcjp
+  asset_path: bin/jp/rev0
+
+  build_path: build
+
+  extensions_path: tools/splat_ext
+  mips_abi_float_regs: o32
+  create_asm_dependencies: True
+  do_c_func_detection: True
+  o_as_suffix: True
+  gfx_ucode: f3dex
+  mnemonic_ljust: 12
+  rom_address_padding: True
+  dump_symbols: True
+  include_macro_inc: False
+  libultra_symbols: True
+  hardware_regs: True
+
+  ld_dependencies: True
+
+
+  use_legacy_include_asm: False
+
+  asm_function_macro: glabel
+  asm_jtbl_label_macro: jlabel
+  asm_data_macro: dlabel
+
+  ld_legacy_generation: True
+  check_consecutive_segment_types: False
+
+  # section_order: [".text", ".data", ".rodata", ".bss"]
+  # auto_all_sections: [".data", ".rodata", ".bss"]
+
+
+
+  
+  mips_abi_float_regs: o32
+
+
+  disasm_unknown: True
+  # string_encoding: ASCII
+  # data_string_encoding: ASCII
+  rodata_string_guesser_level: 2
+  data_string_guesser_level: 2
+  # libultra_symbols: True
+  # hardware_regs: True
+  # gfx_ucode: # one of [f3d, f3db, f3dex, f3dexb, f3dex2]
+segments:
+

yamls/jp/rev0/main.yaml

@@ -0,0 +1,197 @@
+  - name: makerom
+    type: code
+    start: 0x0
+    subsegments:
+      - {start: 0x0, type: header, name: header}
+      - {start: 0x40, type: textbin, name: ipl3}
+      - {start: 0x1000, type: asm, vram: 0x80000400, name: entry}
+
+
+  - name: main
+    type: code
+    start: 0x1050
+    vram: 0x80000450
+    #bss_size: 0x9B1F0
+    subsegments:
+    # Microcode
+      - [0x01050, textbin, BOOT]
+
+    # Nintendo libraries  
+      - [0x3450, asm, sys]
+      - [0x3A90, asm]
+      - [0x41E0, asm]
+      - [0x4660, asm]
+      - [0x5A30, asm]
+      - [0x6460, asm]
+      - [0x7D00, asm]
+      - [0x7F10, asm]
+      - [0x8190, asm]
+      - [0x8E70, asm]
+      - [0xCA20, asm]
+      - [0xF250, asm]
+      - [0x125B0, asm]
+      - [0x14BC0, asm]
+      - [0x17770, asm]
+      - [0x1EC30, asm]
+      - [0x1FB40, asm]
+      - [0x1FB50, asm]
+      - [0x1FBF0, asm]
+      - [0x206C0, asm]
+      - [0x20C90, asm]
+      - [0x21050, asm]
+      - [0x212B0, asm]
+      - [0x213F0, asm]
+      - [0x21540, asm]
+      - [0x215F0, asm]
+      - [0x21880, asm]
+      - [0x21D70, asm]
+      - [0x21DF0, asm]
+      - [0x21EA0, asm]
+      - [0x21FB0, asm]
+      - [0x22050, asm]
+      - [0x22150, asm]
+      - [0x221C0, asm]
+      - [0x22380, asm]
+      - [0x223B0, asm]
+      - [0x22420, asm]
+      - [0x22490, asm]
+      - [0x227F0, asm]
+      - [0x22800, asm]
+      - [0x22950, asm]
+      - [0x22E00, asm]
+      - [0x22F90, asm]
+      - [0x233A0, asm]
+      - [0x233B0, asm]
+      - [0x236E0, asm]
+      - [0x237C0, asm]
+      - [0x23AC0, asm]
+      - [0x242D0, asm]
+      - [0x24370, asm]
+      - [0x25030, asm]
+      - [0x25060, asm]
+      - [0x25070, asm]
+      - [0x252A0, asm]
+      - [0x25380, asm]
+      - [0x25390, asm]
+      - [0x25440, asm]
+      - [0x25450, asm]
+      - [0x25780, asm]
+      - [0x258F0, asm]
+      - [0x25DE0, asm]
+      - [0x26190, asm]
+      - [0x26D30, asm]
+      - [0x26EB0, asm]
+      - [0x27000, asm]
+      - [0x27010, asm]
+      - [0x27020, asm]
+      - [0x27330, asm]
+      - [0x273B0, asm]
+      - [0x27430, asm]
+      - [0x27470, asm]
+      - [0x27500, asm]
+      - [0x27530, asm]
+      - [0x27950, asm]
+      - [0x27960, asm]
+      - [0x27CC0, asm]
+      - [0x283F0, asm]
+      - [0x284A0, asm]
+      - [0x28500, asm]
+      - [0x28560, asm]
+      - [0x28600, asm]
+      - [0x28E80, asm]
+      - [0x29120, asm]
+      - [0x29BE0, asm]
+      - [0x29C40, asm]
+      - [0x2A1D0, asm]
+      - [0x2A370, asm]
+      - [0x2A430, asm]
+      - [0x2A440, asm]
+      - [0x2A4A0, asm]
+      - [0x2A4F0, asm]
+      - [0x2A590, asm]
+      - [0x2A5E0, asm]
+      - [0x2A6D0, asm]
+      - [0x2A860, asm]
+      - [0x337A0, asm]
+      - [0x40940, asm]
+      - [0x40D00, asm]
+      - [0x565D0, asm, fox_load]
+      - [0x570F0, asm]
+      - [0x68190, asm]
+      - [0x81D10, asm]
+      - [0x822D0, asm]
+      - [0x94EA0, asm]
+      - [0x9F480, asm]
+      - [0xA1B10, asm]
+      - [0xB8370, asm]
+      - [0xBA0A0, asm]
+      - [0xBA350, asm]
+      - [0xC0580, asm]
+      - [0xC0D00, asm]
+      - [0xC1030, data]
+      - [0xC5CA4, rodata]
+      - [0xC6FF4, rodata]
+      - [0xC73D4, rodata]
+      - [0xC7534, rodata]
+      - [0xC7604, rodata]
+      - [0xC76B4, rodata]
+      - [0xC7954, rodata]
+      - [0xC7994, rodata]
+      - [0xC79D4, rodata]
+      - [0xC79F4, rodata]
+      - [0xC7AD4, rodata]
+      - [0xC7AF4, rodata]
+      - [0xC8014, rodata]
+      - [0xC8144, rodata]
+      - [0xC8274, rodata]
+      - [0xC83A4, rodata]
+      - [0xC8734, rodata]
+      - [0xC8864, rodata]
+      - [0xC8994, rodata]
+      - [0xC8BF4, rodata]
+      - [0xC8D24, rodata]
+      - [0xC8E54, rodata]
+      - [0xC90B4, rodata]
+      - [0xC9314, rodata]
+      - [0xC9574, rodata]
+      - [0xC9594, rodata]
+      - [0xCD104, rodata]
+      - [0xCD4A4, rodata]
+      - [0xCD4D4, rodata]
+      - [0xCD6F4, rodata]
+      - [0xCD7A4, rodata]
+      - [0xCECD4, rodata]
+      - [0xCF084, rodata]
+      - [0xCF194, rodata]
+      - [0xCF8C4, rodata]
+      - [0xCF924, rodata]
+      - [0xCF934, rodata]
+      - [0xCFAB4, rodata]
+      - [0xD0854, rodata]
+      - [0xD09F4, rodata]
+      - [0xD0A64, rodata]
+      - [0xD25A4, rodata]
+      - [0xD58F4, rodata]
+      - [0xD5AD4, rodata]
+      - [0xD5B04, rodata]
+      - [0xD5C14, rodata]
+      - [0xD5D24, rodata]
+      - [0xD6084, rodata]
+      - [0xD6F04, rodata]
+      - [0xD7ED4, rodata]
+      - [0xD82A4, rodata]
+      - [0xD8824, rodata]
+      - [0xD89E4, rodata]
+      - [0xD93D4, rodata]
+      - [0xDA0B4, rodata]
+      - [0xE8D44, rodata]
+
+  - name: dma_table
+    type: code
+    start: 0xE93C0
+    vram: 0x8017E210
+    #follows_vram: main
+    subsegments:
+     - [0xE93C0, .data, dmatable]
+
+

yamls/jp/rev0/overlays.yaml

@@ -0,0 +1,110 @@
+  - name: ovl_i1
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i1
+    start: 0xDA9680
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0xA10
+    symbol_name_format: i1_$VRAM
+    subsegments:
+    - [0xDA9680, c, fox_i1]
+    - [0xDB4240, c]
+    - [0xDBADB0, c]
+  
+  - name: ovl_i2
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i2
+    start: 0xDBD820
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x40
+    symbol_name_format: i2_$VRAM
+    subsegments:
+    - [0xDBD820, c, fox_i2]
+
+  - name: ovl_i3
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i3
+    start: 0xDCC020
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x2220
+    symbol_name_format: i3_$VRAM
+    subsegments:
+    - [0xDCC020, c, fox_i3]
+    - [0xDE3300, c]
+    - [0xDED940, c]
+    - [0xE037E0, c]
+
+  - name: ovl_i4
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i4
+    start: 0xE06D70
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x1B0
+    symbol_name_format: i4_$VRAM
+    subsegments:
+    - [0xE06D70, c, fox_i4]
+    - [0xE06E40, c]
+    - [0xE0B580, c]
+    - [0xE18DD0, c]
+    - [0xE1E2B0, c]
+
+  - name: ovl_i5
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i5
+    start: 0xE1F870
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0xAF80
+    symbol_name_format: i5_$VRAM
+    subsegments:
+    - [0xE1F870, c, fox_i5]
+    - [0xE21280, c]
+    - [0xE4F680, c]
+
+  - name: ovl_i6
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_i6
+    start: 0xE542C0
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x520
+    symbol_name_format: i6_$VRAM
+    subsegments:
+    - [0xE542C0, c, fox_i6]
+
+  - name: ovl_menu
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_menu
+    start: 0xE74D00
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x17590
+    symbol_name_format: menu_$VRAM
+    subsegments:
+    - [0xE74D00, c, fox_i_menu]
+
+  - name: ovl_ending
+    exclusive_ram_id: overlay
+    type: code
+    dir: overlays/ovl_ending
+    start: 0xEA5610
+    vram: 0x8017E7A0
+    #follows_vram: ast_radio
+    #bss_size: 0x2710
+    symbol_name_format: ending_$VRAM
+    subsegments:
+    - [0xEA5610, c, fox_end1]
+    - [0xEAAF10, c]
+    - [0xEB0800, c]
+
+  - [0xEB4CA0]

yamls/us/rev1/header.yaml

@@ -1,7 +1,4 @@
 name: Starfox64 US REV 1 (v1.1)
-# baserom compressed
-# sha1: 09f0d105f476b00efa5303a3ebc42e60a7753b7a
-# baserom uncompressed
 sha1: f7475fb11e7e6830f82883412638e8390791ab87
 options:
   basename: starfox64