From c1d3cfaf04c3a39c1773893f26894dcccc4ab440 Mon Sep 17 00:00:00 2001 From: modeco80 Date: Sun, 25 Jun 2023 06:52:16 -0400 Subject: [PATCH] jmmt_rs: Init A skeleton `jmmt` crate with some stuff has been created. There is also a reimplementation of the jmmt_renamer tool. More stuff will be reimplemented later. --- .clang-format | 61 ------ .editorconfig | 7 + .gitignore | 11 +- CMakeLists.txt | 13 -- Cargo.toml | 7 + README.md | 28 +-- cmake/Policies.cmake | 45 ----- crates/jmmt/Cargo.toml | 7 + crates/jmmt/src/format/mod.rs | 10 + crates/jmmt/src/format/package.rs | 110 ++++++++++ crates/jmmt/src/format/package_toc.rs | 36 ++++ crates/jmmt/src/hash/crc32.rs | 106 ++++++++++ crates/jmmt/src/hash/mod.rs | 4 + crates/jmmt/src/lib.rs | 12 ++ crates/jmmt/src/lzss/header.rs | 28 +++ crates/jmmt/src/lzss/mod.rs | 9 + crates/jmrenamer/Cargo.toml | 7 + crates/jmrenamer/src/main.rs | 74 +++++++ include/jmmt/FourCCObject.h | 20 -- include/jmmt/crc.h | 21 -- include/jmmt/lzss.h | 39 ---- include/jmmt/package.h | 87 -------- rustfmt.toml | 1 + src/libjmmt/CMakeLists.txt | 13 -- src/libjmmt/crc.cpp | 75 ------- src/libjmmt/lzss.cpp | 114 ----------- src/tools/CMakeLists.txt | 25 --- src/tools/jmmt_hashtool.cpp | 125 ------------ src/tools/jmmt_pack_extractor.cpp | 277 -------------------------- src/tools/jmmt_renamer.cpp | 109 ---------- 30 files changed, 428 insertions(+), 1053 deletions(-) delete mode 100755 .clang-format create mode 100644 .editorconfig delete mode 100755 CMakeLists.txt create mode 100644 Cargo.toml delete mode 100755 cmake/Policies.cmake create mode 100644 crates/jmmt/Cargo.toml create mode 100644 crates/jmmt/src/format/mod.rs create mode 100644 crates/jmmt/src/format/package.rs create mode 100644 crates/jmmt/src/format/package_toc.rs create mode 100644 crates/jmmt/src/hash/crc32.rs create mode 100644 crates/jmmt/src/hash/mod.rs create mode 100644 crates/jmmt/src/lib.rs create mode 100644 crates/jmmt/src/lzss/header.rs create mode 100644 crates/jmmt/src/lzss/mod.rs create mode 100644 crates/jmrenamer/Cargo.toml create mode 100644 crates/jmrenamer/src/main.rs delete mode 100644 include/jmmt/FourCCObject.h delete mode 100644 include/jmmt/crc.h delete mode 100644 include/jmmt/lzss.h delete mode 100644 include/jmmt/package.h create mode 100644 rustfmt.toml delete mode 100644 src/libjmmt/CMakeLists.txt delete mode 100644 src/libjmmt/crc.cpp delete mode 100644 src/libjmmt/lzss.cpp delete mode 100644 src/tools/CMakeLists.txt delete mode 100644 src/tools/jmmt_hashtool.cpp delete mode 100644 src/tools/jmmt_pack_extractor.cpp delete mode 100644 src/tools/jmmt_renamer.cpp diff --git a/.clang-format b/.clang-format deleted file mode 100755 index 343112f..0000000 --- a/.clang-format +++ /dev/null @@ -1,61 +0,0 @@ -# Clang-Format file - -# google style is the closest unfortunately -BasedOnStyle: Google - -# force T* or T& -# rather than T * or T & -DerivePointerAlignment: false -PointerAlignment: Left - -# I think if these two aren't the same -# it won't indent with tabs even with UseTab set to Always -TabWidth: 4 -IndentWidth: 4 - -UseTab: Always - -IndentPPDirectives: BeforeHash - -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false -AllowShortFunctionsOnASingleLine: None -AllowShortIfStatementsOnASingleLine: Never - -BinPackArguments: true -BinPackParameters: true -BreakConstructorInitializers: BeforeColon -BreakStringLiterals: false - -# 130 columns is good but causes some weird issues I don't quite like -# especially in some codebases -#ColumnLimit: 130 -ColumnLimit: 0 -CompactNamespaces: false - -ConstructorInitializerAllOnOneLineOrOnePerLine: true -ContinuationIndentWidth: 0 - -# turning this on causes major issues with initalizer lists, -# turn it off -Cpp11BracedListStyle: false - -# this is turned on to allow something like: -# -# T MyTValue { -# initalizer list... -# }; -SpaceBeforeCpp11BracedList: true - -FixNamespaceComments: true - -NamespaceIndentation: All -ReflowComments: true - -SortIncludes: CaseInsensitive -SortUsingDeclarations: true - - -SpacesInSquareBrackets: false -SpaceBeforeParens: Never -SpacesBeforeTrailingComments: 1 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..19546e0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 diff --git a/.gitignore b/.gitignore index 4dd14ed..b64aa9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,7 @@ -/.idea -cmake-build-* -build/ - -# swap +# swap files various editors produce *.kate-swp *.swp -# TEMPORARY!!!! -/attic \ No newline at end of file +# Cargo +/Cargo.lock +/target diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100755 index 40f3eb1..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -cmake_minimum_required(VERSION 3.10) - -# Prohibit in-source tree builds. -if(" ${CMAKE_SOURCE_DIR}" STREQUAL " ${CMAKE_BINARY_DIR}") - message(FATAL_ERROR "In-source builds are strictly prohibited.") -endif() - -include(cmake/Policies.cmake) - -project(jmmt_tools) - -add_subdirectory(src/libjmmt) -add_subdirectory(src/tools) diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..fb4c4d1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] +#resolver = "2" +members = [ + "crates/jmmt", + "crates/jmrenamer" +# "crates/paktool" +] diff --git a/README.md b/README.md index 9815e64..b645a71 100644 --- a/README.md +++ b/README.md @@ -2,30 +2,14 @@ Tools to work with files for Jonny Moseley Mad Trix on the PS2. -## The Libraries +## The Status Quo -### libjmmt +This is an experimental branch for a rewrite of the tools in Rust. -A library which has: -- the CRC32 hash implementation(s) used by the game -- the LZSS decompression implementation used by the game -- Documented package file structures +### Why? -Used by the following tools. +Mostly for my sanity, but just to see how it fares out. If this gets traction, -## The Tools +and I end up adding things to this that aren't in the main branch, it will probably -### `jmmt_renamer` - -Renames the .dat files in data/ on the disc to filenames which are actually useful. - -### `jmmt_pack_extractor` - -Extractor for .pak files. - -Unlike the BMS script, this extractor takes into account several things about the format. - - -### `jmmt_met_extractor` - -TODO +be the main branch. diff --git a/cmake/Policies.cmake b/cmake/Policies.cmake deleted file mode 100755 index e81dfc5..0000000 --- a/cmake/Policies.cmake +++ /dev/null @@ -1,45 +0,0 @@ -# CMake policy configuration - -if(POLICY CMP0026) - cmake_policy(SET CMP0026 NEW) -endif() - -if(POLICY CMP0042) - cmake_policy(SET CMP0042 NEW) # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name -endif() - -if(POLICY CMP0046) - cmake_policy(SET CMP0046 NEW) # warn about non-existed dependencies -endif() - -if(POLICY CMP0051) - cmake_policy(SET CMP0051 NEW) -endif() - -if(POLICY CMP0054) # CMake 3.1: Only interpret if() arguments as variables or keywords when unquoted. - cmake_policy(SET CMP0054 NEW) -endif() - -if(POLICY CMP0056) - cmake_policy(SET CMP0056 NEW) # try_compile(): link flags -endif() - -if(POLICY CMP0066) - cmake_policy(SET CMP0066 NEW) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE -endif() - -if(POLICY CMP0067) - cmake_policy(SET CMP0067 NEW) # CMake 3.8: try_compile(): honor language standard variables (like C++11) -endif() - -if(POLICY CMP0068) - cmake_policy(SET CMP0068 NEW) # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`. -endif() - -if(POLICY CMP0075) - cmake_policy(SET CMP0075 NEW) # CMake 3.12+: Include file check macros honor `CMAKE_REQUIRED_LIBRARIES` -endif() - -if(POLICY CMP0077) - cmake_policy(SET CMP0077 NEW) # CMake 3.13+: option() honors normal variables. -endif() diff --git a/crates/jmmt/Cargo.toml b/crates/jmmt/Cargo.toml new file mode 100644 index 0000000..ab697d2 --- /dev/null +++ b/crates/jmmt/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "jmmt" +version = "0.1.0" +edition = "2021" + +[dependencies] +binext = "1.0.0" \ No newline at end of file diff --git a/crates/jmmt/src/format/mod.rs b/crates/jmmt/src/format/mod.rs new file mode 100644 index 0000000..b1975b5 --- /dev/null +++ b/crates/jmmt/src/format/mod.rs @@ -0,0 +1,10 @@ +//! Low-level structure definitions + +pub mod package; +pub mod package_toc; + +/// A trait validatable format objects should implement. +pub trait Validatable { + /// Returns true if the object is valid, false otherwise. + fn valid(&self) -> bool; +} diff --git a/crates/jmmt/src/format/package.rs b/crates/jmmt/src/format/package.rs new file mode 100644 index 0000000..469e03c --- /dev/null +++ b/crates/jmmt/src/format/package.rs @@ -0,0 +1,110 @@ +//! Package file format structures + +use crate::lzss::header::LzssHeader; +use super::Validatable; + +/// "EOF" header. The QuickBMS script uses this to seek to the PGRP entry. +#[repr(C)] +pub struct PackageEofHeader { + /// Total size of the header. + pub header_size: u32, + + /// Size of the debug string table in bytes. + pub stringtable_size: u32, + + /// Start offset of the [PackageGroup] in the package file. + pub header_start_offset: u32 +} + +/// A Package Group. I have no idea what this is yet +#[repr(C)] +pub struct PackageGroup { + pub fourcc: u32, + + /// Hash of the name of this package group. Hashed with [hash_string](crate::hash::hash_string). + pub group_name_hash: u32, + + /// File count inside of this group. + pub group_file_count: u32, + + /// Padding. Set to a fill of 0xCD. + pub pad: u32 +} + +impl PackageGroup { + /// 'PGRP' + pub const VALID_FOURCC : u32 = 0x50524750; +} + +impl Validatable for PackageGroup { + fn valid(&self) -> bool { + self.fourcc == Self::VALID_FOURCC + } +} + +/// A package file chunk. +#[repr(C)] +pub struct PackageFileChunk { + pub fourcc: u32, + + /// Unknown, stays the same per file. + pub unk: u32, + + /// Unknown, stays the same per file. + pub unk2: u32, + + /// The current chunk sequence number. + pub chunk_sequence_number: u16, + + /// The total amount of chunks which make up this file. + pub chunk_sequence_count: u16, + + /// Hash of file name. Hashed with [hash_string](crate::hash::hash_string), + /// so it is case-insensitive. However, the debug string table is always in a + /// particular case, so this doesn't really matter all too much. + pub file_name_crc: u32, + + /// Unknown data, stays the same per file. Should probably be split out + /// into seperate u32 fields at some point to figure out what they are. + pub unk3: [u32; 7], + + /// Uncompressed size of this file chunk's data. Has a maximum of 65535 bytes. + pub chunk_uncompressed_size: u32, + + /// Where this chunk should start in a larger buffer. + pub chunk_buffer_offset: u32, + + /// Compressed size of this file chunk's data. + pub chunk_compressed_size: u32, + + /// Offset in the package file where this chunk's + /// data starts. + pub chunk_data_offset: u32, + + /// Uncompressed file size. + pub file_uncompressed_size: u32, + + /// LZSS header. Only used if the file chunk is compressed. + pub lzss_header: LzssHeader +} + + +impl PackageFileChunk { + /// 'PFIL' + pub const VALID_FOURCC : u32 = 0x4C494650; + + pub fn is_compressed(&self) -> bool { + // If the compressed size matches the uncompressed size + // then the file chunk is not compressed; likewise, if it does not, + // then the file chunk is compressed. + self.chunk_compressed_size != self.chunk_uncompressed_size + } +} + +impl Validatable for PackageFileChunk { + fn valid(&self) -> bool { + // Note: Even if it's not used, the LZSS header is initalized + // to meaningful values, including magic and such. + self.fourcc == Self::VALID_FOURCC && self.lzss_header.valid() + } +} diff --git a/crates/jmmt/src/format/package_toc.rs b/crates/jmmt/src/format/package_toc.rs new file mode 100644 index 0000000..b2ad43a --- /dev/null +++ b/crates/jmmt/src/format/package_toc.rs @@ -0,0 +1,36 @@ +//! Package.toc structures + +/// An entry inside the `package.toc` file +#[derive(Debug)] +#[repr(C)] +pub struct PackageTocEntry { + /// Package file name. + file_name: [u8; 0x40], + + file_name_hash: u32, + toc_start_offset: u32, + toc_size: u32, + toc_file_count: u32, +} + +impl PackageTocEntry { + fn file_name(&self) -> Option { + String::from_utf8(self.file_name.to_vec()).ok() + } + + fn file_name_hash(&self) -> u32 { + self.file_name_hash + } + + fn toc_start_offset(&self) -> u32 { + self.toc_start_offset + } + + fn toc_size(&self) -> u32 { + self.toc_size + } + + fn toc_file_count(&self) -> u32 { + self.toc_file_count + } +} diff --git a/crates/jmmt/src/hash/crc32.rs b/crates/jmmt/src/hash/crc32.rs new file mode 100644 index 0000000..bc608ec --- /dev/null +++ b/crates/jmmt/src/hash/crc32.rs @@ -0,0 +1,106 @@ +//! CRC32 Hash Algorithm + +use std::cell::Cell; + +/// Standard Ethernet-II CRC32 polynominal table, for polynominal 0x04C11DB7. +const CRC32_TABLE: [u32; 256] = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +]; + +/// A streaming CRC-32 hash. Use this for hashing arbitrary byte slices. +pub struct Crc32Hash { + /// The current state. + state: Cell, +} + +impl Crc32Hash { + /// Create a new hash structure. + pub fn new() -> Self { + Crc32Hash { + state: Cell::new(u32::default()), + } + } + + /// Stream in some new bytes into the hash. + pub fn update(&self, data: &[u8]) { + for b in data.iter() { + let old = self.state.take(); + self.state + .set(CRC32_TABLE[((old ^ *b as u32) & 0xff) as usize] ^ (old >> 8)); + } + } + + /// Stream in new bytes in a case-insensitive manner. (this is techinically internal) + pub fn update_case_insensitive(&self, data: &[u8]) { + for b in data.iter() { + let old = self.state.take(); + self.state.set( + CRC32_TABLE[((old ^ (*b & !0x20) as u32) & 0xff) as usize] ^ (old >> 8), + ); + } + } + + pub fn get(&self) -> u32 { + self.state.get() + } +} + +/// Hash a string. [str] should be ASCII only. +pub fn hash_string(str: String) -> u32 { + let crc = Crc32Hash::new(); + crc.update_case_insensitive(str.as_bytes()); + crc.get() +} + +/// Hash a string, without case sensitivity. [str] should be ASCII only. +pub fn hash_string_case(str: String) -> u32 { + let crc = Crc32Hash::new(); + crc.update(str.as_bytes()); + crc.get() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hash_string_works() { + assert_eq!(hash_string(String::from("Hello world")), 0xebce3281); + } + + #[test] + fn hash_string_case_works() { + assert_eq!(hash_string_case(String::from("Hello world")), 0xe0512fbe); + } +} diff --git a/crates/jmmt/src/hash/mod.rs b/crates/jmmt/src/hash/mod.rs new file mode 100644 index 0000000..9e08a55 --- /dev/null +++ b/crates/jmmt/src/hash/mod.rs @@ -0,0 +1,4 @@ +//! Hash Algorithms + +pub mod crc32; +pub use crc32::*; diff --git a/crates/jmmt/src/lib.rs b/crates/jmmt/src/lib.rs new file mode 100644 index 0000000..461f0c0 --- /dev/null +++ b/crates/jmmt/src/lib.rs @@ -0,0 +1,12 @@ +//! JMMT Utility Library Thing + +pub mod hash; + +// lower level stuff + +pub mod format; +pub mod lzss; + +// higher level I/O? +// pub mod read; +// pub mod write; diff --git a/crates/jmmt/src/lzss/header.rs b/crates/jmmt/src/lzss/header.rs new file mode 100644 index 0000000..5e284f9 --- /dev/null +++ b/crates/jmmt/src/lzss/header.rs @@ -0,0 +1,28 @@ +use std::mem::size_of; +use crate::format::Validatable; + +#[repr(C)] +pub struct LzssHeader { + pub next: u32, // ps2 ptr. usually 0 cause theres no next header + pub byte_id: u8, + pub header_size: u8, // matches size_of::() + pub max_match: u8, + pub fill_byte: u8, + pub ring_size: u16, + pub error_id: u16, + pub uncompressed_bytes: u32, + pub compressed_bytes: u32, + pub crc_hash: u32, + pub file_id: u32, + pub compressed_data_crc: u32, +} + +impl Validatable for LzssHeader { + /// Validate this LzssHeader object. + /// This checks if: + /// - the "magic" byte is correct (0x91) + /// - the [LzssHeader::header_size] member is correct + fn valid(&self) -> bool { + self.byte_id == 0x91 && self.header_size as usize == size_of::() + } +} \ No newline at end of file diff --git a/crates/jmmt/src/lzss/mod.rs b/crates/jmmt/src/lzss/mod.rs new file mode 100644 index 0000000..3718c01 --- /dev/null +++ b/crates/jmmt/src/lzss/mod.rs @@ -0,0 +1,9 @@ +//! LZSS + +// TODO: +// - Port decompression code to Rust +// - investigate using `lzss` crate? +// - if we can't do that, investigate reimplementing compression based on `lzss` crate? + +pub mod header; + diff --git a/crates/jmrenamer/Cargo.toml b/crates/jmrenamer/Cargo.toml new file mode 100644 index 0000000..62a666d --- /dev/null +++ b/crates/jmrenamer/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "jmrenamer" +version = "0.1.0" +edition = "2021" + +[dependencies] +jmmt = { path = "../jmmt" } diff --git a/crates/jmrenamer/src/main.rs b/crates/jmrenamer/src/main.rs new file mode 100644 index 0000000..603314f --- /dev/null +++ b/crates/jmrenamer/src/main.rs @@ -0,0 +1,74 @@ +//! A reimplemntation of jmmt_renamer in Rust. +//! This program should be run in the root directory +//! of an extracted (from image) copy of the game. + +use std::{fs, io, path::Path}; +use jmmt::hash::hash_string; + +const FILENAME_TABLE : [&str; 20] = [ + // First loaded by the game + "package.toc", + + // General packs + "config.pak", + + // This file is referenced in the game files, + // but doesn't seem to exist anymore in the final build. + //"shell.pak", + + "shell_character_select.pak", + "shell_main.pak", + "shell_title.pak", + "shell_venue.pak", + "shell_event.pak", + "shell_option.pak", + "win_screens.pak", + + // Game levels + "SF_san_fran.pak", + "DC_washington.pak", + "MK_MT_KILI.pak", + "MP_MACHU_PIHU.pak", + "LV_Las_Vegas.pak", + "AN_ANTARTICA.pak", + "NP_Nepal.pak", + "TH_TAHOE.pak", + "VA_Valdez_alaska.pak", + "RV_Rome.pak", + "TR_training.pak" +]; + +/// Make a .DAT filename from a cleartext filename. +/// +/// .DAT and .MET filenames are formatted like "[hex char * 8].DAT" +/// The name component is the CRC32 of the original filename. +/// +/// The DAT/MET filename can be a max of 13 characters long. +fn hashed_dat_filename(filename: &str) -> String { + format!("{:X}.DAT", hash_string(String::from(filename))) +} + +fn main() -> io::Result<()> { + + // A relatively simple idiot-check. + if !Path::new("DATA").is_dir() { + println!("This program should be run in the root of an extracted copy."); + std::process::exit(1); + } + + for clearname in FILENAME_TABLE.iter() { + let dat_filename = hashed_dat_filename(clearname); + let dat_src = format!("DATA/{}", dat_filename); + let dat_clearname = format!("DATA/{}", String::from(*clearname)); + + let src_path = Path::new(dat_src.as_str()); + let dest_path = Path::new(dat_clearname.as_str()); + + if src_path.exists() { + fs::rename(src_path, dest_path)?; + println!("moved {} -> {}", src_path.display(), dest_path.display()); + } + } + + Ok(()) +} diff --git a/include/jmmt/FourCCObject.h b/include/jmmt/FourCCObject.h deleted file mode 100644 index 7457c2d..0000000 --- a/include/jmmt/FourCCObject.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef JMMT_TOOLS_FOURCCOBJECT_H -#define JMMT_TOOLS_FOURCCOBJECT_H - -#include - -namespace jmmt { - - - template - struct BasicStructureWithMagic { - using MagicType = TMagic; - constexpr static MagicType TypeMagic = ValidMagic; - }; - - template - using FourCCMagic = BasicStructureWithMagic; - -} - -#endif // JMMT_TOOLS_FOURCCOBJECT_H diff --git a/include/jmmt/crc.h b/include/jmmt/crc.h deleted file mode 100644 index 74b1b34..0000000 --- a/include/jmmt/crc.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef JMMT_CRC_H -#define JMMT_CRC_H - -#include - -namespace jmmt { - /** - * Result type of HashString()/HashStringCase(). - */ - using crc32_t = std::uint32_t; - - crc32_t HashString(const char* s); - - /** - * Hash a case-sensitive string. - */ - crc32_t HashStringCase(const char* s); - -} // namespace jmmt - -#endif // JMMT_CRC_H diff --git a/include/jmmt/lzss.h b/include/jmmt/lzss.h deleted file mode 100644 index 476e5d8..0000000 --- a/include/jmmt/lzss.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef JMMT_LZSS_H -#define JMMT_LZSS_H - -#include - -namespace jmmt { - - struct LzssHeader { - std::uint32_t next; // done to keep data layout consistent with PS2 - std::uint8_t cByteId; - std::uint8_t cHdrSize; // should be sizeof(LzssHeader) - std::uint8_t nMaxMatch; - std::uint8_t nFillByte; - std::uint16_t nRingSize; - std::uint16_t nErrorId; - std::uint32_t nUnCompressedBytes; - std::uint32_t nCompressedBytes; - std::uint32_t nCRC; - std::uint32_t nFileId; - std::uint32_t nCompressedDataCRC; - }; - - static_assert(sizeof(LzssHeader) == 0x20, "LzssHeader doesn't match game expectations, you are CERTAINLY breaking structures"); - - /** - * Decompress TECH LZSS data. - * - * \param[in,out] header LZSS header. Unused. Set to nullptr for now. - * \param[in] compressedInput LZSS compressed input data. - * \param[in] compressedLength Compressed length. - * \param[out] destBuffer Destination buffer. - * - * \return 0 on success. Non zero value means error. - */ - int DecompressLzss(LzssHeader* header, std::uint8_t* compressedInput, std::int32_t compressedLength, std::uint8_t* destBuffer); - -} // namespace jmmt - -#endif diff --git a/include/jmmt/package.h b/include/jmmt/package.h deleted file mode 100644 index f3c52d6..0000000 --- a/include/jmmt/package.h +++ /dev/null @@ -1,87 +0,0 @@ -// JMMT PAK structures - -#ifndef JMMT_PACKAGE_H -#define JMMT_PACKAGE_H - -#include - -// for LzssHeader -#include - -#include "FourCCObject.h" - -namespace jmmt { - - - // This is the "file header" of sorts. - struct PackageEofHeader { - std::uint32_t headerSize; - std::uint32_t debugInfoSize; - - std::uint32_t headerStartOffset; - }; - - struct PackageGroup : public FourCCMagic<0x50524750 /* 'PGRP' */> { - MagicType magic; - uint32_t groupNameCrc; - - uint32_t fileCount; - uint32_t padding; // 0xcdcdcdcd - padding to 0x10 bytes - }; - - struct PackageFile : public FourCCMagic<0x4C494650 /* 'PFIL' */> { - MagicType magic; - uint32_t unk[2]; // Don't know what these are? - - // Sequence number of the chunk. - // This represents the order of each chunk, - // presumably so order can just be whatever. - // - // However, the archives seem to order chunks for files - // in order, and doesn't start/interleave other files - // in between of files. - // - // In other words: this is a nice waste of 16 bits. - uint16_t chunkSequenceNumber; - - // Amount of chunks which need to be read - // from to read this file completely. - // - // 1 means this file starts and ends on this chunk. - uint16_t chunkAmount; - - // A CRC32 hash of the path of this file. - // Hashed with jmmt::HashString(). - uint32_t filenameCrc; - - uint32_t unk2[7]; // more unknown stuff I don't know about yet - - // Uncompressed size of this file chunk. Has a maximum of 65535 bytes. - uint32_t chunkSize; - - // Offset where this file chunk should start, - // inside of a larger buffer. - uint32_t blockOffset; - - // Compressed (stored) size of this chunk. - uint32_t compressedChunkSize; - - // Offset inside of the package file where - // the compressed data blob starts. - uint32_t dataOffset; - - uint32_t fileSize; - - // TECH LZSS header. - // Used to (shocker) configure LZSS decompression. - // - // Duplicates a few things in the file. - LzssHeader lzssHeader; - }; - - static_assert(sizeof(PackageEofHeader) == 0xc, "PackageEofHeader has invalid size. Extractor 100% won't work, good job"); - static_assert(sizeof(PackageGroup) == 0x10, "PackageGroup has invalid size, extractor won't work"); - static_assert(sizeof(PackageFile) == 0x64, "PackageFile has invalid size, extractor won't work"); -} // namespace jmmt - -#endif // JMMT_PACKAGE_H diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..18d655e --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +hard_tabs = true \ No newline at end of file diff --git a/src/libjmmt/CMakeLists.txt b/src/libjmmt/CMakeLists.txt deleted file mode 100644 index bf6f6fb..0000000 --- a/src/libjmmt/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ - - -add_library(jmmt - crc.cpp - lzss.cpp - ) - -target_include_directories(jmmt PUBLIC ${PROJECT_SOURCE_DIR}/include) - -set_target_properties(jmmt PROPERTIES - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - ) \ No newline at end of file diff --git a/src/libjmmt/crc.cpp b/src/libjmmt/crc.cpp deleted file mode 100644 index b94a351..0000000 --- a/src/libjmmt/crc.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include - -namespace jmmt { - - // Standard Ethernet-II CRC32 polynominal table. - constinit static crc32_t Crc32Table[] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d - }; - - // TODO: - // - Use string_view, because C++20 (& it's cleaner) - - crc32_t HashString(const char* s) { - crc32_t crc = 0; - - while(*s) { - crc = Crc32Table[(crc ^ (*s++ & ~0x20)) & 0xff] ^ (crc >> 8); - } - return crc; - } - - // Hash a string which is case-sensitive. - crc32_t HashStringCase(const char* s) { - crc32_t crc = 0; - - while(*s) { - crc = Crc32Table[(crc ^ (*s++)) & 0xff] ^ (crc >> 8); - } - return crc; - } - - -} \ No newline at end of file diff --git a/src/libjmmt/lzss.cpp b/src/libjmmt/lzss.cpp deleted file mode 100644 index e9a2e79..0000000 --- a/src/libjmmt/lzss.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include -#include -#include - -#define LZSS_DEFAULT_RINGSIZE 512 -#define LZSS_DEFAULT_MATCHSIZE 66 -#define LZSS_EOF -1 -#define LZSS_CHUNKSIZE 256 -#define LZSS_RINGBITS 9 -#define LZSS_THRESHOLD 2 -#define LZSS_STALLBIT (30) - -#define LZSS_GETBYTE(in,inb,out) \ - do { \ - if( (inb) <= (nInputBufferIndex) ) \ - (out) = -1; \ - else \ - { \ - /* std::printf("getting '%c'\n", *((in)+nInputBufferIndex)); */ \ - (out) = *((uint8_t*)((in)+nInputBufferIndex)); \ - nInputBufferIndex++; \ - } \ - }while(0) - -// this version logs what it's going to put and where. -//#define LZSS_PUTBYTE(outp,outb) std::printf("LZSS_PUTBYTE(%llu, %x)\n", outp - oldptr, outb); \ - *(outp)++ = (uint8_t)(outb) - -#define LZSS_PUTBYTE(outp,outb) *(outp)++ = (uint8_t)(outb) - -#define FileIO_ZeroMemory(dst, size) memset(dst, 0, size) - -namespace jmmt { - - int DecompressLzss(LzssHeader* header, std::uint8_t* compressedInput, std::int32_t compressedLength, std::uint8_t* destBuffer) { - int32_t nRingIndex, nInSize, nInputBufferIndex, nRingBits, nRingSize; - uint8_t aRingBuffer[LZSS_DEFAULT_RINGSIZE], *pRingBuffer; - uint32_t nBitFlags = 0; - - std::int32_t nInByte; - - //auto* oldptr = destBuffer; // uncomment for logging version of LZSS_PUTBYTE - - // TODO: this is where we might want to place header usage. You know, if we need to. - nBitFlags = 0; - nInputBufferIndex = 0; - nInSize = compressedLength; - nRingSize = LZSS_DEFAULT_RINGSIZE; - nRingBits = LZSS_RINGBITS; - nRingIndex = LZSS_DEFAULT_RINGSIZE - LZSS_DEFAULT_MATCHSIZE; - - // Use stack allocated default ring buffer - pRingBuffer = &aRingBuffer[0]; - - FileIO_ZeroMemory(pRingBuffer, nRingSize); - //memset(pRingBuffer, ' ', nRingSize); - - - for(;;) { // get next 8 opcodes? - if(((nBitFlags >>= 1) & 256) == 0) { - LZSS_GETBYTE(compressedInput, nInSize, nInByte); - if(nInByte == -1) - break; - - //std::printf("LZSS new opcodes\n"); - - // store 255 in upper word, when zero get next 8 opcodes - nBitFlags = nInByte | 0xff00; - } - - // single char - if(nBitFlags & 1) { - LZSS_GETBYTE(compressedInput, nInSize, nInByte); - if(nInByte == -1) - break; - - //std::printf("LZSS single char '%c'\n", nInByte); - - LZSS_PUTBYTE(destBuffer, nInByte); - pRingBuffer[nRingIndex++] = (uint8_t)nInByte; - nRingIndex &= (nRingSize - 1); - } - - // string - else { // get position & length pair (note: 1 bit of position is stored in length word) - int32_t i, j; - LZSS_GETBYTE(compressedInput, nInSize, i); - if(i == -1) - break; - LZSS_GETBYTE(compressedInput, nInSize, j); - - i |= ((j >> (16 - nRingBits)) << 8); - j = (j & (0x00FF >> (nRingBits - 8))) + LZSS_THRESHOLD; - - // std::printf("LZSS string pos %d len %d\n", i , j); - - //LZSS_VALIDATE(j <= LZSS_DEFAULT_MATCHSIZE, "Invalid match size for decompression"); - - for(int32_t k = 0; k <= j; ++k) { - nInByte = pRingBuffer[(i + k) & (nRingSize - 1)]; - - //std::printf("LZSS string byte '%c'\n", nInByte); - - LZSS_PUTBYTE(destBuffer, nInByte); - pRingBuffer[nRingIndex++] = (uint8_t)nInByte; - nRingIndex &= (nRingSize - 1); - } - } - } - - return 0; - } - -} // namespace jmmt diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt deleted file mode 100644 index 4c03005..0000000 --- a/src/tools/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ - -add_executable(jmmt_renamer jmmt_renamer.cpp) -target_link_libraries(jmmt_renamer PUBLIC jmmt) - -set_target_properties(jmmt_renamer PROPERTIES - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - ) - - -add_executable(jmmt_pack_extractor jmmt_pack_extractor.cpp) -target_link_libraries(jmmt_pack_extractor PUBLIC jmmt) - -set_target_properties(jmmt_pack_extractor PROPERTIES - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - ) - -add_executable(jmmt_hashtool jmmt_hashtool.cpp) -target_link_libraries(jmmt_hashtool PUBLIC jmmt) - -set_target_properties(jmmt_hashtool PROPERTIES - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - ) diff --git a/src/tools/jmmt_hashtool.cpp b/src/tools/jmmt_hashtool.cpp deleted file mode 100644 index 4fb3687..0000000 --- a/src/tools/jmmt_hashtool.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// JMMT HashTool - -#include - -#include -#include -#include -#include -#include - -struct Arguments { - enum class OutputMode { - Hex, ///< Hexadecimal output. - Decimal ///< Decimal output. - }; - - char* hashName {}; - bool useCase { false }; - OutputMode outMode { OutputMode::Hex }; - - /** Parse arguments from the main() argv. **/ - static Arguments FromArgv(int argc, char** argv) { - Arguments args; - args.progname = argv[0]; - - // no options provided - if(argc == 1) { - args.DispHelp(); - std::exit(1); - } - - // non-pepper getopt(). I'm too lazy to make it any better though - for(int i = 1; i < argc; ++i) { - if(argv[i][0] == '-' && argv[i][1] != '\0') { - char sw = argv[i][1]; - - switch(sw) { - // flag options - - case 'c': - args.useCase = true; - break; - - case 'd': - if(args.outMode == OutputMode::Hex) - args.outMode = OutputMode::Decimal; - break; - - case 'h': - if(args.outMode == OutputMode::Decimal) - args.outMode = OutputMode::Hex; - break; - - // terminals - case '?': - args.DispHelp(); - std::exit(0); - - default: - std::printf("Unknown command-line switch '-%c'\n", sw); - args.DispHelp(); - std::exit(1); - } - } else { - // Assume any non-positional argument is what we're supposed to hash - args.hashName = argv[i]; - } - } - return args; - } - - bool Validate() const { - if(!hashName) { - std::printf("No hash name provided\n"); - DispHelp(); - return false; - } - return true; - } - - private: - char* progname {}; - - void DispHelp() const { - // no I'm not sorry - std::printf( - // clang-format off - "JMMT HashTool - a thing for generating TECH HashID's\n" - "Usage: %s [-c] [-?] \n" - " -c Use case-senstive HashID variant (default is case-insensitive)\n" - " -d Output in decimal (default hex)\n" - " -h Output as hexadecimal (if previously overridden; kinda pointless)\n" - " -? Show this help message (and exit)\n", - progname - // clang-format on - ); - } -}; - -int main(int argc, char** argv) { - auto args = Arguments::FromArgv(argc, argv); - - if(!args.Validate()) { - return 1; - } - - jmmt::crc32_t result {}; - if(args.useCase) - result = jmmt::HashStringCase(args.hashName); - else - result = jmmt::HashString(args.hashName); - - // clang-format off - using enum Arguments::OutputMode; - switch(args.outMode) { - case Decimal: std::printf("%d\n", result); break; - case Hex: std::printf("0x%08x\n", result); break; -#ifdef __GNUC__ - // Mark this path explicitly as UB - default: __builtin_unreachable(); -#endif - } - // clang-format on - return 0; -} diff --git a/src/tools/jmmt_pack_extractor.cpp b/src/tools/jmmt_pack_extractor.cpp deleted file mode 100644 index 14d8c54..0000000 --- a/src/tools/jmmt_pack_extractor.cpp +++ /dev/null @@ -1,277 +0,0 @@ -// Package file extractor. -// Yes, this code is messy, but I just wanted it to work after days of it not doing so. - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -// This is lame. But it works :) -template -T LameRead(std::istream& is) { - if(!is) - throw std::runtime_error("stream is bad"); - - T t {}; - is.read(reinterpret_cast(&t), sizeof(T)); - return t; -} - -std::string ReadString(std::istream& is) { - std::string s; - char c; - - if(!is) - return ""; - - while(true) { - c = static_cast(is.get()); - - if(c == '\0') - return s; - - s.push_back(c); - } -} - -/** - * Reader for a package file. - */ -struct PackageReader { - - /** - * Decompressed and un-split package file. - */ - struct DecompressedFile { - std::string filename; - std::vector data; - }; - - explicit PackageReader(std::istream& is) - : is(is) { - } - - void Init() { - is.seekg(-static_cast(sizeof(jmmt::PackageEofHeader)), std::istream::end); - eofHeader = LameRead(is); - - // We ideally should be at the end of file after reading the eof header. - auto fileSize = is.tellg(); - - is.seekg(static_cast(eofHeader.headerStartOffset), std::istream::beg); - - group = LameRead(is); - if(group.magic != jmmt::PackageGroup::TypeMagic) { - fileInvalid = true; - return; - } - - // Read the string table, and hash every string in it into a map of CRC to string. - { - is.seekg(static_cast(eofHeader.headerStartOffset) + static_cast(eofHeader.headerSize), std::istream::beg); - auto l = is.tellg(); - - // seek ahead of the "header" of the debug string table, - // since we don't care about it (we read strings until we hit true EOF. - // though it might be smart to trust it? IDK.) - is.seekg(sizeof(uint32_t), std::istream::cur); - - while(l != fileSize - static_cast(sizeof(eofHeader))) { - auto string = ReadString(is); - crcToFilename[jmmt::HashString(string.c_str())] = string; - l = is.tellg(); - } - } - - //std::cout << "Group name: \"" << crcToFilename[group.groupNameCrc] << "\"\n"; - - // Go to the start of the first file chunk, skipping the group that we just read, - // after we have finished creating our CRC->filename map. - is.seekg(static_cast(eofHeader.headerStartOffset) + static_cast(sizeof(jmmt::PackageGroup)), std::istream::beg); - } - - /** - * Get if the package file is invalid. - * - * \return True if file is invalid; false otherwise - */ - [[nodiscard]] bool Invalid() const { - return fileInvalid; - } - - /** - * Read a single file chunk. - */ - void ReadFileChunk() { - currChunk = LameRead(is); - - if(currChunk.magic != jmmt::PackageFile::TypeMagic) { - std::cout << "Invalid file chunk\n"; - std::exit(1); - } - - // If we finished a file, the work buffer is empty. - if(fileWorkBuffer.empty()) { - currFileName = crcToFilename[currChunk.filenameCrc]; - - //std::cout << "Reading \"" << currFileName << "\".\n"; - - chunksLeft = currChunk.chunkAmount - 1; - fileWorkBuffer.resize(currChunk.fileSize); - } - - std::vector compressedBuffer(currChunk.compressedChunkSize); - - // Read into temporary buffer. - auto old = is.tellg(); - is.seekg(currChunk.dataOffset, std::istream::beg); - is.read(reinterpret_cast(compressedBuffer.data()), currChunk.compressedChunkSize); - is.seekg(old, std::istream::beg); - - // If the chunk isn't actually compressed, just copy it into the work buffer. - // If it is, decompress it into the work buffer. - if(currChunk.compressedChunkSize == currChunk.chunkSize) { - memcpy(fileWorkBuffer.data() + currChunk.blockOffset, compressedBuffer.data(), currChunk.chunkSize); - } else { - jmmt::DecompressLzss(nullptr, compressedBuffer.data(), static_cast(currChunk.compressedChunkSize), fileWorkBuffer.data() + currChunk.blockOffset); - } - } - - /** - * Read a file from this package. - * \param[in] cb Called when file is finished being read. - */ - template - void ReadFile(DoneCallback&& cb) { - ReadFileChunk(); - - // Read additional chunks required to complete the file, - // if we (well) have to. - for(auto i = 0; i < chunksLeft; ++i) { - //std::cout << "Reading additional chunk " << i + 1 << '/' << chunksLeft << ".\n"; - ReadFileChunk(); - } - - std::cout << "Read file \"" << currFileName << "\" from archive.\n"; - - // Call user-provided callback - cb(DecompressedFile { .filename = currFileName, - .data = fileWorkBuffer }); - - - fileWorkBuffer.clear(); - } - - /** - * Read all possible files from this package. - * \param[in] cb Called when file is finished being read. - */ - template - void ReadFiles(DoneCallback&& cb) { - for(auto i = 0; i < group.fileCount; ++i) - ReadFile(cb); - } - - [[maybe_unused]] jmmt::PackageGroup& GetGroup() { - return group; - } - - private: - std::istream& is; - - // Set to true on any invalid file data. - bool fileInvalid = false; - - /** - * EOF header. - */ - jmmt::PackageEofHeader eofHeader {}; - - /** - * Group header. - */ - jmmt::PackageGroup group {}; - - /** - * CRC->sensible string map. - * Might be worth renaming. - */ - std::map crcToFilename; - - /** - * The amount of chunks left that we need to read to complete a file. - */ - uint32_t chunksLeft {}; - - /** - * Filename from crcToFilename of the file we're reading. - */ - std::string currFileName; - - /** - * The current chunk the reader is reading. - */ - jmmt::PackageFile currChunk {}; - - /** - * Work buffer used to store the file we are currently trying to read. - * - * Freed when a file is extracted. - */ - std::vector fileWorkBuffer; -}; - -int main(int argc, char** argv) { - if(argc != 2) { - std::cout << "Usage: " << argv[0] << " [path to JMMT PAK file]"; - return 1; - } - - std::ifstream ifs(argv[1], std::ifstream::binary); - - if(!ifs) { - std::cout << "Invalid file \"" << argv[1] << "\"\n"; - return 1; - } - - PackageReader reader(ifs); - - reader.Init(); - - if(reader.Invalid()) { - std::cout << "File \"" << argv[1] << "\" doesn't seem to be a PAK file.\n"; - return 1; - } - - auto path = fs::path(argv[1]).stem(); - - reader.ReadFiles([&](const auto& file) { - auto outpath = path / file.filename; - - if(!fs::exists(outpath.parent_path())) - fs::create_directories(outpath.parent_path()); - - std::ofstream ofs(outpath.string(), std::ofstream::binary); - if(!ofs) { - std::cerr << "Could not open \"" << outpath.string() << "\" for writing.\n"; - return; - } - - ofs.write(reinterpret_cast(file.data.data()), static_cast(file.data.size())); - ofs.close(); - - std::cout << "Wrote \"" << outpath.string() << "\" to disk.\n"; - }); - - std::cout << "Finished extracting successfully.\n"; - - return 0; -} \ No newline at end of file diff --git a/src/tools/jmmt_renamer.cpp b/src/tools/jmmt_renamer.cpp deleted file mode 100644 index 99f4dbf..0000000 --- a/src/tools/jmmt_renamer.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// JMMT .DAT file renamer utility thingy -// -// Renames the .DAT files in /DATA on the disc to -// the original filenames, for easier identification, -// less pain, and.. well just because a bunch of DAT -// files is really stupid to go through every time. -// -// (C) 2022 modeco80. -// -// Usage: -// -// - Compile the tool (or scream at me for a binary) -// - Run the tool in the DATA directory of the files -// - ... -// - Profit? - -#include -#include -#include -#include -#include -namespace fs = std::filesystem; - -#include - -// These are the original filenames that the game tries to load, -// extracted from the game binary. -// -// We could brute-force these, but since the game has them in executable, -// it's a whole lot faster to just try every game filename and see -// what sticks (& rename it if it does). -constexpr static std::array OriginalFilenames = { - // First loaded by the game - "package.toc", - - // General packs - "config.pak", - - // This file is referenced in the game files, - // but doesn't seem to exist anymore in the final build. - //"shell.pak", - - "shell_character_select.pak", - "shell_main.pak", - "shell_title.pak", - "shell_venue.pak", - "shell_event.pak", - "shell_option.pak", - "win_screens.pak", - - // Game levels - "SF_san_fran.pak", - "DC_washington.pak", - "MK_MT_KILI.pak", - "MP_MACHU_PIHU.pak", - "LV_Las_Vegas.pak", - "AN_ANTARTICA.pak", - "NP_Nepal.pak", - "TH_TAHOE.pak", - "VA_Valdez_alaska.pak", - "RV_Rome.pak", - "TR_training.pak" -}; - -std::string MakeDatFilename(const char* filename) { - char datFile[13] {}; - - // .DAT and .MET filenames are formatted like "[hex char * 8].DAT" - // The name component is the CRC32 of the original filename. - // - // The DAT/MET filename can be a max of 13 characters long. - int res = std::snprintf(&datFile[0], 13, "%X.DAT", jmmt::HashString(filename)); - - // FIXME: probably throw exception - if(res == -1) - return ""; - - return { &datFile[0], static_cast(res) }; -} - -int main() { - int renamedFiles = 0; - - for(auto filename : OriginalFilenames) { - auto datFile = MakeDatFilename(filename.data()); - - if(fs::exists(datFile)) { - // Try to rename the .DAT file to the game filename. - try { - fs::rename(datFile, filename); - } catch(std::exception& ex) { - // If there's an error renaming, we already catch - // if the source .DAT file (that's supposed to exist) - // doesn't exist, so print the exception and exit. - std::printf("Got exception: %s\n", ex.what()); - return 1; - } - - std::printf("\"%s\" -> \"%s\"\n", datFile.c_str(), filename.data()); - renamedFiles++; - } else { - // FIXME: should probably stop here? - std::printf("???? Generated hash filename \"%s\" (for \"%s\") which does not exist on disk\n", datFile.c_str(), filename.data()); - } - } - - std::printf("Renamed %d files successfully.\n", renamedFiles); - return 0; -}