From e700c3432d1e65b2bb862c9d098bd52715e9c685 Mon Sep 17 00:00:00 2001 From: modeco80 Date: Sun, 14 Aug 2022 14:32:18 -0500 Subject: [PATCH] Cleanup, add ImHex patterns tool binaries now no longer have any linter warnings add imhex patterns (for documentation's sake I guess) --- hexpat/package.hexpat | 131 ++++++++++++++++++++++++++++++ hexpat/ps2_palette.hexpat | 35 ++++++++ hexpat/ps2_texture.hexpat | 20 +++++ include/jmmt/FourCCObject.h | 19 +++++ include/jmmt/package.h | 9 +- src/tools/jmmt_pack_extractor.cpp | 20 ++--- src/tools/jmmt_renamer.cpp | 3 +- 7 files changed, 218 insertions(+), 19 deletions(-) create mode 100644 hexpat/package.hexpat create mode 100644 hexpat/ps2_palette.hexpat create mode 100644 hexpat/ps2_texture.hexpat create mode 100644 include/jmmt/FourCCObject.h diff --git a/hexpat/package.hexpat b/hexpat/package.hexpat new file mode 100644 index 0000000..a8369dd --- /dev/null +++ b/hexpat/package.hexpat @@ -0,0 +1,131 @@ +#include + + +// TECH types + +// LZSS compression header thingy. +struct Lzss_Header { + u32 next; // this is formally Lzss_Header* + // but hexpat does handle pointers + // and I kinda don't want it to. + + u8 cByteId; + u8 cHdrSize; + u8 nMaxMatch; + u8 nFillByte; + u16 nRingSize; + u16 nErrorId; + u32 nUnCompressedBytes; + u32 nCompressedBytes; + u32 nCRC; + u32 nFileId; + u32 nCompressedDataCRC; + + // if valid, print out (useful) information + //if(cByteId == 0x91 && cHdrSize == 0x20) { + //std::print(""); + //std::print(" header_size: {}, max_match: {}, fill_byte: {} ring_size: {}", cHdrSize, nMaxMatch, nFillByte, nRingSize); + //std::print(" uncompressed_size: {}, compressed_size: {}, crc: {}, file_id: {}, compressed_crc: {}", nUnCompressedBytes, nCompressedBytes, nCRC, nFileId, nCompressedDataCRC); + //std::print(""); + //} +}; + +// Data strutures actually part of the file: + +// "PGRP" entry. +// +// This marks the start of a "package group". +// Whatever that is. +struct JMMT_PGRP { + u32 magic; + u32 groupNameCrc; // This is in the string table + u32 nrfiles; + u32 pad; // seemingly always 0xCDCDCDCD +}; + +// "PFIL" entry. +// +// This represents a file block, +// which can itself represent either a whole file (> 65535 bytes), +// or part of a file (which will need to be stiched together). +struct JMMT_PFIL { + u32 magic; + + u32 unk[2]; // Don't know what these are? + + // Sequence number of the chunk. + // This repressents the order of each chunk, + // presumably so order can just be whatever. + // + // However the game seems to order chunks for files + // in order, and doesn't start/interleave other files + // in between. So this is a nice waste of 16 bits. + u16 chunkSequenceNumber; + + // Amount of chunks which need to be read + // from to read this file completely. + // + // 1 means this file starts and ends on this chunk. + u16 chunkAmount; + + // This is a CRC32 hash of the path of this file. + // + // Hashed with jmmt::HashString() (in the jmmt_tools repo). + u32 filenameCrc; + + u32 unk2[7]; // more unknown stuff I don't care/know about + + // Uncompressed size of this file chunk. Has a maximum of 65535 bytes. + u32 chunkSize; + + // Offset where this file chunk should start, + // inside of a larger buffer. + u32 blockOffset; + + // ? + u32 unk3; + + // Offset inside of the package file where + // the compressed data blob starts. + u32 dataOffset; + + u32 fileSize; + + // TECH LZSS header. + // Used to (shocker) configure LZSS decompression. + Lzss_Header lzssHeader; + + // Debug information. This doesn't print literally everything, + // just the useful stuff to look at it. + if(1) { + std::print(" Chunk seqNum: {}, Filename CRC: {:0x}, File Size: {}, Chunk Size: {}, Block Offset: {}, Data Offset: {}", chunkSequenceNumber, filenameCrc, fileSize, chunkSize, blockOffset, dataOffset); + } + + //if(lzHeader.cByteId == 0x91) + // std::print("file has a valid lzss header"); + +}; + +// This is a wrapper so we can easily do the chunk viewing in imhex. +// While there's probably a way I could do this easier this works. +// So it's what I'm gonna use. +struct PFIL_WRAPPER { + JMMT_PFIL pfilChunkZero; + + if(pfilChunkZero.chunkAmount != 1) { + std::print("This file has {} chunks", pfilChunkZero.chunkAmount); + JMMT_PFIL pfilChunkExtra[pfilChunkZero.chunkAmount - 1]; + } else { + std::print("File ended with 1 chunk"); + } +}; + + +// Group header is hardcoded for config.pak cause that's what I'm using to test + +// Group header. +JMMT_PGRP grp @ 0xd1c30; + +// This isn't right (as one PFIL chunk doesn't actually have to mean one file), +// but it works for testing and trying to understand the format. +PFIL_WRAPPER files[grp.nrfiles] @ $; \ No newline at end of file diff --git a/hexpat/ps2_palette.hexpat b/hexpat/ps2_palette.hexpat new file mode 100644 index 0000000..bedc92f --- /dev/null +++ b/hexpat/ps2_palette.hexpat @@ -0,0 +1,35 @@ +// ImHex pattern for .ps2_palette or palette_*.b files + +struct Pal1Header { + u8 magic[4]; + + u32 unk; + u16 unk2; + + // This is probably implied + // by bpp of the image, I guess + u16 NrColors; + + u16 PaletteBpp; + + u16 unk3; + + u32 HeaderSize; + + // Don't know what this data is + u8 unk4[0x1c]; +}; + +struct Pal1File { + Pal1Header Header; + + if(Header.PaletteBpp == 32) { + u32 Colors[Header.NrColors]; + } else if(Header.PaletteBpp == 16) { + u16 Colors[Header.Nrcolors]; + } +}; + + +Pal1File Pal1 @ 0x0; + diff --git a/hexpat/ps2_texture.hexpat b/hexpat/ps2_texture.hexpat new file mode 100644 index 0000000..ad54772 --- /dev/null +++ b/hexpat/ps2_texture.hexpat @@ -0,0 +1,20 @@ +// ImHex pattern for .ps2_texture or texture_*.b files + +struct Tex1Header { + u8 magic[4]; + + // I don't know what these are? + u32 unk; + u16 unk2; + + u16 width; + u16 height; + + // bpp of the texture. + // > 8 = no palette file + u16 bpp; + + u8 unknown[0x40]; +}; + +Tex1Header tex1 @ $; diff --git a/include/jmmt/FourCCObject.h b/include/jmmt/FourCCObject.h new file mode 100644 index 0000000..26de829 --- /dev/null +++ b/include/jmmt/FourCCObject.h @@ -0,0 +1,19 @@ +#ifndef JMMT_TOOLS_FOURCCOBJECT_H +#define JMMT_TOOLS_FOURCCOBJECT_H + +#include + +namespace jmmt { + + + template + struct BasicStructureWithMagic { + constexpr static TMagic TypeMagic = ValidMagic; + }; + + template + using FourCCMagic = BasicStructureWithMagic; + +} + +#endif // JMMT_TOOLS_FOURCCOBJECT_H diff --git a/include/jmmt/package.h b/include/jmmt/package.h index e213490..294a7ab 100644 --- a/include/jmmt/package.h +++ b/include/jmmt/package.h @@ -8,15 +8,10 @@ // for LzssHeader #include +#include "FourCCObject.h" + namespace jmmt { - template - struct BasicStructureWithMagic { - constexpr static TMagic TypeMagic = ValidMagic; - }; - - template - using FourCCMagic = BasicStructureWithMagic; // This is the "file header" of sorts. struct PackageEofHeader { diff --git a/src/tools/jmmt_pack_extractor.cpp b/src/tools/jmmt_pack_extractor.cpp index a5cc1d1..14d8c54 100644 --- a/src/tools/jmmt_pack_extractor.cpp +++ b/src/tools/jmmt_pack_extractor.cpp @@ -27,13 +27,13 @@ T LameRead(std::istream& is) { std::string ReadString(std::istream& is) { std::string s; - char c {}; + char c; if(!is) return ""; while(true) { - c = is.get(); + c = static_cast(is.get()); if(c == '\0') return s; @@ -60,7 +60,7 @@ struct PackageReader { } void Init() { - is.seekg(-sizeof(jmmt::PackageEofHeader), std::istream::end); + is.seekg(-static_cast(sizeof(jmmt::PackageEofHeader)), std::istream::end); eofHeader = LameRead(is); // We ideally should be at the end of file after reading the eof header. @@ -79,12 +79,12 @@ struct PackageReader { is.seekg(static_cast(eofHeader.headerStartOffset) + static_cast(eofHeader.headerSize), std::istream::beg); auto l = is.tellg(); - // seek ahead of the "header" of the debug info/string table, - // since we don't care about it (we read strings until we "stop". though - // it might be smart to trust it? idk.) + // seek ahead of the "header" of the debug string table, + // since we don't care about it (we read strings until we hit true EOF. + // though it might be smart to trust it? IDK.) is.seekg(sizeof(uint32_t), std::istream::cur); - while(l != fileSize - static_cast(sizeof(eofHeader))) { + while(l != fileSize - static_cast(sizeof(eofHeader))) { auto string = ReadString(is); crcToFilename[jmmt::HashString(string.c_str())] = string; l = is.tellg(); @@ -95,7 +95,7 @@ struct PackageReader { // Go to the start of the first file chunk, skipping the group that we just read, // after we have finished creating our CRC->filename map. - is.seekg(static_cast(eofHeader.headerStartOffset) + sizeof(jmmt::PackageGroup), std::istream::beg); + is.seekg(static_cast(eofHeader.headerStartOffset) + static_cast(sizeof(jmmt::PackageGroup)), std::istream::beg); } /** @@ -141,7 +141,7 @@ struct PackageReader { if(currChunk.compressedChunkSize == currChunk.chunkSize) { memcpy(fileWorkBuffer.data() + currChunk.blockOffset, compressedBuffer.data(), currChunk.chunkSize); } else { - jmmt::DecompressLzss(nullptr, compressedBuffer.data(), currChunk.compressedChunkSize, fileWorkBuffer.data() + currChunk.blockOffset); + jmmt::DecompressLzss(nullptr, compressedBuffer.data(), static_cast(currChunk.compressedChunkSize), fileWorkBuffer.data() + currChunk.blockOffset); } } @@ -180,7 +180,7 @@ struct PackageReader { ReadFile(cb); } - jmmt::PackageGroup& GetGroup() { + [[maybe_unused]] jmmt::PackageGroup& GetGroup() { return group; } diff --git a/src/tools/jmmt_renamer.cpp b/src/tools/jmmt_renamer.cpp index cd6e5bf..99f4dbf 100644 --- a/src/tools/jmmt_renamer.cpp +++ b/src/tools/jmmt_renamer.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include namespace fs = std::filesystem; @@ -30,7 +29,7 @@ namespace fs = std::filesystem; // We could brute-force these, but since the game has them in executable, // it's a whole lot faster to just try every game filename and see // what sticks (& rename it if it does). -constinit static std::array OriginalFilenames = { +constexpr static std::array OriginalFilenames = { // First loaded by the game "package.toc",