From cab58d0d34535eb429fbf1f18f5de39a2e9a496c Mon Sep 17 00:00:00 2001 From: modeco80 Date: Wed, 21 Sep 2022 02:31:33 -0500 Subject: [PATCH] Mishmash commit While working with scrc I made the pak files output sorted by size. This doesn't seem to have helped with performance but I guess it might be nice to maybe do the conversion in hand.. or just make the writer take a vector of pair to use. --- README.md | 37 +++++++++++++-- include/europa/structs/Pak.h | 31 ++++++------ include/europa/util/TupleElement.h | 38 +++++++++++++++ src/libeuropa/io/PakWriter.cpp | 75 +++++++++++++++++++++++++++--- src/tools/pakcreate.cpp | 4 +- 5 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 include/europa/util/TupleElement.h diff --git a/README.md b/README.md index 675f909..3d89283 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,46 @@ -## Europa tools +## EuropaTools -TODO +Tools for working with LEC Europa based games (Star Wars: Starfighter & Star Wars: Jedi Starfighter). + +As per usual for lily, written in C++20. + + +## Building + +```bash +$ git clone https://github.com/modeco80/EuropaTools.git +$ cd EuropaTools +$ cmake -Bbuild -DCMAKE_BUILD_TYPE=Release +$ cmake --build build -j $(nproc) +# ... profit? +``` ## The Libraries ### `libeuropa` -Provides archive IO utilities and the structures. +Provides IO readers and writers for data files, along with the structures. + +Structure documentation is seperately managed as a .hexpat in [/hexpat](https://github.com/modeco80/EuropaTools/tree/master/hexpat). ## The Tools +## `europa_pack_extractor` + +Staging tool to extract paks. Will be removed when eupak is ready. + +### `pakcreate` + +Staging tool to create paks. + +### `paktest` + +A test tool to test building paks, used during development. + +### `texdump` + +Dumper for PS2 `YATF` texture files. Mostly working, but slight WIP. + ### `eupak` Swiss army knife for Europa packfiles. diff --git a/include/europa/structs/Pak.h b/include/europa/structs/Pak.h index 5ea7f0b..c7ed042 100644 --- a/include/europa/structs/Pak.h +++ b/include/europa/structs/Pak.h @@ -16,14 +16,14 @@ namespace europa::structs { - enum class PakVersion : u16 { - Starfighter = 0x4, - Ver2 = 0x5 - }; - struct [[gnu::packed]] PakHeader { constexpr static const char VALID_MAGIC[16] = "Europa Packfile"; + enum class Version : u16 { + Ver4 = 0x4, + Ver5 = 0x5 + }; + char magic[16]; // "Europa Packfile\0" /** @@ -31,7 +31,7 @@ namespace europa::structs { */ u16 headerSize; - PakVersion version; + Version version; u8 pad; u32 tocOffset; @@ -42,6 +42,7 @@ namespace europa::structs { u32 creationUnixTime; + // Zeroes. u32 reservedPad; /** @@ -54,7 +55,7 @@ namespace europa::structs { /** * Initialize this header (used when writing). */ - void Init(PakVersion ver) noexcept { + void Init(Version ver) noexcept { // clear any junk memset(this, 0, sizeof(PakHeader)); @@ -73,21 +74,22 @@ namespace europa::structs { if(std::strcmp(magic, VALID_MAGIC) != 0) return false; - using enum PakVersion; + // Check header size. + if(headerSize != sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1)) + return false; + + using enum Version; // Version must match ones we support, // otherwise it's invalid. switch(version) { - case Starfighter: - case Ver2: - break; + case Ver4: + case Ver5: + return true; default: return false; } - - // Header is okay. - return true; } }; @@ -98,6 +100,7 @@ namespace europa::structs { u32 creationUnixTime; }; + static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!"); static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid when writing archives."); static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!"); diff --git a/include/europa/util/TupleElement.h b/include/europa/util/TupleElement.h new file mode 100644 index 0000000..ca9d8ff --- /dev/null +++ b/include/europa/util/TupleElement.h @@ -0,0 +1,38 @@ +// +// SSX 3 Lobby Server +// +// (C) 2021-2022 modeco80 +// +// This file is licensed under the GNU General Public License Version 3. +// Text is provided in LICENSE. +// + +#ifndef EUROPA_UTIL_TUPLEELEMENT_H +#define EUROPA_UTIL_TUPLEELEMENT_H + +#include +#include + +#include + +namespace europa::util { + + namespace detail { + + template + struct TupleElementImpl { + template + constexpr decltype(auto) operator()(T&& t) const { + using std::get; + return get(std::forward(t)); + } + }; + + template + inline constexpr TupleElementImpl TupleElement; + } + + using detail::TupleElement; +} + +#endif // EUROPA_UTIL_TUPLEELEMENT_H diff --git a/src/libeuropa/io/PakWriter.cpp b/src/libeuropa/io/PakWriter.cpp index 79d96e4..db1400b 100644 --- a/src/libeuropa/io/PakWriter.cpp +++ b/src/libeuropa/io/PakWriter.cpp @@ -12,6 +12,9 @@ #include "StreamUtils.h" +#include +#include + namespace europa::io { void PakWriter::Init(structs::PakVersion version) { @@ -23,28 +26,86 @@ namespace europa::io { return archiveFiles; } + // move to a util/ header + + template + constexpr T AlignBy(T value, std::size_t alignment) { + return (-value) & alignment - 1; + } + + /** + * Class functor for flattening a map. + */ + template + struct MapFlatten { + /** + * Storage type to store one key -> value pair. + */ + using FlattenedType = std::pair; + using ArrayType = std::vector; + + constexpr explicit MapFlatten(Map& mapToFlatten) + : map(mapToFlatten) { + + } + + ArrayType operator()() const { + ArrayType arr; + arr.reserve(map.size()); + + for(auto& [ key, value ] : map) + arr.emplace_back(std::make_pair(key, value)); + + return arr; + } + + private: + Map& map; + }; + + // TODO: + // - Composable operations (WriteTOC, WriteFile, WriteHeader) + // - Add IProgressReportSink reporting + void PakWriter::Write(std::ostream& os) { - pakHeader.fileCount = archiveFiles.size(); + + // This essentially converts our map we use for faster insertion + // into a flat array we can sort easily. + // + // NB: this copies by value, so during this function we use 2x the ram. + // doesn't seem to be a big problem though. + auto sortedFiles = MapFlatten{archiveFiles}(); + + // Sort the flattened array by file size, the biggest first. + // Doesn't seem to help (neither does name length) + std::ranges::sort(sortedFiles, std::greater{}, [](const decltype(MapFlatten{archiveFiles})::FlattenedType& elem) { + return std::get<1>(elem).GetTOCEntry().size; + }); // Leave space for the header os.seekp(sizeof(structs::PakHeader), std::ostream::beg); - // Seek forwards for version 2 PAKs, as the only - // difference seems to be this additional bump + // Version 5 paks seem to have an additional bit of reserved data + // (which is all zeros.) if(pakHeader.version == structs::PakVersion::Ver2) { os.seekp(6, std::ostream::cur); } // Write file data - for(auto& [filename, file] : archiveFiles) { + for(auto& [filename, file] : sortedFiles) { + //std::cout << "PakWriteFile \"" << filename << "\"\n Size " << file.GetTOCEntry().size << "\n"; + file.GetTOCEntry().offset = os.tellp(); os.write(reinterpret_cast(file.GetData().data()), file.GetData().size()); + + // Flush on file writing + os.flush(); } pakHeader.tocOffset = os.tellp(); // Write the TOC - for(auto& [filename, file] : archiveFiles) { + for(auto& [filename, file] : sortedFiles) { file.FillTOCEntry(); // Write the pstring @@ -56,9 +117,11 @@ namespace europa::io { impl::WriteStreamType(os, file.GetTOCEntry()); } - // Fill out the TOC size. + // Fill out the rest of the header. + pakHeader.fileCount = archiveFiles.size(); pakHeader.tocSize = static_cast(os.tellp()) - (pakHeader.tocOffset - 1); + // As the last step, write it. os.seekp(0, std::ostream::beg); impl::WriteStreamType(os, pakHeader); } diff --git a/src/tools/pakcreate.cpp b/src/tools/pakcreate.cpp index 39f3d2b..221e91c 100644 --- a/src/tools/pakcreate.cpp +++ b/src/tools/pakcreate.cpp @@ -69,7 +69,9 @@ int main(int argc, char** argv) { file.SetData(std::move(pakData)); file.FillTOCEntry(); - std::cout << "Added \"" << relativePathName << "\"\n"; + file.GetTOCEntry().creationUnixTime = 0; + + //std::cout << "File \"" << relativePathName << "\"\n"; writer.GetFiles()[relativePathName] = std::move(file); }