diff --git a/.gitmodules b/.gitmodules index e6bcc4f..1f2850e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "third_party/libpixel"] path = third_party/libpixel url = https://github.com/modeco80/libpixel.git +[submodule "third_party/indicators"] + path = third_party/indicators + url = https://github.com/p-ranav/indicators diff --git a/CMakeLists.txt b/CMakeLists.txt index 5426582..e5a249c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ include(cmake/Policies.cmake) project(EuropaTools) add_subdirectory(third_party/libpixel) +add_subdirectory(third_party/indicators) add_subdirectory(src/libeuropa) add_subdirectory(src/tools) diff --git a/README.md b/README.md index 7e4ff5c..675f909 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Provides archive IO utilities and the structures. ## The Tools -### `europa_pak_extractor` +### `eupak` -Extractor for .pak and .pmdl (they use 1:1 the same format) files. +Swiss army knife for Europa packfiles. -TODO: this will be one `eupak` utility later, probably. +Can create, extract, and show info on them. diff --git a/hexpat/pak.hexpat b/hexpat/pak.hexpat index 04f09d2..5dbdf40 100644 --- a/hexpat/pak.hexpat +++ b/hexpat/pak.hexpat @@ -7,6 +7,8 @@ // #pragma endian little +// Big archives need a big pattern limit +#pragma max_patterns 0x40000 namespace europa { @@ -56,7 +58,7 @@ namespace europa { struct PakFile { PakHeader header; - //PakTocEntry toc[header.fileCount] @ header.tocOffset; + PakTocEntry toc[header.fileCount] @ header.tocOffset; }; } // namespace europa diff --git a/include/europa/io/PakFile.h b/include/europa/io/PakFile.h index 71aab29..15fda6c 100644 --- a/include/europa/io/PakFile.h +++ b/include/europa/io/PakFile.h @@ -9,47 +9,43 @@ #ifndef EUROPA_IO_PAKFILE_H #define EUROPA_IO_PAKFILE_H -#include -#include - #include +#include +#include + namespace europa::io { - struct PakReader; - struct PakWriter; + struct PakReader; + struct PakWriter; - struct PakFile { + struct PakFile { + using DataType = std::vector; - using DataType = std::vector; + /** + * Get the file data. + */ + [[nodiscard]] const DataType& GetData() const; - /** - * Get the file data. - */ - [[nodiscard]] const DataType& GetData() const; + /** + * Get the TOC entry responsible. + */ + [[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const; - /** - * Get the TOC entry responsible. - */ - [[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const; + void SetData(DataType&& data); - void SetData(DataType&& data); + structs::PakTocEntry& GetTOCEntry(); - structs::PakTocEntry& GetTOCEntry(); + void FillTOCEntry(); - void FillTOCEntry(); + private: + friend PakReader; + friend PakWriter; - private: - friend PakReader; - friend PakWriter; + std::vector data; + structs::PakTocEntry tocData; + }; +} // namespace europa::io - std::vector data; - structs::PakTocEntry tocData; - }; - - -} - - -#endif //EUROPA_IO_PAKFILE_H +#endif // EUROPA_IO_PAKFILE_H diff --git a/include/europa/io/PakReader.h b/include/europa/io/PakReader.h index 15f7919..56442de 100644 --- a/include/europa/io/PakReader.h +++ b/include/europa/io/PakReader.h @@ -10,7 +10,6 @@ #define EUROPA_IO_PAKREADER_H #include - #include #include @@ -20,16 +19,25 @@ namespace europa::io { struct PakReader { + using MapType = std::unordered_map; explicit PakReader(std::istream& is); void ReadData(); + void ReadFiles(); + + /** + * Read in a specific file. + */ + void ReadFile(const std::string& file); + bool Invalid() const { return invalid; } - const std::unordered_map& GetFiles() const; + MapType& GetFiles(); + const MapType& GetFiles() const; private: std::istream& stream; @@ -37,7 +45,7 @@ namespace europa::io { structs::PakHeader header {}; - std::unordered_map files; + MapType files; }; } // namespace europa::io diff --git a/include/europa/io/PakWriter.h b/include/europa/io/PakWriter.h index e820a2f..750f59b 100644 --- a/include/europa/io/PakWriter.h +++ b/include/europa/io/PakWriter.h @@ -9,12 +9,9 @@ #ifndef EUROPA_IO_PAKWRITER_H #define EUROPA_IO_PAKWRITER_H - #include #include - - #include #include @@ -24,24 +21,20 @@ namespace europa::io { * Writer for package files. */ struct PakWriter { + void Init(structs::PakVersion version); - void Init(); - - void AddFile(const std::string& path, const PakFile& data); - - void RemoveFile(const std::string& path); + std::unordered_map& GetFiles(); /** * Write the resulting archive to the given output stream. */ void Write(std::ostream& os); - private: - structs::PakHeader pakHeader{}; - std::unordered_map archiveFiles; + private: + structs::PakHeader pakHeader {}; + std::unordered_map archiveFiles; }; - -} +} // namespace europa::io #endif // EUROPA_IO_PAKWRITER_H diff --git a/include/europa/structs/Pak.h b/include/europa/structs/Pak.h index 4151c92..c431263 100644 --- a/include/europa/structs/Pak.h +++ b/include/europa/structs/Pak.h @@ -53,20 +53,47 @@ namespace europa::structs { return sizeof(magic) + static_cast(headerSize); } - void Init(PakVersion ver) { - // clear any junk - memset(this, 0, sizeof(PakHeader)); + /** + * Initialize this header (used when writing). + */ + void Init(PakVersion ver) noexcept { + // clear any junk + memset(this, 0, sizeof(PakHeader)); - // Copy important things. - std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC)); - headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1); + // Copy important things. + std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC)); - // Set archive version - version = ver; - } + // Set proper header size. + headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1); + + // Set archive version + version = ver; + } + + [[nodiscard]] bool Valid() const noexcept { + // Magic must match. + if(std::strcmp(magic, VALID_MAGIC) != 0) + return false; + + using enum PakVersion; + + // Version must match ones we support, + // otherwise it's invalid. + switch(version) { + case Starfighter: + case Ver2: + break; + + default: + return false; + } + + // Header is okay. + return true; + } }; - // A Toc entry (without string. Needs to be read in seperately) + // A Toc entry (without string. Needs to be read in separately) struct [[gnu::packed]] PakTocEntry { u32 offset; u32 size; @@ -76,8 +103,8 @@ namespace europa::structs { u32 unk3; }; - static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!"); - static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid"); + static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!"); + static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid when writing archives."); static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!"); } // namespace europa::structs diff --git a/include/europa/util/FixedString.h b/include/europa/util/FixedString.h index cfa3116..d40c4eb 100644 --- a/include/europa/util/FixedString.h +++ b/include/europa/util/FixedString.h @@ -16,12 +16,12 @@ namespace europa::util { /** * A compile-time string. Usable as a C++20 cNTTP. */ - template + template struct FixedString { - char buf[N + 1]{}; + char buf[N + 1] {}; constexpr FixedString(const char* s) { // NOLINT - for (unsigned i = 0; i != N; ++i) + for(unsigned i = 0; i != N; ++i) buf[i] = s[i]; } @@ -34,9 +34,9 @@ namespace europa::util { } }; - template + template FixedString(char const (&)[N]) -> FixedString; -} +} // namespace europa::util #endif // EUROPA_UTIL_FIXEDSTRING_H diff --git a/include/europa/util/FourCC.h b/include/europa/util/FourCC.h index e91fc23..48e6461 100644 --- a/include/europa/util/FourCC.h +++ b/include/europa/util/FourCC.h @@ -9,17 +9,17 @@ #ifndef EUROPA_FOURCC_H #define EUROPA_FOURCC_H -#include - #include +#include + namespace europa::util { /** * A multi-endian, compile-time FourCC generator. * You love to see it. */ - template + template consteval std::uint32_t FourCC() { static_assert(fccString.Length() == 4, "Provided string is not a FourCC"); @@ -36,6 +36,6 @@ namespace europa::util { return 0xffffffff; } -} +} // namespace europa::util #endif // EUROPA_FOURCC_H diff --git a/src/libeuropa/CMakeLists.txt b/src/libeuropa/CMakeLists.txt index d5697b1..23942be 100644 --- a/src/libeuropa/CMakeLists.txt +++ b/src/libeuropa/CMakeLists.txt @@ -23,4 +23,4 @@ set_target_properties(libeuropa PROPERTIES # Projects which libeuropa depends on target_link_libraries(libeuropa PUBLIC pixel::libpixel -) + ) diff --git a/src/libeuropa/io/PakFile.cpp b/src/libeuropa/io/PakFile.cpp index 5254d50..5adda43 100644 --- a/src/libeuropa/io/PakFile.cpp +++ b/src/libeuropa/io/PakFile.cpp @@ -10,25 +10,24 @@ namespace europa::io { - const PakFile::DataType& PakFile::GetData() const { - return data; - } + const PakFile::DataType& PakFile::GetData() const { + return data; + } - const structs::PakTocEntry& PakFile::GetTOCEntry() const { - return tocData; - } + const structs::PakTocEntry& PakFile::GetTOCEntry() const { + return tocData; + } + structs::PakTocEntry& PakFile::GetTOCEntry() { + return tocData; + } - structs::PakTocEntry& PakFile::GetTOCEntry() { - return tocData; - } + void PakFile::SetData(PakFile::DataType&& newData) { + data = std::move(newData); + } - void PakFile::SetData(PakFile::DataType&& newData) { - data = std::move(newData); - } + void PakFile::FillTOCEntry() { + tocData.size = static_cast(data.size()); + } - void PakFile::FillTOCEntry() { - tocData.size = static_cast(data.size()); - } - -} \ No newline at end of file +} // namespace europa::io \ No newline at end of file diff --git a/src/libeuropa/io/PakReader.cpp b/src/libeuropa/io/PakReader.cpp index e6b7b39..ba0a531 100644 --- a/src/libeuropa/io/PakReader.cpp +++ b/src/libeuropa/io/PakReader.cpp @@ -20,57 +20,51 @@ namespace europa::io { } void PakReader::ReadData() { - auto ReadHeader = [&]() { - header = impl::ReadStreamType(stream); - }; + header = impl::ReadStreamType(stream); - auto ReadTocEntry = [&]() { + if(!header.Valid()) { + invalid = true; + return; + } + + // Read the archive TOC + stream.seekg(header.tocOffset, std::istream::beg); + for(auto i = 0; i < header.fileCount; ++i) { // The first part of the TOC entry is a VLE string, // which we don't store inside the type (because we can't) // // Read this in first. auto filename = impl::ReadPString(stream); - files[filename].GetTOCEntry() = impl::ReadStreamType(stream); - }; - - ReadHeader(); - - // Validate the archive header - if(std::strcmp(header.magic, structs::PakHeader::VALID_MAGIC) != 0) { - invalid = true; - return; - } - - switch(header.version) { - case structs::PakVersion::Starfighter: - case structs::PakVersion::Ver2: - break; - - default: - invalid = true; - return; - } - - stream.seekg(header.tocOffset, std::istream::beg); - - // Read the archive TOC - for(auto i = 0; i < header.fileCount; ++i) - ReadTocEntry(); - - // Read all file data in - for(auto& [filename, file] : files) { - auto& toc = file.GetTOCEntry(); - file.data.resize(toc.size); - - stream.seekg(toc.offset, std::istream::beg); - stream.read(reinterpret_cast(&file.data[0]), toc.size); } } - const std::unordered_map& PakReader::GetFiles() const { + void PakReader::ReadFiles() { + for(auto& [filename, file] : files) + ReadFile(filename); + } + + void PakReader::ReadFile(const std::string& file) { + auto& fileObject = files[file]; + + // This file was already read in, or has data + // the user may not want to overwrite. + if(!fileObject.data.empty()) + return; + + auto& toc = fileObject.GetTOCEntry(); + fileObject.data.resize(toc.size); + + stream.seekg(toc.offset, std::istream::beg); + stream.read(reinterpret_cast(&fileObject.data[0]), toc.size); + } + + PakReader::MapType& PakReader::GetFiles() { return files; } + const PakReader::MapType& PakReader::GetFiles() const { + return files; + } } // namespace europa::io \ No newline at end of file diff --git a/src/libeuropa/io/PakWriter.cpp b/src/libeuropa/io/PakWriter.cpp index dfadcc3..fa609b6 100644 --- a/src/libeuropa/io/PakWriter.cpp +++ b/src/libeuropa/io/PakWriter.cpp @@ -7,56 +7,58 @@ // #include + #include #include "StreamUtils.h" namespace europa::io { + void PakWriter::Init(structs::PakVersion version) { + // for now. + pakHeader.Init(version); + } - void PakWriter::Init() { - // for now. - pakHeader.Init(structs::PakVersion::Starfighter); - } + std::unordered_map& PakWriter::GetFiles() { + return archiveFiles; + } - void PakWriter::AddFile(const std::string &path, const PakFile& data) { - archiveFiles[path] = data; - } + void PakWriter::Write(std::ostream& os) { + // Set up the header a bit more... + pakHeader.fileCount = archiveFiles.size(); - void PakWriter::RemoveFile(const std::string &path) { - archiveFiles.erase(path); - } + // Leave space for the header + os.seekp(sizeof(structs::PakHeader), std::ostream::beg); - void PakWriter::Write(std::ostream &os) { - // Set up the header a bit more... - pakHeader.fileCount = archiveFiles.size(); + // Seek forwards for version 2 PAKs, as the only + // difference seems to be + if(pakHeader.version == structs::PakVersion::Ver2) { + os.seekp(6, std::ostream::cur); + } - // Leave space for the header - os.seekp(sizeof(structs::PakHeader), std::ostream::beg); + // Write file data + for(auto& [filename, file] : archiveFiles) { + file.GetTOCEntry().offset = os.tellp(); + os.write(reinterpret_cast(file.GetData().data()), file.GetData().size()); + } - // Write file data - for (auto &[filename, file]: archiveFiles) { - file.GetTOCEntry().offset = os.tellp(); - os.write(reinterpret_cast(file.GetData().data()), file.GetData().size()); - } + pakHeader.tocOffset = os.tellp(); - pakHeader.tocOffset = os.tellp(); + // Write the TOC + for(auto& [filename, file] : archiveFiles) { + file.FillTOCEntry(); - // Write the TOC - for (auto &[filename, file]: archiveFiles) { - file.FillTOCEntry(); + // Write the pstring + os.put(static_cast(filename.length() + 1)); + for(const auto c : filename) + os.put(c); + os.put('\0'); - // Write the pstring - os.put(static_cast(filename.length() + 1)); - for (const auto c: filename) - os.put(c); - os.put('\0'); + impl::WriteStreamType(os, file.GetTOCEntry()); + } - impl::WriteStreamType(os, file.GetTOCEntry()); - } + os.seekp(0, std::ostream::beg); + impl::WriteStreamType(os, pakHeader); + } - os.seekp(0, std::ostream::beg); - impl::WriteStreamType(os, pakHeader); - } - -} \ No newline at end of file +} // namespace europa::io \ No newline at end of file diff --git a/src/libeuropa/io/StreamUtils.cpp b/src/libeuropa/io/StreamUtils.cpp index 5467158..054aeff 100644 --- a/src/libeuropa/io/StreamUtils.cpp +++ b/src/libeuropa/io/StreamUtils.cpp @@ -19,7 +19,6 @@ namespace europa::io::impl { is.read(&buffer[0], size); } - void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size) { os.write(&buffer[0], buffer_size); } diff --git a/src/libeuropa/io/StreamUtils.h b/src/libeuropa/io/StreamUtils.h index 66c1aa1..a64357d 100644 --- a/src/libeuropa/io/StreamUtils.h +++ b/src/libeuropa/io/StreamUtils.h @@ -17,7 +17,7 @@ namespace europa::io::impl { namespace detail { void ReadStreamTypeImpl(std::istream& is, char* buffer, std::size_t size); void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size); - } + } // namespace detail // This is lame. But it works :) template diff --git a/src/libeuropa/io/YatfReader.cpp b/src/libeuropa/io/YatfReader.cpp index c1b6118..29be3bf 100644 --- a/src/libeuropa/io/YatfReader.cpp +++ b/src/libeuropa/io/YatfReader.cpp @@ -7,10 +7,11 @@ // #include -#include "StreamUtils.h" #include +#include "StreamUtils.h" + namespace europa::io { YatfReader::YatfReader(std::istream& is) @@ -56,6 +57,4 @@ namespace europa::io { return header; } - - -} \ No newline at end of file +} // namespace europa::io \ No newline at end of file diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 5da379a..f81b4ba 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,6 +1,8 @@ add_executable(europa_pack_extractor europa_pack_extractor.cpp) -target_link_libraries(europa_pack_extractor PUBLIC libeuropa) +target_link_libraries(europa_pack_extractor PUBLIC libeuropa + indicators::indicators + ) set_target_properties(europa_pack_extractor PROPERTIES CXX_STANDARD 20 diff --git a/src/tools/europa_pack_extractor.cpp b/src/tools/europa_pack_extractor.cpp index 0b67fc9..130bafb 100644 --- a/src/tools/europa_pack_extractor.cpp +++ b/src/tools/europa_pack_extractor.cpp @@ -10,6 +10,8 @@ #include #include +#include +#include #include namespace fs = std::filesystem; @@ -38,6 +40,19 @@ int main(int argc, char** argv) { return 1; } + indicators::ProgressBar progress { + indicators::option::BarWidth { 50 }, + indicators::option::ForegroundColor { indicators::Color::green }, + indicators::option::MaxProgress { reader.GetFiles().size() }, + indicators::option::ShowPercentage { true }, + indicators::option::ShowElapsedTime { true }, + indicators::option::ShowRemainingTime { true }, + + indicators::option::PrefixText { "Extracting archive " } + }; + + indicators::show_console_cursor(false); + for(auto& [filename, file] : reader.GetFiles()) { auto nameCopy = filename; @@ -50,11 +65,15 @@ int main(int argc, char** argv) { } #endif + progress.set_option(indicators::option::PostfixText { filename }); + auto outpath = (baseDirectory / nameCopy); if(!fs::exists(outpath.parent_path())) fs::create_directories(outpath.parent_path()); + reader.ReadFile(filename); + std::ofstream ofs(outpath.string(), std::ofstream::binary); if(!ofs) { @@ -63,10 +82,9 @@ int main(int argc, char** argv) { } ofs.write(reinterpret_cast(file.GetData().data()), static_cast(file.GetTOCEntry().size)); - ofs.close(); - - std::cout << "Wrote \"" << outpath.string() << "\" to disk.\n"; + progress.tick(); } + indicators::show_console_cursor(true); return 0; } \ No newline at end of file diff --git a/src/tools/paktest.cpp b/src/tools/paktest.cpp index cf61d28..7944ee1 100644 --- a/src/tools/paktest.cpp +++ b/src/tools/paktest.cpp @@ -11,32 +11,32 @@ #include #include -#include #include +#include int main(int argc, char** argv) { - std::ifstream ifs(argv[1], std::ifstream::binary); - std::ofstream ofs("new_archive.pak", std::ofstream::binary); + std::ifstream ifs(argv[1], std::ifstream::binary); + std::ofstream ofs("new_archive.pak", std::ofstream::binary); - europa::io::PakWriter writer; + europa::io::PakWriter writer; - writer.Init(); + writer.Init(europa::structs::PakVersion::Ver2); - // Read pak data and vomit it into the writer. - // This will temporarily consume 2x the memory (so about 240mb for the biggest paks I've seen), - // but the writer will contain the first copy, - // until it's cleared. - { - europa::io::PakReader reader(ifs); - reader.ReadData(); + // Read pak data and vomit it into the writer. + // This will temporarily consume 2x the memory (so about 240mb for the biggest paks I've seen), + // but the writer will contain the first copy, + // until it's cleared. + { + europa::io::PakReader reader(ifs); + reader.ReadData(); - for (auto &[filename, file]: reader.GetFiles()) { - writer.AddFile(filename, file); - } - } + for(auto& [filename, file] : reader.GetFiles()) { + writer.AddFile(filename, file); + } + } - writer.Write(ofs); + writer.Write(ofs); - std::cout << "Wrote regurgitated archive to new.pak!\n"; - return 0; + std::cout << "Wrote regurgitated archive to new.pak!\n"; + return 0; } \ No newline at end of file