From 5d03f49e211f9dacac346bfe1ccd4bf9a284a347 Mon Sep 17 00:00:00 2001 From: modeco80 Date: Mon, 5 Sep 2022 19:59:46 -0500 Subject: [PATCH] Add possibly not WIP package writer Don't have a tool for building "fresh" packages yet, but the "paktest" tests regurgitating a pak into a new pak file. Usage is: ./paktest [pak file] it will write to "new.pak" always, which meh. It's just a test utility. --- include/europa/io/PakFile.h | 55 ++++++++++++++++++++++++ include/europa/io/PakReader.h | 36 +++++----------- include/europa/io/PakWriter.h | 47 ++++++++++++++++++++ include/europa/io/YatfReader.h | 8 ++-- include/europa/structs/ImHexAdapter.h | 18 ++++---- include/europa/structs/Pak.h | 25 ++++++++--- include/europa/structs/Yatf.h | 5 +-- include/europa/util/FixedString.h | 8 ++-- include/europa/util/FourCC.h | 5 +-- src/libeuropa/CMakeLists.txt | 8 +++- src/libeuropa/io/PakFile.cpp | 34 +++++++++++++++ src/libeuropa/io/PakReader.cpp | 49 ++++++++------------- src/libeuropa/io/PakWriter.cpp | 62 +++++++++++++++++++++++++++ src/libeuropa/io/StreamUtils.cpp | 5 +++ src/libeuropa/io/StreamUtils.h | 15 ++++++- src/libeuropa/io/YatfReader.cpp | 2 - src/tools/CMakeLists.txt | 11 ++++- src/tools/europa_pack_extractor.cpp | 6 +-- src/tools/paktest.cpp | 42 ++++++++++++++++++ src/tools/texdump.cpp | 9 ++-- 20 files changed, 351 insertions(+), 99 deletions(-) create mode 100644 include/europa/io/PakFile.h create mode 100644 include/europa/io/PakWriter.h create mode 100644 src/libeuropa/io/PakFile.cpp create mode 100644 src/libeuropa/io/PakWriter.cpp create mode 100644 src/tools/paktest.cpp diff --git a/include/europa/io/PakFile.h b/include/europa/io/PakFile.h new file mode 100644 index 0000000..71aab29 --- /dev/null +++ b/include/europa/io/PakFile.h @@ -0,0 +1,55 @@ +// +// EuropaTools +// +// (C) 2021-2022 modeco80 +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// + +#ifndef EUROPA_IO_PAKFILE_H +#define EUROPA_IO_PAKFILE_H + +#include +#include + +#include + +namespace europa::io { + + struct PakReader; + struct PakWriter; + + struct PakFile { + + using DataType = std::vector; + + /** + * Get the file data. + */ + [[nodiscard]] const DataType& GetData() const; + + /** + * Get the TOC entry responsible. + */ + [[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const; + + void SetData(DataType&& data); + + structs::PakTocEntry& GetTOCEntry(); + + void FillTOCEntry(); + + private: + friend PakReader; + friend PakWriter; + + + std::vector data; + structs::PakTocEntry tocData; + }; + + +} + + +#endif //EUROPA_IO_PAKFILE_H diff --git a/include/europa/io/PakReader.h b/include/europa/io/PakReader.h index a043382..15f7919 100644 --- a/include/europa/io/PakReader.h +++ b/include/europa/io/PakReader.h @@ -9,53 +9,37 @@ #ifndef EUROPA_IO_PAKREADER_H #define EUROPA_IO_PAKREADER_H -#include -#include -#include -#include +#include #include +#include +#include +#include + namespace europa::io { - struct PakReader { - struct File { - File(std::vector&& data, structs::PakTocEntry& tocData); - - const std::vector& GetData() const; - - const structs::PakTocEntry& GetTOCEntry() const; - - private: - std::vector data; - structs::PakTocEntry tocData; - }; - explicit PakReader(std::istream& is); void ReadData(); - bool Invalid() const { return invalid; } - const std::unordered_map& GetFiles() const; + const std::unordered_map& GetFiles() const; private: std::istream& stream; - bool invalid{false}; + bool invalid { false }; - structs::PakHeader header{}; + structs::PakHeader header {}; - std::unordered_map tocData; - std::unordered_map files; + std::unordered_map files; }; - - -} +} // namespace europa::io #endif // EUROPA_IO_PAKREADER_H diff --git a/include/europa/io/PakWriter.h b/include/europa/io/PakWriter.h new file mode 100644 index 0000000..e820a2f --- /dev/null +++ b/include/europa/io/PakWriter.h @@ -0,0 +1,47 @@ +// +// EuropaTools +// +// (C) 2021-2022 modeco80 +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// + +#ifndef EUROPA_IO_PAKWRITER_H +#define EUROPA_IO_PAKWRITER_H + + +#include + +#include + + +#include +#include + +namespace europa::io { + + /** + * Writer for package files. + */ + struct PakWriter { + + void Init(); + + void AddFile(const std::string& path, const PakFile& data); + + void RemoveFile(const std::string& path); + + /** + * Write the resulting archive to the given output stream. + */ + void Write(std::ostream& os); + + private: + structs::PakHeader pakHeader{}; + std::unordered_map archiveFiles; + }; + + +} + +#endif // EUROPA_IO_PAKWRITER_H diff --git a/include/europa/io/YatfReader.h b/include/europa/io/YatfReader.h index 57cfcc8..e6637d1 100644 --- a/include/europa/io/YatfReader.h +++ b/include/europa/io/YatfReader.h @@ -9,10 +9,11 @@ #ifndef EUROPA_IO_YATFREADER_H #define EUROPA_IO_YATFREADER_H -#include #include #include +#include + namespace europa::io { /** @@ -35,8 +36,7 @@ namespace europa::io { private: std::istream& stream; - bool invalid {false}; - + bool invalid { false }; structs::YatfHeader header; @@ -46,6 +46,6 @@ namespace europa::io { pixel::RgbaImage image; }; -} +} // namespace europa::io #endif // EUROPA_IO_YATFREADER_H diff --git a/include/europa/structs/ImHexAdapter.h b/include/europa/structs/ImHexAdapter.h index 118b56a..f8fc3f5 100644 --- a/include/europa/structs/ImHexAdapter.h +++ b/include/europa/structs/ImHexAdapter.h @@ -14,17 +14,17 @@ #include namespace europa::structs { - using u8 = std::uint8_t; - using s8 = std::int8_t; + using u8 = std::uint8_t; + using s8 = std::int8_t; - using u16 = std::uint16_t; - using s16 = std::int16_t; + using u16 = std::uint16_t; + using s16 = std::int16_t; - using u32 = std::uint32_t; - using s32 = std::int32_t; + using u32 = std::uint32_t; + using s32 = std::int32_t; - using u64 = std::uint64_t; - using s64 = std::int64_t; -} + using u64 = std::uint64_t; + using s64 = std::int64_t; +} // namespace europa::structs #endif // EUROPA_STRUCTS_IMHEXADAPTER_H diff --git a/include/europa/structs/Pak.h b/include/europa/structs/Pak.h index a3e9f76..4151c92 100644 --- a/include/europa/structs/Pak.h +++ b/include/europa/structs/Pak.h @@ -9,10 +9,11 @@ #ifndef EUROPA_STRUCTS_PAK_H #define EUROPA_STRUCTS_PAK_H -#include - #include +#include +#include + namespace europa::structs { enum class PakVersion : u16 { @@ -20,7 +21,7 @@ namespace europa::structs { Ver2 = 0x5 }; - struct [[gnu::packed]] PakHeader { + struct [[gnu::packed]] PakHeader { constexpr static const char VALID_MAGIC[16] = "Europa Packfile"; char magic[16]; // "Europa Packfile\0" @@ -51,6 +52,18 @@ namespace europa::structs { [[nodiscard]] constexpr std::size_t RealHeaderSize() const { return sizeof(magic) + static_cast(headerSize); } + + void Init(PakVersion ver) { + // clear any junk + memset(this, 0, sizeof(PakHeader)); + + // Copy important things. + std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC)); + headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1); + + // Set archive version + version = ver; + } }; // A Toc entry (without string. Needs to be read in seperately) @@ -63,10 +76,10 @@ namespace europa::structs { u32 unk3; }; - - static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!"); + static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!"); + static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid"); static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!"); -} +} // namespace europa::structs #endif // EUROPA_STRUCTS_PAK_H diff --git a/include/europa/structs/Yatf.h b/include/europa/structs/Yatf.h index 8da9d31..fd5a6e9 100644 --- a/include/europa/structs/Yatf.h +++ b/include/europa/structs/Yatf.h @@ -9,15 +9,14 @@ #ifndef EUROPA_STRUCTS_YATF_H #define EUROPA_STRUCTS_YATF_H +#include #include -#include #include namespace europa::structs { struct [[gnu::packed]] YatfHeader { - constexpr static u32 TextureFlag_Unknown = 0x1; /** @@ -47,6 +46,6 @@ namespace europa::structs { } }; -} +} // namespace europa::structs #endif // EUROPA_STRUCTS_YATF_H diff --git a/include/europa/util/FixedString.h b/include/europa/util/FixedString.h index 85813a3..cfa3116 100644 --- a/include/europa/util/FixedString.h +++ b/include/europa/util/FixedString.h @@ -1,10 +1,9 @@ // -// SSX 3 Lobby Server +// EuropaTools // // (C) 2021-2022 modeco80 // -// This file is licensed under the GNU General Public License Version 3. -// Text is provided in LICENSE. +// SPDX-License-Identifier: LGPL-3.0-or-later // #ifndef EUROPA_UTIL_FIXEDSTRING_H @@ -14,6 +13,9 @@ namespace europa::util { + /** + * A compile-time string. Usable as a C++20 cNTTP. + */ template struct FixedString { char buf[N + 1]{}; diff --git a/include/europa/util/FourCC.h b/include/europa/util/FourCC.h index df64635..e91fc23 100644 --- a/include/europa/util/FourCC.h +++ b/include/europa/util/FourCC.h @@ -1,10 +1,9 @@ // -// SSX 3 Lobby Server +// EuropaTools // // (C) 2021-2022 modeco80 // -// This file is licensed under the GNU General Public License Version 3. -// Text is provided in LICENSE. +// SPDX-License-Identifier: LGPL-3.0-or-later // #ifndef EUROPA_FOURCC_H diff --git a/src/libeuropa/CMakeLists.txt b/src/libeuropa/CMakeLists.txt index 2448907..d5697b1 100644 --- a/src/libeuropa/CMakeLists.txt +++ b/src/libeuropa/CMakeLists.txt @@ -1,8 +1,15 @@ add_library(libeuropa + # Implementation details io/StreamUtils.cpp + + # Pak IO + io/PakFile.cpp io/PakReader.cpp + io/PakWriter.cpp + + # Yatf IO io/YatfReader.cpp ) @@ -15,6 +22,5 @@ set_target_properties(libeuropa PROPERTIES # Projects which libeuropa depends on target_link_libraries(libeuropa PUBLIC - pixel::libpixel ) diff --git a/src/libeuropa/io/PakFile.cpp b/src/libeuropa/io/PakFile.cpp new file mode 100644 index 0000000..5254d50 --- /dev/null +++ b/src/libeuropa/io/PakFile.cpp @@ -0,0 +1,34 @@ +// +// EuropaTools +// +// (C) 2021-2022 modeco80 +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// + +#include + +namespace europa::io { + + const PakFile::DataType& PakFile::GetData() const { + return data; + } + + const structs::PakTocEntry& PakFile::GetTOCEntry() const { + return tocData; + } + + + structs::PakTocEntry& PakFile::GetTOCEntry() { + return tocData; + } + + void PakFile::SetData(PakFile::DataType&& newData) { + data = std::move(newData); + } + + void PakFile::FillTOCEntry() { + tocData.size = static_cast(data.size()); + } + +} \ No newline at end of file diff --git a/src/libeuropa/io/PakReader.cpp b/src/libeuropa/io/PakReader.cpp index a3575f5..e6b7b39 100644 --- a/src/libeuropa/io/PakReader.cpp +++ b/src/libeuropa/io/PakReader.cpp @@ -31,19 +31,26 @@ namespace europa::io { // Read this in first. auto filename = impl::ReadPString(stream); - // Then read in the rest. - tocData[filename] = impl::ReadStreamType(stream); + files[filename].GetTOCEntry() = impl::ReadStreamType(stream); }; ReadHeader(); - // Validate the archive magic + // Validate the archive header if(std::strcmp(header.magic, structs::PakHeader::VALID_MAGIC) != 0) { invalid = true; return; } - // std::cout << (int)header.headerSize << " version " << (int)header.version << '\n'; + switch(header.version) { + case structs::PakVersion::Starfighter: + case structs::PakVersion::Ver2: + break; + + default: + invalid = true; + return; + } stream.seekg(header.tocOffset, std::istream::beg); @@ -51,39 +58,19 @@ namespace europa::io { for(auto i = 0; i < header.fileCount; ++i) ReadTocEntry(); - // for(auto& [ filename, data ] : tocData) { - // std::cout << filename << " offset " << data.offset << " size " << data.size << '\n'; - // } + // Read all file data in + for(auto& [filename, file] : files) { + auto& toc = file.GetTOCEntry(); + file.data.resize(toc.size); - // Read all files in - for(auto& [filename, data] : tocData) { - std::vector dataBuffer; - - dataBuffer.resize(data.size); - stream.seekg(data.offset, std::istream::beg); - - stream.read(reinterpret_cast(&dataBuffer[0]), data.size); - - File file(std::move(dataBuffer), data); - files.insert_or_assign(filename, file); + stream.seekg(toc.offset, std::istream::beg); + stream.read(reinterpret_cast(&file.data[0]), toc.size); } } - const std::unordered_map& PakReader::GetFiles() const { + const std::unordered_map& PakReader::GetFiles() const { return files; } - PakReader::File::File(std::vector&& data, structs::PakTocEntry& tocData) - : data(std::move(data)), - tocData(tocData) { - } - - const std::vector& PakReader::File::GetData() const { - return data; - } - - const structs::PakTocEntry& PakReader::File::GetTOCEntry() const { - return tocData; - } } // namespace europa::io \ No newline at end of file diff --git a/src/libeuropa/io/PakWriter.cpp b/src/libeuropa/io/PakWriter.cpp new file mode 100644 index 0000000..dfadcc3 --- /dev/null +++ b/src/libeuropa/io/PakWriter.cpp @@ -0,0 +1,62 @@ +// +// EuropaTools +// +// (C) 2021-2022 modeco80 +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// + +#include +#include + +#include "StreamUtils.h" + +namespace europa::io { + + + void PakWriter::Init() { + // for now. + pakHeader.Init(structs::PakVersion::Starfighter); + } + + void PakWriter::AddFile(const std::string &path, const PakFile& data) { + archiveFiles[path] = data; + } + + void PakWriter::RemoveFile(const std::string &path) { + archiveFiles.erase(path); + } + + void PakWriter::Write(std::ostream &os) { + // Set up the header a bit more... + pakHeader.fileCount = archiveFiles.size(); + + // Leave space for the header + os.seekp(sizeof(structs::PakHeader), std::ostream::beg); + + // Write file data + for (auto &[filename, file]: archiveFiles) { + file.GetTOCEntry().offset = os.tellp(); + os.write(reinterpret_cast(file.GetData().data()), file.GetData().size()); + } + + pakHeader.tocOffset = os.tellp(); + + // Write the TOC + for (auto &[filename, file]: archiveFiles) { + file.FillTOCEntry(); + + // Write the pstring + os.put(static_cast(filename.length() + 1)); + for (const auto c: filename) + os.put(c); + os.put('\0'); + + impl::WriteStreamType(os, file.GetTOCEntry()); + } + + os.seekp(0, std::ostream::beg); + impl::WriteStreamType(os, pakHeader); + } + +} \ No newline at end of file diff --git a/src/libeuropa/io/StreamUtils.cpp b/src/libeuropa/io/StreamUtils.cpp index 1c9eb20..5467158 100644 --- a/src/libeuropa/io/StreamUtils.cpp +++ b/src/libeuropa/io/StreamUtils.cpp @@ -19,6 +19,11 @@ namespace europa::io::impl { is.read(&buffer[0], size); } + + void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size) { + os.write(&buffer[0], buffer_size); + } + } // namespace detail std::string ReadZeroTerminatedString(std::istream& is) { diff --git a/src/libeuropa/io/StreamUtils.h b/src/libeuropa/io/StreamUtils.h index 6800d0b..66c1aa1 100644 --- a/src/libeuropa/io/StreamUtils.h +++ b/src/libeuropa/io/StreamUtils.h @@ -16,9 +16,9 @@ namespace europa::io::impl { namespace detail { void ReadStreamTypeImpl(std::istream& is, char* buffer, std::size_t size); + void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size); } - // This is lame. But it works :) template constexpr T ReadStreamType(std::istream& is) { @@ -37,6 +37,19 @@ namespace europa::io::impl { return object; } + template + constexpr void WriteStreamType(std::ostream& os, const T& object) { + // Absolutely UB. + union Hack { + const T* t; + const char* c; + } address { + .t = &object + }; + + detail::WriteStreamTypeImpl(os, address.c, sizeof(T)); + } + std::string ReadZeroTerminatedString(std::istream& is); std::string ReadPString(std::istream& is); diff --git a/src/libeuropa/io/YatfReader.cpp b/src/libeuropa/io/YatfReader.cpp index 654dd15..c1b6118 100644 --- a/src/libeuropa/io/YatfReader.cpp +++ b/src/libeuropa/io/YatfReader.cpp @@ -29,7 +29,6 @@ namespace europa::io { void YatfReader::ReadImage() { if(header.flags & structs::YatfHeader::TextureFlag_NoPalette) { image.Resize({ static_cast(header.width), static_cast(header.height) }); - stream.read(reinterpret_cast(image.GetBuffer()), (header.width * header.height) * sizeof(pixel::RgbaColor)); } else { pixel::RgbaColor palette[256]; @@ -37,7 +36,6 @@ namespace europa::io { // NB: sizeof() does pre-multiplication, so it's 100% ok for us to do this. stream.read(reinterpret_cast(&palette[0]), sizeof(palette)); - stream.read(reinterpret_cast(&tempBuffer[0]), tempBuffer.size()); image.Resize({ static_cast(header.width), static_cast(header.height) }); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index c3de1c3..5da379a 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -15,4 +15,13 @@ target_link_libraries(texdump PUBLIC libeuropa) set_target_properties(texdump PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON - ) \ No newline at end of file + ) + +add_executable(paktest paktest.cpp) + +target_link_libraries(paktest PUBLIC libeuropa) + +set_target_properties(paktest PROPERTIES + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + ) diff --git a/src/tools/europa_pack_extractor.cpp b/src/tools/europa_pack_extractor.cpp index c025b1c..0b67fc9 100644 --- a/src/tools/europa_pack_extractor.cpp +++ b/src/tools/europa_pack_extractor.cpp @@ -6,12 +6,12 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // +#include + #include #include #include -#include - namespace fs = std::filesystem; int main(int argc, char** argv) { @@ -38,7 +38,7 @@ int main(int argc, char** argv) { return 1; } - for(auto& [ filename, file ] : reader.GetFiles()) { + for(auto& [filename, file] : reader.GetFiles()) { auto nameCopy = filename; #ifndef _WIN32 diff --git a/src/tools/paktest.cpp b/src/tools/paktest.cpp new file mode 100644 index 0000000..cf61d28 --- /dev/null +++ b/src/tools/paktest.cpp @@ -0,0 +1,42 @@ +// +// EuropaTools +// +// (C) 2021-2022 modeco80 +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// + +// A test utility to regurgitate a pak. + +#include +#include + +#include +#include + +int main(int argc, char** argv) { + std::ifstream ifs(argv[1], std::ifstream::binary); + std::ofstream ofs("new_archive.pak", std::ofstream::binary); + + europa::io::PakWriter writer; + + writer.Init(); + + // Read pak data and vomit it into the writer. + // This will temporarily consume 2x the memory (so about 240mb for the biggest paks I've seen), + // but the writer will contain the first copy, + // until it's cleared. + { + europa::io::PakReader reader(ifs); + reader.ReadData(); + + for (auto &[filename, file]: reader.GetFiles()) { + writer.AddFile(filename, file); + } + } + + writer.Write(ofs); + + std::cout << "Wrote regurgitated archive to new.pak!\n"; + return 0; +} \ No newline at end of file diff --git a/src/tools/texdump.cpp b/src/tools/texdump.cpp index aeec82e..139d517 100644 --- a/src/tools/texdump.cpp +++ b/src/tools/texdump.cpp @@ -6,15 +6,13 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // +#include +#include #include #include #include -#include - -#include - namespace fs = std::filesystem; int main(int argc, char** argv) { if(argc != 2) { @@ -38,8 +36,7 @@ int main(int argc, char** argv) { reader.ReadImage(); - - pixel::ImageWriter writer{}; + pixel::ImageWriter writer {}; auto outPath = fs::path(argv[1]).replace_extension(".png");