Add possibly not WIP package writer

Don't have a tool for building "fresh" packages yet, but the "paktest" tests regurgitating a pak into a new pak file.

Usage is:

./paktest [pak file]

it will write to "new.pak" always, which meh. It's just a test utility.

include/europa/io/PakFile.h

@@ -0,0 +1,55 @@
+//
+// EuropaTools
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+
+#ifndef EUROPA_IO_PAKFILE_H
+#define EUROPA_IO_PAKFILE_H
+
+#include <vector>
+#include <cstdint>
+
+#include <europa/structs/Pak.h>
+
+namespace europa::io {
+
+    struct PakReader;
+    struct PakWriter;
+
+    struct PakFile {
+
+        using DataType = std::vector<std::uint8_t>;
+
+        /**
+         * Get the file data.
+         */
+        [[nodiscard]] const DataType& GetData() const;
+
+        /**
+         * Get the TOC entry responsible.
+         */
+        [[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const;
+
+        void SetData(DataType&& data);
+
+        structs::PakTocEntry& GetTOCEntry();
+
+        void FillTOCEntry();
+
+    private:
+        friend PakReader;
+        friend PakWriter;
+
+
+        std::vector<std::uint8_t> data;
+        structs::PakTocEntry tocData;
+    };
+
+
+}
+
+
+#endif //EUROPA_IO_PAKFILE_H

include/europa/io/PakReader.h

@@ -9,53 +9,37 @@
 #ifndef EUROPA_IO_PAKREADER_H
 #define EUROPA_IO_PAKREADER_H
 
-#include <iosfwd>
+#include <europa/io/PakFile.h>
-#include <string>
-#include <unordered_map>
-#include <vector>
 
 #include <europa/structs/Pak.h>
 
+#include <iosfwd>
+#include <string>
+#include <unordered_map>
+
 namespace europa::io {
 
-
 	struct PakReader {
 
-		struct File {
-			File(std::vector<std::uint8_t>&& data, structs::PakTocEntry& tocData);
-
-			const std::vector<std::uint8_t>& GetData() const;
-
-			const structs::PakTocEntry& GetTOCEntry() const;
-
-		   private:
-			std::vector<std::uint8_t> data;
-			structs::PakTocEntry tocData;
-		};
-
 		explicit PakReader(std::istream& is);
 
 		void ReadData();
 
-
 		bool Invalid() const {
 			return invalid;
 		}
 
-		const std::unordered_map<std::string, File>& GetFiles() const;
+		const std::unordered_map<std::string, PakFile>& GetFiles() const;
 
 	   private:
 		std::istream& stream;
-		bool invalid{false};
+		bool invalid { false };
 
-		structs::PakHeader header{};
+		structs::PakHeader header {};
 
-		std::unordered_map<std::string, structs::PakTocEntry> tocData;
+		std::unordered_map<std::string, PakFile> files;
-		std::unordered_map<std::string, File> files;
 	};
 
-
+} // namespace europa::io
-
-}
 
 #endif // EUROPA_IO_PAKREADER_H

include/europa/io/PakWriter.h

@@ -0,0 +1,47 @@
+//
+// EuropaTools
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+
+#ifndef EUROPA_IO_PAKWRITER_H
+#define EUROPA_IO_PAKWRITER_H
+
+
+#include <europa/io/PakFile.h>
+
+#include <iosfwd>
+
+
+#include <string>
+#include <unordered_map>
+
+namespace europa::io {
+
+	/**
+	 * Writer for package files.
+	 */
+	struct PakWriter {
+
+        void Init();
+
+        void AddFile(const std::string& path, const PakFile& data);
+
+        void RemoveFile(const std::string& path);
+
+		/**
+		 * Write the resulting archive to the given output stream.
+		 */
+		void Write(std::ostream& os);
+
+    private:
+        structs::PakHeader pakHeader{};
+        std::unordered_map<std::string, PakFile> archiveFiles;
+	};
+
+
+}
+
+#endif // EUROPA_IO_PAKWRITER_H

include/europa/io/YatfReader.h

@@ -9,10 +9,11 @@
 #ifndef EUROPA_IO_YATFREADER_H
 #define EUROPA_IO_YATFREADER_H
 
-#include <iosfwd>
 #include <europa/structs/Yatf.h>
 #include <pixel/RgbaImage.h>
 
+#include <iosfwd>
+
 namespace europa::io {
 
 	/**
@@ -35,8 +36,7 @@ namespace europa::io {
 
 	   private:
 		std::istream& stream;
-		bool invalid {false};
+		bool invalid { false };
-
 
 		structs::YatfHeader header;
 
@@ -46,6 +46,6 @@ namespace europa::io {
 		pixel::RgbaImage image;
 	};
 
-}
+} // namespace europa::io
 
 #endif // EUROPA_IO_YATFREADER_H

include/europa/structs/ImHexAdapter.h

@@ -25,6 +25,6 @@ namespace europa::structs {
 
 	using u64 = std::uint64_t;
 	using s64 = std::int64_t;
-}
+} // namespace europa::structs
 
 #endif // EUROPA_STRUCTS_IMHEXADAPTER_H

include/europa/structs/Pak.h

@@ -9,10 +9,11 @@
 #ifndef EUROPA_STRUCTS_PAK_H
 #define EUROPA_STRUCTS_PAK_H
 
-#include <cstdint>
-
 #include <europa/structs/ImHexAdapter.h>
 
+#include <cstdint>
+#include <cstring>
+
 namespace europa::structs {
 
 	enum class PakVersion : u16 {
@@ -51,6 +52,18 @@ namespace europa::structs {
 		[[nodiscard]] constexpr std::size_t RealHeaderSize() const {
 			return sizeof(magic) + static_cast<std::size_t>(headerSize);
 		}
+
+        void Init(PakVersion ver) {
+            // clear any junk
+            memset(this, 0, sizeof(PakHeader));
+
+            // Copy important things.
+            std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC));
+            headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1);
+
+            // Set archive version
+            version = ver;
+        }
 	};
 
 	// A Toc entry (without string. Needs to be read in seperately)
@@ -63,10 +76,10 @@ namespace europa::structs {
 		u32 unk3;
 	};
 
-
     static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!");
+    static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid");
 	static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!");
 
-}
+} // namespace europa::structs
 
 #endif // EUROPA_STRUCTS_PAK_H

include/europa/structs/Yatf.h

@@ -9,15 +9,14 @@
 #ifndef EUROPA_STRUCTS_YATF_H
 #define EUROPA_STRUCTS_YATF_H
 
+#include <europa/structs/ImHexAdapter.h>
 #include <europa/util/FourCC.h>
 
-#include <europa/structs/ImHexAdapter.h>
 #include <type_traits>
 
 namespace europa::structs {
 
 	struct [[gnu::packed]] YatfHeader {
-
 		constexpr static u32 TextureFlag_Unknown = 0x1;
 
 		/**
@@ -47,6 +46,6 @@ namespace europa::structs {
 		}
 	};
 
-}
+} // namespace europa::structs
 
 #endif // EUROPA_STRUCTS_YATF_H

include/europa/util/FixedString.h

@@ -1,10 +1,9 @@
 //
-// SSX 3 Lobby Server
+// EuropaTools
 //
 // (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
 //
-// This file is licensed under the GNU General Public License Version 3.
+// SPDX-License-Identifier: LGPL-3.0-or-later
-// Text is provided in LICENSE.
 //
 
 #ifndef EUROPA_UTIL_FIXEDSTRING_H
@@ -14,6 +13,9 @@
 
 namespace europa::util {
 
+	/**
+	 * A compile-time string. Usable as a C++20 cNTTP.
+	 */
 	template<std::size_t N>
 	struct FixedString {
 		char buf[N + 1]{};

include/europa/util/FourCC.h

@@ -1,10 +1,9 @@
 //
-// SSX 3 Lobby Server
+// EuropaTools
 //
 // (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
 //
-// This file is licensed under the GNU General Public License Version 3.
+// SPDX-License-Identifier: LGPL-3.0-or-later
-// Text is provided in LICENSE.
 //
 
 #ifndef EUROPA_FOURCC_H

src/libeuropa/CMakeLists.txt

@@ -1,8 +1,15 @@
  
 
 add_library(libeuropa
+		# Implementation details
 		io/StreamUtils.cpp
+
+		# Pak IO
+		io/PakFile.cpp
 		io/PakReader.cpp
+		io/PakWriter.cpp
+
+		# Yatf IO
 		io/YatfReader.cpp
 		)
 
@@ -15,6 +22,5 @@ set_target_properties(libeuropa PROPERTIES
 
 # Projects which libeuropa depends on
 target_link_libraries(libeuropa PUBLIC
-
 		pixel::libpixel
 )

src/libeuropa/io/PakFile.cpp

@@ -0,0 +1,34 @@
+//
+// EuropaTools
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+
+#include <europa/io/PakFile.h>
+
+namespace europa::io {
+
+    const PakFile::DataType& PakFile::GetData() const {
+        return data;
+    }
+
+    const structs::PakTocEntry& PakFile::GetTOCEntry() const {
+        return tocData;
+    }
+
+
+    structs::PakTocEntry& PakFile::GetTOCEntry() {
+        return tocData;
+    }
+
+    void PakFile::SetData(PakFile::DataType&& newData) {
+        data = std::move(newData);
+    }
+
+    void PakFile::FillTOCEntry() {
+        tocData.size = static_cast<std::uint32_t>(data.size());
+    }
+
+}

src/libeuropa/io/PakReader.cpp

@@ -31,19 +31,26 @@ namespace europa::io {
 			// Read this in first.
 			auto filename = impl::ReadPString(stream);
 
-			// Then read in the rest.
+			files[filename].GetTOCEntry() = impl::ReadStreamType<structs::PakTocEntry>(stream);
-			tocData[filename] = impl::ReadStreamType<structs::PakTocEntry>(stream);
 		};
 
 		ReadHeader();
 
-		// Validate the archive magic
+		// Validate the archive header
 		if(std::strcmp(header.magic, structs::PakHeader::VALID_MAGIC) != 0) {
 			invalid = true;
 			return;
 		}
 
-		// std::cout << (int)header.headerSize << " version " << (int)header.version  << '\n';
+		switch(header.version) {
+			case structs::PakVersion::Starfighter:
+			case structs::PakVersion::Ver2:
+				break;
+
+			default:
+				invalid = true;
+				return;
+		}
 
 		stream.seekg(header.tocOffset, std::istream::beg);
 
@@ -51,39 +58,19 @@ namespace europa::io {
 		for(auto i = 0; i < header.fileCount; ++i)
 			ReadTocEntry();
 
-		//		for(auto& [ filename, data ] : tocData) {
+		// Read all file data in
-		//			std::cout << filename << " offset " << data.offset << " size " << data.size << '\n';
+		for(auto& [filename, file] : files) {
-		//		}
+			auto& toc = file.GetTOCEntry();
+			file.data.resize(toc.size);
 
-		// Read all files in
+			stream.seekg(toc.offset, std::istream::beg);
-		for(auto& [filename, data] : tocData) {
+			stream.read(reinterpret_cast<char*>(&file.data[0]), toc.size);
-			std::vector<std::uint8_t> dataBuffer;
-
-			dataBuffer.resize(data.size);
-			stream.seekg(data.offset, std::istream::beg);
-
-			stream.read(reinterpret_cast<char*>(&dataBuffer[0]), data.size);
-
-			File file(std::move(dataBuffer), data);
-			files.insert_or_assign(filename, file);
 		}
 	}
 
-	const std::unordered_map<std::string, PakReader::File>& PakReader::GetFiles() const {
+	const std::unordered_map<std::string, PakFile>& PakReader::GetFiles() const {
 		return files;
 	}
 
-	PakReader::File::File(std::vector<std::uint8_t>&& data, structs::PakTocEntry& tocData)
-		: data(std::move(data)),
-		  tocData(tocData) {
-	}
-
-	const std::vector<std::uint8_t>& PakReader::File::GetData() const {
-		return data;
-	}
-
-	const structs::PakTocEntry& PakReader::File::GetTOCEntry() const {
-		return tocData;
-	}
 
 } // namespace europa::io

src/libeuropa/io/PakWriter.cpp

@@ -0,0 +1,62 @@
+//
+// EuropaTools
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+
+#include <europa/io/PakWriter.h>
+#include <iostream>
+
+#include "StreamUtils.h"
+
+namespace europa::io {
+
+
+    void PakWriter::Init() {
+        // for now.
+        pakHeader.Init(structs::PakVersion::Starfighter);
+    }
+
+    void PakWriter::AddFile(const std::string &path, const PakFile& data) {
+        archiveFiles[path] = data;
+    }
+
+    void PakWriter::RemoveFile(const std::string &path) {
+        archiveFiles.erase(path);
+    }
+
+    void PakWriter::Write(std::ostream &os) {
+        // Set up the header a bit more...
+        pakHeader.fileCount = archiveFiles.size();
+
+        // Leave space for the header
+        os.seekp(sizeof(structs::PakHeader), std::ostream::beg);
+
+        // Write file data
+        for (auto &[filename, file]: archiveFiles) {
+            file.GetTOCEntry().offset = os.tellp();
+            os.write(reinterpret_cast<const char *>(file.GetData().data()), file.GetData().size());
+        }
+
+        pakHeader.tocOffset = os.tellp();
+
+        // Write the TOC
+        for (auto &[filename, file]: archiveFiles) {
+            file.FillTOCEntry();
+
+            // Write the pstring
+            os.put(static_cast<char>(filename.length() + 1));
+            for (const auto c: filename)
+                os.put(c);
+            os.put('\0');
+
+            impl::WriteStreamType(os, file.GetTOCEntry());
+        }
+
+        os.seekp(0, std::ostream::beg);
+        impl::WriteStreamType(os, pakHeader);
+    }
+
+}

src/libeuropa/io/StreamUtils.cpp

@@ -19,6 +19,11 @@ namespace europa::io::impl {
 			is.read(&buffer[0], size);
 		}
 
+
+		void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size) {
+			os.write(&buffer[0], buffer_size);
+		}
+
 	} // namespace detail
 
 	std::string ReadZeroTerminatedString(std::istream& is) {

src/libeuropa/io/StreamUtils.h

@@ -16,9 +16,9 @@ namespace europa::io::impl {
 
 	namespace detail {
 		void ReadStreamTypeImpl(std::istream& is, char* buffer, std::size_t size);
+		void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size);
 	}
 
-
 	// This is lame. But it works :)
 	template <class T>
 	constexpr T ReadStreamType(std::istream& is) {
@@ -37,6 +37,19 @@ namespace europa::io::impl {
 		return object;
 	}
 
+	template <class T>
+	constexpr void WriteStreamType(std::ostream& os, const T& object) {
+		// Absolutely UB.
+		union Hack {
+			const T* t;
+			const char* c;
+		} address {
+			.t = &object
+		};
+
+		detail::WriteStreamTypeImpl(os, address.c, sizeof(T));
+	}
+
 	std::string ReadZeroTerminatedString(std::istream& is);
 	std::string ReadPString(std::istream& is);
 

src/libeuropa/io/YatfReader.cpp

@@ -29,7 +29,6 @@ namespace europa::io {
 	void YatfReader::ReadImage() {
 		if(header.flags & structs::YatfHeader::TextureFlag_NoPalette) {
 			image.Resize({ static_cast<std::uint16_t>(header.width), static_cast<std::uint16_t>(header.height) });
-
 			stream.read(reinterpret_cast<char*>(image.GetBuffer()), (header.width * header.height) * sizeof(pixel::RgbaColor));
 		} else {
 			pixel::RgbaColor palette[256];
@@ -37,7 +36,6 @@ namespace europa::io {
 
 			// NB: sizeof() does pre-multiplication, so it's 100% ok for us to do this.
 			stream.read(reinterpret_cast<char*>(&palette[0]), sizeof(palette));
-
 			stream.read(reinterpret_cast<char*>(&tempBuffer[0]), tempBuffer.size());
 
 			image.Resize({ static_cast<std::uint16_t>(header.width), static_cast<std::uint16_t>(header.height) });

src/tools/CMakeLists.txt

@@ -16,3 +16,12 @@ set_target_properties(texdump PROPERTIES
 		CXX_STANDARD 20
 		CXX_STANDARD_REQUIRED ON
 		)
+
+add_executable(paktest paktest.cpp)
+
+target_link_libraries(paktest PUBLIC libeuropa)
+
+set_target_properties(paktest PROPERTIES
+		CXX_STANDARD 20
+		CXX_STANDARD_REQUIRED ON
+		)

src/tools/europa_pack_extractor.cpp

@@ -6,12 +6,12 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 //
 
+#include <europa/io/PakReader.h>
+
 #include <filesystem>
 #include <fstream>
 #include <iostream>
 
-#include <europa/io/PakReader.h>
-
 namespace fs = std::filesystem;
 
 int main(int argc, char** argv) {
@@ -38,7 +38,7 @@ int main(int argc, char** argv) {
 		return 1;
 	}
 
-	for(auto& [ filename, file ] : reader.GetFiles()) {
+	for(auto& [filename, file] : reader.GetFiles()) {
 		auto nameCopy = filename;
 
 #ifndef _WIN32

src/tools/paktest.cpp

@@ -0,0 +1,42 @@
+//
+// EuropaTools
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+
+// A test utility to regurgitate a pak.
+
+#include <europa/io/PakReader.h>
+#include <europa/io/PakWriter.h>
+
+#include <iostream>
+#include <fstream>
+
+int main(int argc, char** argv) {
+    std::ifstream ifs(argv[1], std::ifstream::binary);
+    std::ofstream ofs("new_archive.pak", std::ofstream::binary);
+
+    europa::io::PakWriter writer;
+
+    writer.Init();
+
+    // Read pak data and vomit it into the writer.
+    // This will temporarily consume 2x the memory (so about 240mb for the biggest paks I've seen),
+    // but the writer will contain the first copy,
+    // until it's cleared.
+    {
+        europa::io::PakReader reader(ifs);
+        reader.ReadData();
+
+        for (auto &[filename, file]: reader.GetFiles()) {
+            writer.AddFile(filename, file);
+        }
+    }
+
+    writer.Write(ofs);
+
+    std::cout << "Wrote regurgitated archive to new.pak!\n";
+    return 0;
+}

src/tools/texdump.cpp

@@ -6,15 +6,13 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 //
 
+#include <europa/io/YatfReader.h>
+#include <pixel/ImageWriter.h>
 
 #include <filesystem>
 #include <fstream>
 #include <iostream>
 
-#include <europa/io/YatfReader.h>
-
-#include <pixel/ImageWriter.h>
-
 namespace fs = std::filesystem;
 int main(int argc, char** argv) {
 	if(argc != 2) {
@@ -38,8 +36,7 @@ int main(int argc, char** argv) {
 
 	reader.ReadImage();
 
-
+	pixel::ImageWriter writer {};
-	pixel::ImageWriter writer{};
 
 	auto outPath = fs::path(argv[1]).replace_extension(".png");