Mishmash commit

While working with scrc I made the pak files output sorted by size. This doesn't
seem to have helped with performance but I guess it might be nice to maybe
do the conversion in hand.. or just make the writer take a vector of pair to use.

README.md

@@ -1,15 +1,46 @@
-## Europa tools
+## EuropaTools
 
-TODO
+Tools for working with LEC Europa based games (Star Wars: Starfighter & Star Wars: Jedi Starfighter).
+
+As per usual for lily, written in C++20.
+
+
+## Building
+
+```bash
+$ git clone https://github.com/modeco80/EuropaTools.git
+$ cd EuropaTools
+$ cmake -Bbuild -DCMAKE_BUILD_TYPE=Release
+$ cmake --build build -j $(nproc)
+# ... profit?
+```
 
 ## The Libraries
 
 ### `libeuropa`
 
-Provides archive IO utilities and the structures.
+Provides IO readers and writers for data files, along with the structures.
+
+Structure documentation is seperately managed as a .hexpat in [/hexpat](https://github.com/modeco80/EuropaTools/tree/master/hexpat).
 
 ## The Tools
 
+## `europa_pack_extractor`
+
+Staging tool to extract paks. Will be removed when eupak is ready.
+
+### `pakcreate`
+
+Staging tool to create paks.
+
+### `paktest`
+
+A test tool to test building paks, used during development.
+
+### `texdump`
+
+Dumper for PS2 `YATF` texture files. Mostly working, but slight WIP.
+
 ### `eupak`
 
 Swiss army knife for Europa packfiles.

include/europa/structs/Pak.h

@@ -16,14 +16,14 @@
 
 namespace europa::structs {
 
-	enum class PakVersion : u16 {
-		Starfighter = 0x4,
-		Ver2 = 0x5
-	};
-
 	struct [[gnu::packed]] PakHeader {
 		constexpr static const char VALID_MAGIC[16] = "Europa Packfile";
 
+		enum class Version : u16 {
+			Ver4 = 0x4,
+			Ver5 = 0x5
+		};
+
 		char magic[16]; // "Europa Packfile\0"
 
 		/**
@@ -31,7 +31,7 @@ namespace europa::structs {
 		 */
 		u16 headerSize;
 
-		PakVersion version;
+		Version version;
 		u8 pad;
 
 		u32 tocOffset;
@@ -42,6 +42,7 @@ namespace europa::structs {
 
 		u32 creationUnixTime;
 
+		// Zeroes.
 		u32 reservedPad;
 
 		/**
@@ -54,7 +55,7 @@ namespace europa::structs {
 		/**
 		 * Initialize this header (used when writing).
 		 */
-		void Init(PakVersion ver) noexcept {
+		void Init(Version ver) noexcept {
 			// clear any junk
 			memset(this, 0, sizeof(PakHeader));
 
@@ -73,21 +74,22 @@ namespace europa::structs {
 			if(std::strcmp(magic, VALID_MAGIC) != 0)
 				return false;
 
-			using enum PakVersion;
+			// Check header size.
+			if(headerSize != sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1))
+				return false;
+
+			using enum Version;
 
 			// Version must match ones we support,
 			// otherwise it's invalid.
 			switch(version) {
-				case Starfighter:
+				case Ver4:
-				case Ver2:
+				case Ver5:
-					break;
+					return true;
 
 				default:
 					return false;
 			}
-
-			// Header is okay.
-			return true;
 		}
 	};
 
@@ -98,6 +100,7 @@ namespace europa::structs {
 		u32 creationUnixTime;
 	};
 
+
 	static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!");
 	static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid when writing archives.");
 	static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!");

include/europa/util/TupleElement.h

@@ -0,0 +1,38 @@
+//
+// SSX 3 Lobby Server
+//
+// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
+//
+// This file is licensed under the GNU General Public License Version 3.
+// Text is provided in LICENSE.
+//
+
+#ifndef EUROPA_UTIL_TUPLEELEMENT_H
+#define EUROPA_UTIL_TUPLEELEMENT_H
+
+#include <cstdint>
+#include <cstddef>
+
+#include <tuple>
+
+namespace europa::util {
+
+	namespace detail {
+
+		template<std::size_t N>
+		struct TupleElementImpl {
+			template<typename T>
+			constexpr decltype(auto) operator()(T&& t) const {
+				using std::get;
+				return get<N>(std::forward<T>(t));
+			}
+		};
+
+		template<std::size_t N>
+		inline constexpr TupleElementImpl<N> TupleElement;
+	}
+
+	using detail::TupleElement;
+}
+
+#endif // EUROPA_UTIL_TUPLEELEMENT_H

src/libeuropa/io/PakWriter.cpp

@@ -12,6 +12,9 @@
 
 #include "StreamUtils.h"
 
+#include <algorithm>
+#include <europa/util/TupleElement.h>
+
 namespace europa::io {
 
 	void PakWriter::Init(structs::PakVersion version) {
@@ -23,28 +26,86 @@ namespace europa::io {
 		return archiveFiles;
 	}
 
+	// move to a util/ header
+
+	template<class T>
+	constexpr T AlignBy(T value, std::size_t alignment) {
+		return (-value) & alignment - 1;
+	}
+
+	/**
+	 * Class functor for flattening a map.
+	 */
+	template<class Map>
+	struct MapFlatten {
+		/**
+		 * Storage type to store one key -> value pair.
+		 */
+		using FlattenedType = std::pair<typename Map::key_type, typename Map::mapped_type>;
+		using ArrayType = std::vector<FlattenedType>;
+
+		constexpr explicit MapFlatten(Map& mapToFlatten)
+			: map(mapToFlatten) {
+
+		}
+
+		ArrayType operator()() const {
+			ArrayType arr;
+			arr.reserve(map.size());
+
+			for(auto& [ key, value ] : map)
+				arr.emplace_back(std::make_pair(key, value));
+
+			return arr;
+		}
+
+		private:
+		 Map& map;
+	};
+
+	// TODO:
+	// 	 - Composable operations (WriteTOC, WriteFile, WriteHeader)
+	//	 - Add IProgressReportSink reporting
+
 	void PakWriter::Write(std::ostream& os) {
-		pakHeader.fileCount = archiveFiles.size();
+
+		// This essentially converts our map we use for faster insertion
+		// into a flat array we can sort easily.
+		//
+		// NB: this copies by value, so during this function we use 2x the ram.
+		// doesn't seem to be a big problem though.
+		auto sortedFiles = MapFlatten{archiveFiles}();
+
+		// Sort the flattened array by file size, the biggest first.
+		// Doesn't seem to help (neither does name length)
+		std::ranges::sort(sortedFiles, std::greater{}, [](const decltype(MapFlatten{archiveFiles})::FlattenedType& elem) {
+			return std::get<1>(elem).GetTOCEntry().size;
+		});
 
 		// Leave space for the header
 		os.seekp(sizeof(structs::PakHeader), std::ostream::beg);
 
-		// Seek forwards for version 2 PAKs, as the only
+		// Version 5 paks seem to have an additional bit of reserved data
-		// difference seems to be this additional bump
+		// (which is all zeros.)
 		if(pakHeader.version == structs::PakVersion::Ver2) {
 			os.seekp(6, std::ostream::cur);
 		}
 
 		// Write file data
-		for(auto& [filename, file] : archiveFiles) {
+		for(auto& [filename, file] : sortedFiles) {
+			//std::cout << "PakWriteFile \"" << filename << "\"\n    Size " << file.GetTOCEntry().size << "\n";
+
 			file.GetTOCEntry().offset = os.tellp();
 			os.write(reinterpret_cast<const char*>(file.GetData().data()), file.GetData().size());
+
+			// Flush on file writing
+			os.flush();
 		}
 
 		pakHeader.tocOffset = os.tellp();
 
 		// Write the TOC
-		for(auto& [filename, file] : archiveFiles) {
+		for(auto& [filename, file] : sortedFiles) {
 			file.FillTOCEntry();
 
 			// Write the pstring
@@ -56,9 +117,11 @@ namespace europa::io {
 			impl::WriteStreamType(os, file.GetTOCEntry());
 		}
 
-		// Fill out the TOC size.
+		// Fill out the rest of the header.
+		pakHeader.fileCount = archiveFiles.size();
 		pakHeader.tocSize = static_cast<std::uint32_t>(os.tellp()) - (pakHeader.tocOffset - 1);
 
+		// As the last step, write it.
 		os.seekp(0, std::ostream::beg);
 		impl::WriteStreamType(os, pakHeader);
 	}

src/tools/pakcreate.cpp

@@ -69,7 +69,9 @@ int main(int argc, char** argv) {
 		file.SetData(std::move(pakData));
 		file.FillTOCEntry();
 
-		std::cout << "Added \"" << relativePathName << "\"\n";
+		file.GetTOCEntry().creationUnixTime = 0;
+
+		//std::cout << "File \"" << relativePathName << "\"\n";
 		writer.GetFiles()[relativePathName] = std::move(file);
 	}