WIP attempt at extracting Namco Museum GCN bolt library file.. seems to only kind of work

lib/base/ByteSwap.hpp

@@ -0,0 +1,28 @@
+#pragma once
+#include <bit>
+#include <base/Types.hpp>
+
+namespace lightningbolt {
+	namespace detail {
+
+		template <class T, class U>
+		constexpr T PunCast(U v) {
+			return *std::bit_cast<const T*>(&v);
+		}
+
+		template <class T>
+		constexpr T ByteSwap(T v) {
+			if constexpr(sizeof(T) == 2)
+				return static_cast<T>(__builtin_bswap16(PunCast<u16>(v)));
+			else if constexpr(sizeof(T) == 4)
+				return static_cast<T>(__builtin_bswap32(PunCast<u32>(v)));
+			else if constexpr(sizeof(T) == 8)
+				return static_cast<T>(__builtin_bswap64(PunCast<u64>(v)));
+
+			return v;
+		}
+
+	} // namespace detail
+
+	using detail::ByteSwap;
+} // namespace lightningbolt

lib/bolt/Reader.cpp

@@ -1,94 +1,162 @@
 
 
+#include <base/ByteSwap.hpp>
 #include <base/MmapFile.hpp>
 #include <bolt/Reader.hpp>
-
 #include <structs/BoltStructs.hpp>
-#include "base/ErrorOr.hpp"
-
-#if 0
-
-#endif
 
 namespace lightningbolt {
 
+	fs::path MakeGamePath(BoltReader::Game game) {
+		switch(game) {
+			case BoltReader::Game::LooseBolt:
+				return "./"; // for now
+				break;
+
+			case BoltReader::Game::SimpsonsSkateboarding: return "ASSETS.BLT"; break;
+			case BoltReader::Game::NamcoMuseumGCN: return "Data0.blt"; break;
+		}
+	}
+
 	struct BoltReader::Impl {
+		Impl(Game game) : game(game) { std::printf("game %04x\n", game); }
 
 		ErrorOr<void> OpenBolt(const fs::path& path) {
 			// Load the BOLT file
-			if(auto error = boltFile.Open(path); error.HasError())
+			if(auto error = boltFile.Open(path / MakeGamePath(game)); error.HasError())
 				return error;
 
+			std::printf("mewow\n");
+
 			lib = std::bit_cast<BoltLibraryHeader*>(boltFile.GetMapping());
 
 			if(!lib->Validate())
 				return std::make_error_code(BoltErrc::InvalidMagic);
 
-			auto p = path;
+			if(game == Game::SimpsonsSkateboarding) {
-			p.replace_filename("SLUS_201.14");
+				if(auto error = elfFile.Open(path / "SLUS_201.14"); error.HasError())
+					return error;
 
-			if(auto error = elfFile.Open(p); error.HasError())
+				// Load table entries.
-				return error;
+				GetTableEntries();
+			}
 
-			// Load table entries.
+			std::printf("ret\n");
-			GetTableEntries();
 
 			return {};
 		}
 
 		const std::vector<BoltReader::File>& GetTableEntries() {
-			if(entryTable.empty()) {
+			if(game == Game::SimpsonsSkateboarding) {
-				auto* base = elfFile.GetMapping();
+				if(entryTable.empty()) {
-				auto* table = std::bit_cast<elf::BoltTableEntry*>(base + elf::BoltTableOffsets.usTable);
+					auto* base = elfFile.GetMapping();
-				while(table->filenamePtr != 0x0) {
+					auto* table = std::bit_cast<elf::BoltTableEntry*>(base + elf::BoltTableOffsets.usTable);
-					auto string_offset = elf::AddressToElfFileOffset(table->filenamePtr);
+					while(table->filenamePtr != 0x0) {
+						auto string_offset = elf::AddressToElfFileOffset(table->filenamePtr);
 
-					BoltReader::File te;
+						BoltReader::File te;
-					te.filename = { std::bit_cast<char*>(base + string_offset) };
+						te.filename = { std::bit_cast<char*>(base + string_offset) };
-					te.index = table->entryId;
+						te.index = table->entryId;
-					te.gid = table->groupId;
+						te.gid = table->groupId;
 
-					if(te.filename == "")
+						if(te.filename == "")
-						break;
+							break;
 
-					entryTable.emplace_back(te);
+						entryTable.emplace_back(te);
 
-					table++;
+						table++;
+					}
 				}
 			}
 			return entryTable;
 		}
 
+		template <class T>
+		auto SwapIfGcn(T v) {
+			return (game == Game::NamcoMuseumGCN) ? ByteSwap(v) : v;
+		}
+
 		template <class F>
 		void ForEachFile(F f) {
-			for(auto& file : entryTable) {
+			if(game == Game::SimpsonsSkateboarding) {
-				if(file.uncompressedData == nullptr) {
+				for(auto& file : entryTable) {
-					auto gid = (file.gid >> 8);
+					if(file.uncompressedData == nullptr) {
-					auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping());
+						auto gid = (file.gid >> 8);
-					auto size = entries[file.index & 0x00ff].fileSize;
+						auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping());
-					auto offset = entries[file.index & 0x00ff].fileOffset;
+						auto size = entries[file.index & 0x00ff].fileSize;
+						auto offset = entries[file.index & 0x00ff].fileOffset;
 
-					file.compressed = !(entries[file.index & 0x00ff].unk & 0x8);
+						file.compressed = !(entries[file.index & 0x00ff].unk & 0x8);
 
-					file.uncompressedData = std::bit_cast<u8*>(boltFile.GetMapping() + offset);
+						file.uncompressedData = std::bit_cast<u8*>(boltFile.GetMapping() + offset);
-					file.uncompressedSize = size;
+						file.uncompressedSize = size;
+					}
+
+					if(!f(file))
+						break;
 				}
+			} else {
+				// generic "We don't have a table" implementation. You don't get nice filenames like this,
+				// simply because we cannot create them (.. maybe, if the unk3 is a hash, we could crack it,
+				// but that would bloat extraction times to potentionally hours if not days, and also might
+				// end up with wrong filenames at the end anyways due to the hash algorithm possibly being weak)
+				//
+				// We still end up synthesizing a File object for the user to interact with anyways.
+				
+				for(u32 i = 0; i < lib->groupCount; ++i) {
+					auto groupDescriptor = lib->GroupDescriptors()[i];
 
-				if(!f(file))
+					// We can't use the helper since it doesn't know about GCN byteswapping.
-					break;
+					// FIXME/TODO?
+					std::span<BoltGroupEntry> groupEntries = {
+						std::bit_cast<BoltGroupEntry*>(boltFile.GetMapping() + SwapIfGcn(groupDescriptor.groupOffset)), groupDescriptor.EntryCount()
+					};
+
+					for(u32 j = 0; j < groupEntries.size(); ++j) {
+						auto& ent = groupEntries[j];
+						auto filename = std::format("blt_{:02x}_{:02x}.bin", i, j);
+						BoltReader::File file;
+
+						file.gid = i;
+						file.index = j;
+
+						// Maybe BANG! but this is only accessed here.
+						file.filename = filename;
+
+						file.compressed = !(ent.unk & 0x8);
+
+						file.uncompressedSize = SwapIfGcn(ent.fileSize);
+						file.uncompressedData = std::bit_cast<u8*>(boltFile.GetMapping() + SwapIfGcn(ent.fileOffset));
+
+						if(!f(file))
+							return;
+					}
+				}
 			}
 		}
 
 	   private:
-		std::vector<BoltReader::File> entryTable;
+		Game game;
-		lightningbolt::MmapFile elfFile;
+		MmapFile boltFile;
-		lightningbolt::MmapFile boltFile;
 
-		lightningbolt::BoltLibraryHeader* lib;
+		// Only applicable if game == Game::SimpsonsSkateboarding
+		std::vector<BoltReader::File> entryTable;
+		MmapFile elfFile;
+
+		BoltLibraryHeader* lib;
 	};
 
-	BoltReader::BoltReader():
+	BoltReader::Game BoltReader::GuessGame(const fs::path& path) {
-		impl(std::make_unique<Impl>()) {}
+		if(fs::exists(path / "SLUS_201.14") && fs::exists(path / "ASSETS.BLT"))
+			return Game::SimpsonsSkateboarding;
+		else if(fs::exists(path / "Data0.blt"))
+			return Game::NamcoMuseumGCN;
+		else
+			return Game::LooseBolt;
+	}
+
+	BoltReader::BoltReader(Game game) : impl(std::make_unique<Impl>(game)) {
+	}
 
 	BoltReader::~BoltReader() = default;
 
@@ -97,9 +165,7 @@ namespace lightningbolt {
 	}
 
 	void BoltReader::ForEachFile(std::function<bool(File&)> f) {
-		impl->ForEachFile([&f](auto& file) {
+		impl->ForEachFile([&f](auto& file) { return f(file); });
-			return f(file);
-		});
 	}
 
 } // namespace lightningbolt

lib/bolt/Reader.hpp

@@ -6,6 +6,15 @@
 namespace lightningbolt {
 
 	struct BoltReader {
+		enum class Game {
+			LooseBolt, ///< Use this for games with no bolt entry
+			SimpsonsSkateboarding,
+			NamcoMuseumGCN
+		};
+
+		// Guess the game the user wants to extract
+		static Game GuessGame(const fs::path& path);
+
 		struct File {
 			std::string_view filename;
 			u16 index;
@@ -15,10 +24,9 @@ namespace lightningbolt {
 
 			u8* uncompressedData { nullptr };
 			usize uncompressedSize;
-
 		};
 
-		BoltReader();
+		BoltReader(Game game = Game::SimpsonsSkateboarding);
 		~BoltReader();
 
 		ErrorOr<void> OpenBolt(const fs::path& path);

lib/structs/BoltStructs.hpp

@@ -7,6 +7,7 @@
 
 namespace lightningbolt {
 
+	/// Elf structures, only applicable to Simpsons Skateboarding
 	namespace elf {
 
 		/// Table entry in the ELF. We use this to create file names.
@@ -66,8 +67,10 @@ namespace lightningbolt {
 	};
 
 	struct [[gnu::packed]] BoltLibraryHeader {
-		static constexpr char VALID_MAGIC[] = "BOLT\r\n";
+		static constexpr char VALID_MAGIC[] = "BOLT";
-		char magic[6];
+		char magic[4];
+
+		u8 unkmaybemagic[2]; // could be platform flags ?
 
 		u8 unk;
 		u8 unk2;

src/main.cpp

@@ -24,13 +24,16 @@ fs::path ConvertBoltPath(const std::string_view filename) {
 }
 
 int main() {
-	lightningbolt::BoltReader reader;
+	auto path = fs::current_path();
-	if(auto error = reader.OpenBolt(fs::current_path() / "ASSETS.BLT"); error.HasError()) {
+
+	lightningbolt::BoltReader reader{lightningbolt::BoltReader::GuessGame(path)};
+	if(auto error = reader.OpenBolt(path); error.HasError()) {
 		std::cout << std::format("Error initalizing bolt reader: {}\n", error.Error());
 	}
 
 	reader.ForEachFile([](lightningbolt::BoltReader::File& ent) {
 		std::cout << std::format("File: {}", ent.filename);
+#if 1
 
 		auto p = ConvertBoltPath(ent.filename);
 
@@ -64,6 +67,7 @@ int main() {
 			ofs.write((char*)ent.uncompressedData, ent.uncompressedSize);
 			std::cout << std::format(", written to \"{}\"\n", p.string());
 		}
+#endif
 		return true;
 	});