Initial prototype - reads the TOC from the ELF and maps the BOLT file

.clang-format

@@ -0,0 +1,46 @@
+# .clang-format for native code portion
+
+BasedOnStyle: Google
+
+# force T* or T&
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+TabWidth: 4
+IndentWidth: 4
+UseTab: Always
+IndentPPDirectives: BeforeHash
+
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortFunctionsOnASingleLine: InlineOnly
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: true
+
+BinPackArguments: true
+BinPackParameters: true
+BreakConstructorInitializers: BeforeColon
+BreakStringLiterals: false
+
+ColumnLimit: 150
+CompactNamespaces: false
+
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ContinuationIndentWidth: 0
+
+# turning this on causes major issues with initializer lists
+Cpp11BracedListStyle: false
+SpaceBeforeCpp11BracedList: true
+
+FixNamespaceComments: true
+
+NamespaceIndentation: All
+ReflowComments: true
+
+SortIncludes: CaseInsensitive
+SortUsingDeclarations: true
+
+SpacesInSquareBrackets: false
+SpaceBeforeParens: Never
+SpacesBeforeTrailingComments: 1

.gitignore

@@ -0,0 +1,5 @@
+build/
+/out
+.cache/
+/compile_commands.json
+

CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.15)
+
+project(lightningbolt
+	LANGUAGES CXX
+)
+
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
+
+include(Policies)
+include(ProjectFuncs)
+
+# default linker
+if(NOT LIGHTNINGBOLT_LINKER AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+	set(LIGHTNINGBOLT_LINKER "lld")
+elseif(NOT LIGHTNINGBOLT_LINKER)
+	set(LIGHTNINGBOLT_LINKER "bfd")
+endif()
+
+lb_set_alternate_linker()
+
+add_subdirectory(lib/base)
+
+# third party vendor dependencies
+#add_subdirectory(third_party)
+
+add_executable(lightningbolt
+	src/main.cpp
+)
+
+target_link_libraries(lightningbolt PRIVATE
+	lb::base
+)
+
+lb_target(lightningbolt)

cmake/Policies.cmake

@@ -0,0 +1,22 @@
+# CMake policy configuration
+
+# Macro to enable new CMake policy.
+# Makes this file a *LOT* shorter.
+macro (_new_cmake_policy policy)
+	if(POLICY ${policy})
+		#message(STATUS "Enabling new policy ${policy}")
+		cmake_policy(SET ${policy} NEW)
+	endif()
+endmacro()
+
+_new_cmake_policy(CMP0026) # CMake 3.0: Disallow use of the LOCATION property for build targets.
+_new_cmake_policy(CMP0042) # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name
+_new_cmake_policy(CMP0046) # warn about non-existent dependencies
+_new_cmake_policy(CMP0048) # CMake 3.0+: project() command now maintains VERSION
+_new_cmake_policy(CMP0054) # CMake 3.1: Only interpret if() arguments as variables or keywords when unquoted.
+_new_cmake_policy(CMP0056) # try_compile() linker flags
+_new_cmake_policy(CMP0066) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE
+_new_cmake_policy(CMP0067) # CMake 3.8: try_compile(): honor language standard variables (like C++11)
+_new_cmake_policy(CMP0068) # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`.
+_new_cmake_policy(CMP0075) # CMake 3.12+: Include file check macros honor `CMAKE_REQUIRED_LIBRARIES`
+_new_cmake_policy(CMP0077) # CMake 3.13+: option() honors normal variables.

cmake/ProjectFuncs.cmake

@@ -0,0 +1,59 @@
+function(lb_target target)
+	target_compile_definitions(${target} PRIVATE "$<$<CONFIG:DEBUG>:LIGHTNINGBOLT_DEBUG>")
+	#target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR})
+
+	target_compile_features(${target} PRIVATE cxx_std_20)
+	target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/lib ${CMAKE_CURRENT_BINARY_DIR})
+
+	set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS -Wall -Wextra)
+	if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
+		set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS})
+
+		# If on Release use link-time optimizations.
+		# On clang we use ThinLTO for even better build performance.
+		if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+			set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto=thin)
+			target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto=thin)
+		else()
+			set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto)
+			target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto)
+		endif()
+
+		target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS})
+	endif()
+
+
+	if("asan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
+		# Error if someone's trying to mix asan and tsan together,
+		# they aren't compatible.
+		if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
+			message(FATAL_ERROR "ASAN and TSAN cannot be used together.")
+		endif()
+
+		message(STATUS "Enabling ASAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
+		target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=address)
+		target_link_libraries(${target} PRIVATE -fsanitize=address)
+	endif()
+
+	if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
+		message(STATUS "Enabling TSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
+		target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=thread)
+		target_link_libraries(${target} PRIVATE -fsanitize=thread)
+	endif()
+
+	if("ubsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
+		message(STATUS "Enabling UBSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
+		target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=undefined)
+		target_link_libraries(${target} PRIVATE -fsanitize=undefined)
+	endif()
+
+endfunction()
+
+function(lb_set_alternate_linker)
+	find_program(LINKER_EXECUTABLE ld.${LIGHTNINGBOLT_LINKER} ${COLLABVM_LINKER})
+	if(LINKER_EXECUTABLE)
+		message(STATUS "Using ${LIGHTNINGBOLT_LINKER} as linker")
+	else()
+		message(FATAL_ERROR "Linker ${LIGHTNINGBOLT_LINKER} does not exist on your system. Please specify one which does or omit this option from your configure command.")
+	endif()
+endfunction()

lib/base/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_library(lb_base
+    MmapFile.cpp
+)
+
+lb_target(lb_base)
+add_library(lb::base ALIAS lb_base)

lib/base/ErrorOr.hpp

@@ -0,0 +1,90 @@
+#pragma once
+#include <optional>
+#include <system_error>
+#include <variant>
+
+template <class T>
+struct ErrorOr {
+   private:
+	using Variant = std::variant<T, std::error_code>;
+
+   public:
+	constexpr static auto IsVoid = false;
+	ErrorOr() = default;
+	ErrorOr(const ErrorOr&) = default;
+	ErrorOr(ErrorOr&&) = default;
+
+	ErrorOr(const T& v) : maybeVariant(v) {}
+	ErrorOr(const std::error_code& v) : maybeVariant(v) {}
+
+	ErrorOr& operator=(const T& value) {
+		maybeVariant = value;
+		return *this;
+	}
+
+	ErrorOr& operator=(const std::error_code& ec) {
+		maybeVariant = ec;
+		return *this;
+	}
+
+	bool HasValue() const {
+		if(maybeVariant)
+			return std::holds_alternative<T>(*maybeVariant);
+
+		// No value.
+		return false;
+	}
+
+	bool HasError() const {
+		if(maybeVariant)
+			return std::holds_alternative<std::error_code>(*maybeVariant);
+
+		// No error. Possibly in the default constructed state
+		return false;
+	}
+
+	// ASSERT TODO
+
+	T& Value() { return std::get<T>(maybeVariant.value()); }
+
+	const T& Value() const { return std::get<T>(maybeVariant.value()); }
+
+	operator T&() { return Value(); }
+	operator const T&() { return Value(); }
+
+	std::error_code& Error() { return std::get<std::error_code>(maybeVariant.value()); }
+
+	const std::error_code& Error() const { return std::get<std::error_code>(maybeVariant.value()); }
+
+	operator std::error_code&() { return Error(); }
+	operator const std::error_code&() const { return Error(); }
+
+   private:
+	std::optional<Variant> maybeVariant;
+};
+
+template <>
+struct ErrorOr<void> {
+	constexpr static auto IsVoid = true;
+	ErrorOr() = default;
+	ErrorOr(const ErrorOr&) = default;
+	ErrorOr(ErrorOr&&) = default;
+	ErrorOr(const std::error_code& v) : maybeEc(v) {}
+
+	ErrorOr& operator=(const std::error_code& ec) {
+		maybeEc = ec;
+		return *this;
+	}
+
+	bool HasError() const { return maybeEc.has_value(); }
+
+	std::error_code& Error() { return maybeEc.value(); }
+
+	const std::error_code& Error() const { return maybeEc.value(); }
+
+	operator std::error_code&() { return Error(); }
+	operator const std::error_code&() const { return Error(); }
+
+   private:
+	std::optional<std::error_code> maybeEc;
+};

lib/base/MmapFile.cpp

@@ -0,0 +1,31 @@
+#include <base/MmapFile.hpp>
+
+#ifdef __linux__
+	#include "MmapFile.linux.cpp"
+#else
+	#error Invalid platform
+#endif
+
+namespace lightningbolt {
+
+	MmapFile::MmapFile() : impl(std::make_unique<Impl>()) {
+	}
+
+	MmapFile::~MmapFile() = default;
+
+	ErrorOr<void> MmapFile::Open(const fs::path& path) {
+		return impl->Open(path);
+	}
+
+	void MmapFile::Close() {
+		return impl->Close();
+	}
+
+	u8* MmapFile::GetMapping() const {
+		return impl->GetMapping();
+	}
+	usize MmapFile::GetMappingSize() const {
+		return impl->GetMappingSize();
+	}
+
+} // namespace lightningbolt

lib/base/MmapFile.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include <base/Types.hpp>
+#include <base/ErrorOr.hpp>
+
+namespace lightningbolt {
+
+    /// A read-only file opened via memory mapping.
+    /// On POSIX systems, we use mmap(2). Etc etc.
+    struct MmapFile {
+        MmapFile();
+        ~MmapFile();
+
+        // Opens for read-only mode.
+        ErrorOr<void> Open(const fs::path& path);
+        void Close();
+
+        u8* GetMapping() const;
+        usize GetMappingSize() const;
+
+    private:
+        struct Impl;
+        Unique<Impl> impl;
+    };
+
+}

lib/base/MmapFile.linux.cpp

@@ -0,0 +1,63 @@
+#include "MmapFile.hpp"
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/mman.h>
+
+namespace lightningbolt {
+
+	struct MmapFile::Impl {
+
+        ~Impl() {
+            Close();
+        }
+
+        void Close() {
+            if(mapping) {
+                munmap(mapping, mappingSize);
+                mapping = nullptr;
+                mappingSize = 0;
+            }
+        }
+
+        ErrorOr<void> Open(const fs::path& path) {
+            int fd = open(path.string().c_str(), O_RDONLY);
+
+            // Error opening file.
+            if(fd == -1)
+                return std::error_code{errno, std::system_category()};
+
+            {
+                auto last = lseek64(fd, 0, SEEK_END);
+                mappingSize = lseek64(fd, 0, SEEK_CUR);
+                lseek64(fd, last, SEEK_SET);
+            }
+
+            mapping = static_cast<u8*>(mmap(nullptr, mappingSize, PROT_READ, MAP_PRIVATE, fd, 0));
+            if(mapping == static_cast<u8*>(MAP_FAILED)) {
+                mappingSize = 0;
+                return std::error_code{errno, std::system_category()};
+            }
+
+            // Once the mapping has successfully been created
+            // we can close the file descriptor instead of needing
+            // to remember it (the kernel will do so for us.)
+            close(fd);
+
+            // No error.
+            return {};
+        }
+
+        u8* GetMapping() const {
+            return mapping;
+        }
+
+        usize GetMappingSize() const {
+            return mappingSize;
+        }
+
+    private:
+        u8* mapping;
+        usize mappingSize;
+    };
+
+} // namespace lightningbolt

lib/base/OffsetPtr.hpp

@@ -0,0 +1,85 @@
+#pragma once
+#include <bit>
+#include <base/Types.hpp>
+#include <type_traits>
+#include <span>
+
+namespace lightningbolt {
+
+	namespace detail {
+
+		template <class NativeT, class OffsetType>
+		constexpr NativeT* CreatePointerFromAddend(void* BasePointer, OffsetType addend) noexcept {
+			return std::bit_cast<NativeT*>(static_cast<char*>(BasePointer) + addend);
+		}
+
+	} // namespace detail
+
+	/// An "auto-resolving" semi-sweet (/s) pointer type.
+	/// This is designed to allow resolving offsets in data for 
+	/// games written before 64-bit pointers were common/used at all.
+	/// This allows looking up data a lot easier :)
+	///
+	/// [NativeT] is the type of data this would point to
+	/// [OffsetType] is the type of data the "pointer" is repressented as
+	template <class NativeT, class OffsetType = u32>
+	struct OffsetPtr final {
+		using Type = std::remove_cvref_t<NativeT>;
+		using Pointer = Type*;
+		using ConstPointer = const Type*;
+
+		/// Set the offset. Duh!
+		constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; }
+
+		[[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; }
+
+		[[nodiscard]] constexpr Pointer operator()(void* baseAddr) const noexcept {
+			// While yucky, it should show problem areas which aren't checking things
+			// immediately rather than read invalid data that might do much worse.
+			if(rawOffset == 0)
+				return nullptr;
+
+			return detail::CreatePointerFromAddend<Type>(baseAddr, rawOffset);
+		}
+
+		template<class NativeU>
+		constexpr OffsetPtr<NativeU, OffsetType>& PtrCast() {
+			// Safety: The data layout of OffsetPtr<> stays
+			// the exact same regardless of the result type, therefore
+			// even though this is *techinically* UB (? using bit_cast it shouldn't be ?),
+			// this isn't problematic
+			return *std::bit_cast<OffsetPtr<NativeU, OffsetType>*>(this);
+		}
+
+	   private:
+		OffsetType rawOffset;
+	};
+
+	/// Like OffsetPtr<T> but for arrays of data
+	template <class NativeT, class OffsetType = u32>
+	struct OffsetArrayPtr final {
+		using Type = std::remove_cvref_t<NativeT>;
+		using Pointer = Type*;
+		using ConstPointer = const Type*;
+
+		using Span = std::span<NativeT>;
+
+		/// Set the offset. Duh!
+		constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; }
+
+		[[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; }
+
+		[[nodiscard]] constexpr Span operator()(void* baseAddr, OffsetType length) const noexcept {
+			// While yucky, it should show problem areas which aren't checking things
+			// immediately rather than read invalid data that might do much worse.
+			if(rawOffset == 0 || length == 0)
+				return {};
+
+			return { detail::CreatePointerFromAddend<Type>(baseAddr, rawOffset), length };
+		}
+
+	   private:
+		OffsetType rawOffset;
+	};
+
+} // namespace ssxtools::core

lib/base/Types.hpp

@@ -0,0 +1,31 @@
+//! Core types and includes
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <filesystem>
+
+// these are in the global namespace since most libraries
+// won't try defining anything like this in the global namespace
+// (and I'd like these types to be used globally a lot more anyways)
+using u8 = std::uint8_t;
+using i8 = std::int8_t;
+using u16 = std::uint16_t;
+using i16 = std::int16_t;
+using u32 = std::uint32_t;
+using i32 = std::int32_t;
+using u64 = std::uint64_t;
+using i64 = std::int64_t;
+using usize = std::size_t;
+using isize = std::intptr_t;
+
+namespace lightningbolt {
+    namespace fs = std::filesystem;
+
+	template <class T, class Deleter = std::default_delete<T>>
+	using Unique = std::unique_ptr<T, Deleter>;
+
+	template <class T>
+	using Ref = std::shared_ptr<T>;
+
+} // namespace lightningbolt

lib/structs/BoltStructs.hpp

@@ -0,0 +1,86 @@
+#include <base/Types.hpp>
+#include <cstring>
+#include <span>
+
+namespace lightningbolt {
+
+	namespace elf {
+
+        /// Table entry in the ELF. We use this to create file names.
+		struct [[gnu::packed]] BoltTableEntry {
+			/// Pointer to filename. Should be adjusted
+			u32 filenamePtr;
+			u16 entryId; // (GID >> 8) | ID
+			u16 groupId; // (entryId & 0xff00)
+		};
+
+        /// Offsets in the ELF to the table. 
+        struct BoltTableOffsets {
+            u32 usTable;
+        };
+
+        /// Convert a address to ELF file offset.
+        constexpr u32 AddressToElfFileOffset(u32 address) {
+            constexpr u32 LoadAddress = 0x00100000;
+            constexpr u32 SectionOffset = 0x80;
+            return (address - LoadAddress) + SectionOffset;
+        }
+
+        static constexpr BoltTableOffsets BoltTableOffsets = {
+            .usTable = AddressToElfFileOffset(0x0033d400)
+        };
+
+	} // namespace elf
+
+	struct [[gnu::packed]] BoltGroupEntry {
+		u32 unk;
+		u32 fileSize;
+		u32 fileOffset;
+		u32 unk3; // name hash?
+	};
+
+	struct [[gnu::packed]] BoltGroupDescriptor {
+		u8 unk;
+		u8 unk2;
+		u8 unk3;
+		u8 entryCount;
+
+		u32 groupSize;
+		u32 groupOffset;
+
+		u32 EntryCount() {
+			// Special case: 0x0 == 256 entries.
+			// I have NO idea why they did it like this,
+			// this really seems extra when they could have
+			// used a short field in the same exact space.
+			if(entryCount == 0x0)
+				return 256;
+			return entryCount;
+		}
+
+		std::span<BoltGroupEntry> Entries(u8* base) { return { std::bit_cast<BoltGroupEntry*>(base + groupOffset), EntryCount() }; }
+	};
+
+	struct [[gnu::packed]] BoltLibraryHeader {
+		static constexpr char VALID_MAGIC[] = "BOLT\r\n";
+		char magic[6];
+
+		u8 unk;
+		u8 unk2;
+
+		u8 unk3;
+		u8 unk4;
+		u8 unk5;
+		u8 groupCount;
+
+		u32 libSize;
+
+		std::span<BoltGroupDescriptor> GroupDescriptors() {
+			// The group descriptors are after the primary library header
+			return { std::bit_cast<BoltGroupDescriptor*>(this + 1), groupCount };
+		}
+
+		inline bool Validate() const { return !std::memcmp(&magic[0], &VALID_MAGIC[0], sizeof(magic)); }
+	};
+
+} // namespace lightningbolt

src/main.cpp

@@ -0,0 +1,80 @@
+#include <base/MmapFile.hpp>
+#include <filesystem>
+#include <structs/BoltStructs.hpp>
+#include <vector>
+#include <format>
+
+#include <iostream>
+
+struct ParsedTableEntry {
+	std::string_view filename;
+	u16 index;
+	u32 gid;
+};
+
+struct BoltReader {
+	BoltReader() {}
+
+	ErrorOr<void> OpenBolt(const lightningbolt::fs::path& path) {
+		auto p = path;
+		p.replace_filename("SLUS_201.14");
+
+		if(auto error = elfFile.Open(p); error.HasError())
+			return error;
+
+		// Load table entries.
+		GetTableEntries();
+
+		// Load the BOLT file
+		if(auto error = boltFile.Open(path); error.HasError())
+			return error;
+
+		return {};
+	}
+
+	const std::vector<ParsedTableEntry>& GetTableEntries() {
+		if(entryTable.empty()) {
+			auto* base = elfFile.GetMapping();
+			auto* table = std::bit_cast<lightningbolt::elf::BoltTableEntry*>(base + lightningbolt::elf::BoltTableOffsets.usTable);
+			while(table->filenamePtr != 0x0) {
+				auto string_offset = lightningbolt::elf::AddressToElfFileOffset(table->filenamePtr);
+
+				ParsedTableEntry te;
+				te.filename = { std::bit_cast<char*>(base + string_offset) };
+				te.index = table->entryId;
+				te.gid = table->groupId;
+
+				if(te.filename == "")
+					break;
+
+				std::cout << std::format("te: {} {:04x} {:04x}\n", te.filename, te.index, te.gid);
+				entryTable.emplace_back(te);
+
+				table++;
+			}
+
+			// The ELF file isn't needed after this so unmap it
+			elfFile.Close();
+		}
+		return entryTable;
+	}
+
+	template<class F>
+	void ForEachFile(F f) {
+		//for()()
+	}
+
+   private:
+	std::vector<ParsedTableEntry> entryTable;
+	lightningbolt::MmapFile elfFile;
+	lightningbolt::MmapFile boltFile;
+};
+
+int main() {
+	BoltReader reader;
+	if(auto error = reader.OpenBolt(lightningbolt::fs::current_path() / "ASSETS.BLT"); error.HasError()) {
+		std::cout << "Error opening Bolt file: " << error.Error();
+	}
+
+	return 0;
+}