commit d7a48f39b47e0caa9767f12aaf94b9b35b9a3bcb Author: modeco80 Date: Sun Nov 19 06:19:15 2023 -0500 Initial prototype - reads the TOC from the ELF and maps the BOLT file diff --git a/.clang-format b/.clang-format new file mode 100755 index 0000000..8a3795a --- /dev/null +++ b/.clang-format @@ -0,0 +1,46 @@ +# .clang-format for native code portion + +BasedOnStyle: Google + +# force T* or T& +DerivePointerAlignment: false +PointerAlignment: Left + +TabWidth: 4 +IndentWidth: 4 +UseTab: Always +IndentPPDirectives: BeforeHash + +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true + +BinPackArguments: true +BinPackParameters: true +BreakConstructorInitializers: BeforeColon +BreakStringLiterals: false + +ColumnLimit: 150 +CompactNamespaces: false + +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ContinuationIndentWidth: 0 + +# turning this on causes major issues with initializer lists +Cpp11BracedListStyle: false +SpaceBeforeCpp11BracedList: true + +FixNamespaceComments: true + +NamespaceIndentation: All +ReflowComments: true + +SortIncludes: CaseInsensitive +SortUsingDeclarations: true + +SpacesInSquareBrackets: false +SpaceBeforeParens: Never +SpacesBeforeTrailingComments: 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..308b307 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build/ +/out +.cache/ +/compile_commands.json + diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..2edb747 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.15) + +project(lightningbolt + LANGUAGES CXX +) + +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + +include(Policies) +include(ProjectFuncs) + +# default linker +if(NOT LIGHTNINGBOLT_LINKER AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(LIGHTNINGBOLT_LINKER "lld") +elseif(NOT LIGHTNINGBOLT_LINKER) + set(LIGHTNINGBOLT_LINKER "bfd") +endif() + +lb_set_alternate_linker() + +add_subdirectory(lib/base) + +# third party vendor dependencies +#add_subdirectory(third_party) + +add_executable(lightningbolt + src/main.cpp +) + +target_link_libraries(lightningbolt PRIVATE + lb::base +) + +lb_target(lightningbolt) diff --git a/cmake/Policies.cmake b/cmake/Policies.cmake new file mode 100644 index 0000000..d7a084b --- /dev/null +++ b/cmake/Policies.cmake @@ -0,0 +1,22 @@ +# CMake policy configuration + +# Macro to enable new CMake policy. +# Makes this file a *LOT* shorter. +macro (_new_cmake_policy policy) + if(POLICY ${policy}) + #message(STATUS "Enabling new policy ${policy}") + cmake_policy(SET ${policy} NEW) + endif() +endmacro() + +_new_cmake_policy(CMP0026) # CMake 3.0: Disallow use of the LOCATION property for build targets. +_new_cmake_policy(CMP0042) # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name +_new_cmake_policy(CMP0046) # warn about non-existent dependencies +_new_cmake_policy(CMP0048) # CMake 3.0+: project() command now maintains VERSION +_new_cmake_policy(CMP0054) # CMake 3.1: Only interpret if() arguments as variables or keywords when unquoted. +_new_cmake_policy(CMP0056) # try_compile() linker flags +_new_cmake_policy(CMP0066) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE +_new_cmake_policy(CMP0067) # CMake 3.8: try_compile(): honor language standard variables (like C++11) +_new_cmake_policy(CMP0068) # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`. +_new_cmake_policy(CMP0075) # CMake 3.12+: Include file check macros honor `CMAKE_REQUIRED_LIBRARIES` +_new_cmake_policy(CMP0077) # CMake 3.13+: option() honors normal variables. diff --git a/cmake/ProjectFuncs.cmake b/cmake/ProjectFuncs.cmake new file mode 100644 index 0000000..3278ccd --- /dev/null +++ b/cmake/ProjectFuncs.cmake @@ -0,0 +1,59 @@ +function(lb_target target) + target_compile_definitions(${target} PRIVATE "$<$:LIGHTNINGBOLT_DEBUG>") + #target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}) + + target_compile_features(${target} PRIVATE cxx_std_20) + target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/lib ${CMAKE_CURRENT_BINARY_DIR}) + + set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS -Wall -Wextra) + if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo") + set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS}) + + # If on Release use link-time optimizations. + # On clang we use ThinLTO for even better build performance. + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto=thin) + target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto=thin) + else() + set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto) + target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto) + endif() + + target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS}) + endif() + + + if("asan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES) + # Error if someone's trying to mix asan and tsan together, + # they aren't compatible. + if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES) + message(FATAL_ERROR "ASAN and TSAN cannot be used together.") + endif() + + message(STATUS "Enabling ASAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES") + target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=address) + target_link_libraries(${target} PRIVATE -fsanitize=address) + endif() + + if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES) + message(STATUS "Enabling TSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES") + target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=thread) + target_link_libraries(${target} PRIVATE -fsanitize=thread) + endif() + + if("ubsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES) + message(STATUS "Enabling UBSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES") + target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=undefined) + target_link_libraries(${target} PRIVATE -fsanitize=undefined) + endif() + +endfunction() + +function(lb_set_alternate_linker) + find_program(LINKER_EXECUTABLE ld.${LIGHTNINGBOLT_LINKER} ${COLLABVM_LINKER}) + if(LINKER_EXECUTABLE) + message(STATUS "Using ${LIGHTNINGBOLT_LINKER} as linker") + else() + message(FATAL_ERROR "Linker ${LIGHTNINGBOLT_LINKER} does not exist on your system. Please specify one which does or omit this option from your configure command.") + endif() +endfunction() diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt new file mode 100644 index 0000000..b61f9b0 --- /dev/null +++ b/lib/base/CMakeLists.txt @@ -0,0 +1,6 @@ +add_library(lb_base + MmapFile.cpp +) + +lb_target(lb_base) +add_library(lb::base ALIAS lb_base) \ No newline at end of file diff --git a/lib/base/ErrorOr.hpp b/lib/base/ErrorOr.hpp new file mode 100644 index 0000000..ed96e65 --- /dev/null +++ b/lib/base/ErrorOr.hpp @@ -0,0 +1,90 @@ +#pragma once +#include +#include +#include + +template +struct ErrorOr { + private: + using Variant = std::variant; + + public: + constexpr static auto IsVoid = false; + ErrorOr() = default; + ErrorOr(const ErrorOr&) = default; + ErrorOr(ErrorOr&&) = default; + + ErrorOr(const T& v) : maybeVariant(v) {} + ErrorOr(const std::error_code& v) : maybeVariant(v) {} + + ErrorOr& operator=(const T& value) { + maybeVariant = value; + return *this; + } + + ErrorOr& operator=(const std::error_code& ec) { + maybeVariant = ec; + return *this; + } + + bool HasValue() const { + if(maybeVariant) + return std::holds_alternative(*maybeVariant); + + // No value. + return false; + } + + bool HasError() const { + if(maybeVariant) + return std::holds_alternative(*maybeVariant); + + // No error. Possibly in the default constructed state + return false; + } + + // ASSERT TODO + + T& Value() { return std::get(maybeVariant.value()); } + + const T& Value() const { return std::get(maybeVariant.value()); } + + operator T&() { return Value(); } + operator const T&() { return Value(); } + + std::error_code& Error() { return std::get(maybeVariant.value()); } + + const std::error_code& Error() const { return std::get(maybeVariant.value()); } + + operator std::error_code&() { return Error(); } + operator const std::error_code&() const { return Error(); } + + private: + std::optional maybeVariant; +}; + +template <> +struct ErrorOr { + constexpr static auto IsVoid = true; + ErrorOr() = default; + ErrorOr(const ErrorOr&) = default; + ErrorOr(ErrorOr&&) = default; + ErrorOr(const std::error_code& v) : maybeEc(v) {} + + ErrorOr& operator=(const std::error_code& ec) { + maybeEc = ec; + return *this; + } + + bool HasError() const { return maybeEc.has_value(); } + + std::error_code& Error() { return maybeEc.value(); } + + const std::error_code& Error() const { return maybeEc.value(); } + + operator std::error_code&() { return Error(); } + operator const std::error_code&() const { return Error(); } + + private: + std::optional maybeEc; +}; \ No newline at end of file diff --git a/lib/base/MmapFile.cpp b/lib/base/MmapFile.cpp new file mode 100644 index 0000000..f3eb1db --- /dev/null +++ b/lib/base/MmapFile.cpp @@ -0,0 +1,31 @@ +#include + +#ifdef __linux__ + #include "MmapFile.linux.cpp" +#else + #error Invalid platform +#endif + +namespace lightningbolt { + + MmapFile::MmapFile() : impl(std::make_unique()) { + } + + MmapFile::~MmapFile() = default; + + ErrorOr MmapFile::Open(const fs::path& path) { + return impl->Open(path); + } + + void MmapFile::Close() { + return impl->Close(); + } + + u8* MmapFile::GetMapping() const { + return impl->GetMapping(); + } + usize MmapFile::GetMappingSize() const { + return impl->GetMappingSize(); + } + +} // namespace lightningbolt \ No newline at end of file diff --git a/lib/base/MmapFile.hpp b/lib/base/MmapFile.hpp new file mode 100644 index 0000000..afb3f0f --- /dev/null +++ b/lib/base/MmapFile.hpp @@ -0,0 +1,25 @@ +#pragma once +#include +#include + +namespace lightningbolt { + + /// A read-only file opened via memory mapping. + /// On POSIX systems, we use mmap(2). Etc etc. + struct MmapFile { + MmapFile(); + ~MmapFile(); + + // Opens for read-only mode. + ErrorOr Open(const fs::path& path); + void Close(); + + u8* GetMapping() const; + usize GetMappingSize() const; + + private: + struct Impl; + Unique impl; + }; + +} \ No newline at end of file diff --git a/lib/base/MmapFile.linux.cpp b/lib/base/MmapFile.linux.cpp new file mode 100644 index 0000000..711302a --- /dev/null +++ b/lib/base/MmapFile.linux.cpp @@ -0,0 +1,63 @@ +#include "MmapFile.hpp" +#include +#include +#include + +namespace lightningbolt { + + struct MmapFile::Impl { + + ~Impl() { + Close(); + } + + void Close() { + if(mapping) { + munmap(mapping, mappingSize); + mapping = nullptr; + mappingSize = 0; + } + } + + ErrorOr Open(const fs::path& path) { + int fd = open(path.string().c_str(), O_RDONLY); + + // Error opening file. + if(fd == -1) + return std::error_code{errno, std::system_category()}; + + { + auto last = lseek64(fd, 0, SEEK_END); + mappingSize = lseek64(fd, 0, SEEK_CUR); + lseek64(fd, last, SEEK_SET); + } + + mapping = static_cast(mmap(nullptr, mappingSize, PROT_READ, MAP_PRIVATE, fd, 0)); + if(mapping == static_cast(MAP_FAILED)) { + mappingSize = 0; + return std::error_code{errno, std::system_category()}; + } + + // Once the mapping has successfully been created + // we can close the file descriptor instead of needing + // to remember it (the kernel will do so for us.) + close(fd); + + // No error. + return {}; + } + + u8* GetMapping() const { + return mapping; + } + + usize GetMappingSize() const { + return mappingSize; + } + + private: + u8* mapping; + usize mappingSize; + }; + +} // namespace lightningbolt \ No newline at end of file diff --git a/lib/base/OffsetPtr.hpp b/lib/base/OffsetPtr.hpp new file mode 100644 index 0000000..d5b8c2a --- /dev/null +++ b/lib/base/OffsetPtr.hpp @@ -0,0 +1,85 @@ +#pragma once +#include +#include +#include +#include + +namespace lightningbolt { + + namespace detail { + + template + constexpr NativeT* CreatePointerFromAddend(void* BasePointer, OffsetType addend) noexcept { + return std::bit_cast(static_cast(BasePointer) + addend); + } + + } // namespace detail + + /// An "auto-resolving" semi-sweet (/s) pointer type. + /// This is designed to allow resolving offsets in data for + /// games written before 64-bit pointers were common/used at all. + /// This allows looking up data a lot easier :) + /// + /// [NativeT] is the type of data this would point to + /// [OffsetType] is the type of data the "pointer" is repressented as + template + struct OffsetPtr final { + using Type = std::remove_cvref_t; + using Pointer = Type*; + using ConstPointer = const Type*; + + /// Set the offset. Duh! + constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; } + + [[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; } + + [[nodiscard]] constexpr Pointer operator()(void* baseAddr) const noexcept { + // While yucky, it should show problem areas which aren't checking things + // immediately rather than read invalid data that might do much worse. + if(rawOffset == 0) + return nullptr; + + return detail::CreatePointerFromAddend(baseAddr, rawOffset); + } + + template + constexpr OffsetPtr& PtrCast() { + // Safety: The data layout of OffsetPtr<> stays + // the exact same regardless of the result type, therefore + // even though this is *techinically* UB (? using bit_cast it shouldn't be ?), + // this isn't problematic + return *std::bit_cast*>(this); + } + + private: + OffsetType rawOffset; + }; + + /// Like OffsetPtr but for arrays of data + template + struct OffsetArrayPtr final { + using Type = std::remove_cvref_t; + using Pointer = Type*; + using ConstPointer = const Type*; + + using Span = std::span; + + /// Set the offset. Duh! + constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; } + + [[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; } + + [[nodiscard]] constexpr Span operator()(void* baseAddr, OffsetType length) const noexcept { + // While yucky, it should show problem areas which aren't checking things + // immediately rather than read invalid data that might do much worse. + if(rawOffset == 0 || length == 0) + return {}; + + return { detail::CreatePointerFromAddend(baseAddr, rawOffset), length }; + } + + private: + OffsetType rawOffset; + }; + +} // namespace ssxtools::core diff --git a/lib/base/Types.hpp b/lib/base/Types.hpp new file mode 100644 index 0000000..f16ab7d --- /dev/null +++ b/lib/base/Types.hpp @@ -0,0 +1,31 @@ +//! Core types and includes +#pragma once + +#include +#include +#include + +// these are in the global namespace since most libraries +// won't try defining anything like this in the global namespace +// (and I'd like these types to be used globally a lot more anyways) +using u8 = std::uint8_t; +using i8 = std::int8_t; +using u16 = std::uint16_t; +using i16 = std::int16_t; +using u32 = std::uint32_t; +using i32 = std::int32_t; +using u64 = std::uint64_t; +using i64 = std::int64_t; +using usize = std::size_t; +using isize = std::intptr_t; + +namespace lightningbolt { + namespace fs = std::filesystem; + + template > + using Unique = std::unique_ptr; + + template + using Ref = std::shared_ptr; + +} // namespace lightningbolt \ No newline at end of file diff --git a/lib/structs/BoltStructs.hpp b/lib/structs/BoltStructs.hpp new file mode 100644 index 0000000..3c3f171 --- /dev/null +++ b/lib/structs/BoltStructs.hpp @@ -0,0 +1,86 @@ +#include +#include +#include + +namespace lightningbolt { + + namespace elf { + + /// Table entry in the ELF. We use this to create file names. + struct [[gnu::packed]] BoltTableEntry { + /// Pointer to filename. Should be adjusted + u32 filenamePtr; + u16 entryId; // (GID >> 8) | ID + u16 groupId; // (entryId & 0xff00) + }; + + /// Offsets in the ELF to the table. + struct BoltTableOffsets { + u32 usTable; + }; + + /// Convert a address to ELF file offset. + constexpr u32 AddressToElfFileOffset(u32 address) { + constexpr u32 LoadAddress = 0x00100000; + constexpr u32 SectionOffset = 0x80; + return (address - LoadAddress) + SectionOffset; + } + + static constexpr BoltTableOffsets BoltTableOffsets = { + .usTable = AddressToElfFileOffset(0x0033d400) + }; + + } // namespace elf + + struct [[gnu::packed]] BoltGroupEntry { + u32 unk; + u32 fileSize; + u32 fileOffset; + u32 unk3; // name hash? + }; + + struct [[gnu::packed]] BoltGroupDescriptor { + u8 unk; + u8 unk2; + u8 unk3; + u8 entryCount; + + u32 groupSize; + u32 groupOffset; + + u32 EntryCount() { + // Special case: 0x0 == 256 entries. + // I have NO idea why they did it like this, + // this really seems extra when they could have + // used a short field in the same exact space. + if(entryCount == 0x0) + return 256; + return entryCount; + } + + std::span Entries(u8* base) { return { std::bit_cast(base + groupOffset), EntryCount() }; } + }; + + struct [[gnu::packed]] BoltLibraryHeader { + static constexpr char VALID_MAGIC[] = "BOLT\r\n"; + char magic[6]; + + u8 unk; + u8 unk2; + + u8 unk3; + u8 unk4; + u8 unk5; + u8 groupCount; + + u32 libSize; + + std::span GroupDescriptors() { + // The group descriptors are after the primary library header + return { std::bit_cast(this + 1), groupCount }; + } + + inline bool Validate() const { return !std::memcmp(&magic[0], &VALID_MAGIC[0], sizeof(magic)); } + }; + +} // namespace lightningbolt \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..3b618a9 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,80 @@ +#include +#include +#include +#include +#include + +#include + +struct ParsedTableEntry { + std::string_view filename; + u16 index; + u32 gid; +}; + +struct BoltReader { + BoltReader() {} + + ErrorOr OpenBolt(const lightningbolt::fs::path& path) { + auto p = path; + p.replace_filename("SLUS_201.14"); + + if(auto error = elfFile.Open(p); error.HasError()) + return error; + + // Load table entries. + GetTableEntries(); + + // Load the BOLT file + if(auto error = boltFile.Open(path); error.HasError()) + return error; + + return {}; + } + + const std::vector& GetTableEntries() { + if(entryTable.empty()) { + auto* base = elfFile.GetMapping(); + auto* table = std::bit_cast(base + lightningbolt::elf::BoltTableOffsets.usTable); + while(table->filenamePtr != 0x0) { + auto string_offset = lightningbolt::elf::AddressToElfFileOffset(table->filenamePtr); + + ParsedTableEntry te; + te.filename = { std::bit_cast(base + string_offset) }; + te.index = table->entryId; + te.gid = table->groupId; + + if(te.filename == "") + break; + + std::cout << std::format("te: {} {:04x} {:04x}\n", te.filename, te.index, te.gid); + entryTable.emplace_back(te); + + table++; + } + + // The ELF file isn't needed after this so unmap it + elfFile.Close(); + } + return entryTable; + } + + template + void ForEachFile(F f) { + //for()() + } + + private: + std::vector entryTable; + lightningbolt::MmapFile elfFile; + lightningbolt::MmapFile boltFile; +}; + +int main() { + BoltReader reader; + if(auto error = reader.OpenBolt(lightningbolt::fs::current_path() / "ASSETS.BLT"); error.HasError()) { + std::cout << "Error opening Bolt file: " << error.Error(); + } + + return 0; +}