GameModding
/
lightningbolt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial prototype - reads the TOC from the ELF and maps the BOLT file

This commit is contained in:
Lily Tsuru 2023-11-19 06:19:15 -05:00
commit d7a48f39b4
14 changed files with 663 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.clang-format Executable file
View File

@ -0,0 +1,46 @@
# .clang-format for native code portion
BasedOnStyle: Google
# force T* or T&
DerivePointerAlignment: false
PointerAlignment: Left
TabWidth: 4
IndentWidth: 4
UseTab: Always
IndentPPDirectives: BeforeHash
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortFunctionsOnASingleLine: InlineOnly
AllowShortIfStatementsOnASingleLine: Never
AllowShortLoopsOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: true
BinPackArguments: true
BinPackParameters: true
BreakConstructorInitializers: BeforeColon
BreakStringLiterals: false
ColumnLimit: 150
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ContinuationIndentWidth: 0
# turning this on causes major issues with initializer lists
Cpp11BracedListStyle: false
SpaceBeforeCpp11BracedList: true
FixNamespaceComments: true
NamespaceIndentation: All
ReflowComments: true
SortIncludes: CaseInsensitive
SortUsingDeclarations: true
SpacesInSquareBrackets: false
SpaceBeforeParens: Never
SpacesBeforeTrailingComments: 1

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
build/
/out
.cache/
/compile_commands.json

34
CMakeLists.txt Normal file
View File

@ -0,0 +1,34 @@
cmake_minimum_required(VERSION 3.15)
project(lightningbolt
LANGUAGES CXX
)
list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
include(Policies)
include(ProjectFuncs)
# default linker
if(NOT LIGHTNINGBOLT_LINKER AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(LIGHTNINGBOLT_LINKER "lld")
elseif(NOT LIGHTNINGBOLT_LINKER)
set(LIGHTNINGBOLT_LINKER "bfd")
endif()
lb_set_alternate_linker()
add_subdirectory(lib/base)
# third party vendor dependencies
#add_subdirectory(third_party)
add_executable(lightningbolt
src/main.cpp
)
target_link_libraries(lightningbolt PRIVATE
lb::base
)
lb_target(lightningbolt)

22
cmake/Policies.cmake Normal file
View File

@ -0,0 +1,22 @@
# CMake policy configuration
# Macro to enable new CMake policy.
# Makes this file a *LOT* shorter.
macro (_new_cmake_policy policy)
if(POLICY ${policy})
#message(STATUS "Enabling new policy ${policy}")
cmake_policy(SET ${policy} NEW)
endif()
endmacro()
_new_cmake_policy(CMP0026) # CMake 3.0: Disallow use of the LOCATION property for build targets.
_new_cmake_policy(CMP0042) # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name
_new_cmake_policy(CMP0046) # warn about non-existent dependencies
_new_cmake_policy(CMP0048) # CMake 3.0+: project() command now maintains VERSION
_new_cmake_policy(CMP0054) # CMake 3.1: Only interpret if() arguments as variables or keywords when unquoted.
_new_cmake_policy(CMP0056) # try_compile() linker flags
_new_cmake_policy(CMP0066) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE
_new_cmake_policy(CMP0067) # CMake 3.8: try_compile(): honor language standard variables (like C++11)
_new_cmake_policy(CMP0068) # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`.
_new_cmake_policy(CMP0075) # CMake 3.12+: Include file check macros honor `CMAKE_REQUIRED_LIBRARIES`
_new_cmake_policy(CMP0077) # CMake 3.13+: option() honors normal variables.

59
cmake/ProjectFuncs.cmake Normal file
View File

@ -0,0 +1,59 @@
function(lb_target target)
target_compile_definitions(${target} PRIVATE "$<$<CONFIG:DEBUG>:LIGHTNINGBOLT_DEBUG>")
#target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR})
target_compile_features(${target} PRIVATE cxx_std_20)
target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/lib ${CMAKE_CURRENT_BINARY_DIR})
set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS -Wall -Wextra)
if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS})
# If on Release use link-time optimizations.
# On clang we use ThinLTO for even better build performance.
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto=thin)
target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto=thin)
else()
set(_LIGHTNINGBOLT_CORE_COMPILE_ARGS ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -flto)
target_link_options(${target} PRIVATE -fuse-ld=${LIGHTNINGBOLT_LINKER} -flto)
endif()
target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS})
endif()
if("asan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
# Error if someone's trying to mix asan and tsan together,
# they aren't compatible.
if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
message(FATAL_ERROR "ASAN and TSAN cannot be used together.")
endif()
message(STATUS "Enabling ASAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=address)
target_link_libraries(${target} PRIVATE -fsanitize=address)
endif()
if("tsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
message(STATUS "Enabling TSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=thread)
target_link_libraries(${target} PRIVATE -fsanitize=thread)
endif()
if("ubsan" IN_LIST LIGHTNINGBOLT_BUILD_FEATURES)
message(STATUS "Enabling UBSAN for target ${target} because it was in LIGHTNINGBOLT_BUILD_FEATURES")
target_compile_options(${target} PRIVATE ${_LIGHTNINGBOLT_CORE_COMPILE_ARGS} -fsanitize=undefined)
target_link_libraries(${target} PRIVATE -fsanitize=undefined)
endif()
endfunction()
function(lb_set_alternate_linker)
find_program(LINKER_EXECUTABLE ld.${LIGHTNINGBOLT_LINKER} ${COLLABVM_LINKER})
if(LINKER_EXECUTABLE)
message(STATUS "Using ${LIGHTNINGBOLT_LINKER} as linker")
else()
message(FATAL_ERROR "Linker ${LIGHTNINGBOLT_LINKER} does not exist on your system. Please specify one which does or omit this option from your configure command.")
endif()
endfunction()

6
lib/base/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
add_library(lb_base
MmapFile.cpp
)
lb_target(lb_base)
add_library(lb::base ALIAS lb_base)

90
lib/base/ErrorOr.hpp Normal file
View File

@ -0,0 +1,90 @@
#pragma once
#include <optional>
#include <system_error>
#include <variant>
template <class T>
struct ErrorOr {
private:
using Variant = std::variant<T, std::error_code>;
public:
constexpr static auto IsVoid = false;
ErrorOr() = default;
ErrorOr(const ErrorOr&) = default;
ErrorOr(ErrorOr&&) = default;
ErrorOr(const T& v) : maybeVariant(v) {}
ErrorOr(const std::error_code& v) : maybeVariant(v) {}
ErrorOr& operator=(const T& value) {
maybeVariant = value;
return *this;
}
ErrorOr& operator=(const std::error_code& ec) {
maybeVariant = ec;
return *this;
}
bool HasValue() const {
if(maybeVariant)
return std::holds_alternative<T>(*maybeVariant);
// No value.
return false;
}
bool HasError() const {
if(maybeVariant)
return std::holds_alternative<std::error_code>(*maybeVariant);
// No error. Possibly in the default constructed state
return false;
}
// ASSERT TODO
T& Value() { return std::get<T>(maybeVariant.value()); }
const T& Value() const { return std::get<T>(maybeVariant.value()); }
operator T&() { return Value(); }
operator const T&() { return Value(); }
std::error_code& Error() { return std::get<std::error_code>(maybeVariant.value()); }
const std::error_code& Error() const { return std::get<std::error_code>(maybeVariant.value()); }
operator std::error_code&() { return Error(); }
operator const std::error_code&() const { return Error(); }
private:
std::optional<Variant> maybeVariant;
};
template <>
struct ErrorOr<void> {
constexpr static auto IsVoid = true;
ErrorOr() = default;
ErrorOr(const ErrorOr&) = default;
ErrorOr(ErrorOr&&) = default;
ErrorOr(const std::error_code& v) : maybeEc(v) {}
ErrorOr& operator=(const std::error_code& ec) {
maybeEc = ec;
return *this;
}
bool HasError() const { return maybeEc.has_value(); }
std::error_code& Error() { return maybeEc.value(); }
const std::error_code& Error() const { return maybeEc.value(); }
operator std::error_code&() { return Error(); }
operator const std::error_code&() const { return Error(); }
private:
std::optional<std::error_code> maybeEc;
};

31
lib/base/MmapFile.cpp Normal file
View File

@ -0,0 +1,31 @@
#include <base/MmapFile.hpp>
#ifdef __linux__
#include "MmapFile.linux.cpp"
#else
#error Invalid platform
#endif
namespace lightningbolt {
MmapFile::MmapFile() : impl(std::make_unique<Impl>()) {
}
MmapFile::~MmapFile() = default;
ErrorOr<void> MmapFile::Open(const fs::path& path) {
return impl->Open(path);
}
void MmapFile::Close() {
return impl->Close();
}
u8* MmapFile::GetMapping() const {
return impl->GetMapping();
}
usize MmapFile::GetMappingSize() const {
return impl->GetMappingSize();
}
} // namespace lightningbolt

25
lib/base/MmapFile.hpp Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <base/Types.hpp>
#include <base/ErrorOr.hpp>
namespace lightningbolt {
/// A read-only file opened via memory mapping.
/// On POSIX systems, we use mmap(2). Etc etc.
struct MmapFile {
MmapFile();
~MmapFile();
// Opens for read-only mode.
ErrorOr<void> Open(const fs::path& path);
void Close();
u8* GetMapping() const;
usize GetMappingSize() const;
private:
struct Impl;
Unique<Impl> impl;
};
}

63
lib/base/MmapFile.linux.cpp Normal file
View File

@ -0,0 +1,63 @@
#include "MmapFile.hpp"
#include <fcntl.h>
#include <sys/file.h>
#include <sys/mman.h>
namespace lightningbolt {
struct MmapFile::Impl {
~Impl() {
Close();
}
void Close() {
if(mapping) {
munmap(mapping, mappingSize);
mapping = nullptr;
mappingSize = 0;
}
}
ErrorOr<void> Open(const fs::path& path) {
int fd = open(path.string().c_str(), O_RDONLY);
// Error opening file.
if(fd == -1)
return std::error_code{errno, std::system_category()};
{
auto last = lseek64(fd, 0, SEEK_END);
mappingSize = lseek64(fd, 0, SEEK_CUR);
lseek64(fd, last, SEEK_SET);
}
mapping = static_cast<u8*>(mmap(nullptr, mappingSize, PROT_READ, MAP_PRIVATE, fd, 0));
if(mapping == static_cast<u8*>(MAP_FAILED)) {
mappingSize = 0;
return std::error_code{errno, std::system_category()};
}
// Once the mapping has successfully been created
// we can close the file descriptor instead of needing
// to remember it (the kernel will do so for us.)
close(fd);
// No error.
return {};
}
u8* GetMapping() const {
return mapping;
}
usize GetMappingSize() const {
return mappingSize;
}
private:
u8* mapping;
usize mappingSize;
};
} // namespace lightningbolt

85
lib/base/OffsetPtr.hpp Normal file
View File

@ -0,0 +1,85 @@
#pragma once
#include <bit>
#include <base/Types.hpp>
#include <type_traits>
#include <span>
namespace lightningbolt {
namespace detail {
template <class NativeT, class OffsetType>
constexpr NativeT* CreatePointerFromAddend(void* BasePointer, OffsetType addend) noexcept {
return std::bit_cast<NativeT*>(static_cast<char*>(BasePointer) + addend);
}
} // namespace detail
/// An "auto-resolving" semi-sweet (/s) pointer type.
/// This is designed to allow resolving offsets in data for
/// games written before 64-bit pointers were common/used at all.
/// This allows looking up data a lot easier :)
///
/// [NativeT] is the type of data this would point to
/// [OffsetType] is the type of data the "pointer" is repressented as
template <class NativeT, class OffsetType = u32>
struct OffsetPtr final {
using Type = std::remove_cvref_t<NativeT>;
using Pointer = Type*;
using ConstPointer = const Type*;
/// Set the offset. Duh!
constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; }
[[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; }
[[nodiscard]] constexpr Pointer operator()(void* baseAddr) const noexcept {
// While yucky, it should show problem areas which aren't checking things
// immediately rather than read invalid data that might do much worse.
if(rawOffset == 0)
return nullptr;
return detail::CreatePointerFromAddend<Type>(baseAddr, rawOffset);
}
template<class NativeU>
constexpr OffsetPtr<NativeU, OffsetType>& PtrCast() {
// Safety: The data layout of OffsetPtr<> stays
// the exact same regardless of the result type, therefore
// even though this is *techinically* UB (? using bit_cast it shouldn't be ?),
// this isn't problematic
return *std::bit_cast<OffsetPtr<NativeU, OffsetType>*>(this);
}
private:
OffsetType rawOffset;
};
/// Like OffsetPtr<T> but for arrays of data
template <class NativeT, class OffsetType = u32>
struct OffsetArrayPtr final {
using Type = std::remove_cvref_t<NativeT>;
using Pointer = Type*;
using ConstPointer = const Type*;
using Span = std::span<NativeT>;
/// Set the offset. Duh!
constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; }
[[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; }
[[nodiscard]] constexpr Span operator()(void* baseAddr, OffsetType length) const noexcept {
// While yucky, it should show problem areas which aren't checking things
// immediately rather than read invalid data that might do much worse.
if(rawOffset == 0 || length == 0)
return {};
return { detail::CreatePointerFromAddend<Type>(baseAddr, rawOffset), length };
}
private:
OffsetType rawOffset;
};
} // namespace ssxtools::core

31
lib/base/Types.hpp Normal file
View File

@ -0,0 +1,31 @@
//! Core types and includes
#pragma once
#include <cstdint>
#include <memory>
#include <filesystem>
// these are in the global namespace since most libraries
// won't try defining anything like this in the global namespace
// (and I'd like these types to be used globally a lot more anyways)
using u8 = std::uint8_t;
using i8 = std::int8_t;
using u16 = std::uint16_t;
using i16 = std::int16_t;
using u32 = std::uint32_t;
using i32 = std::int32_t;
using u64 = std::uint64_t;
using i64 = std::int64_t;
using usize = std::size_t;
using isize = std::intptr_t;
namespace lightningbolt {
namespace fs = std::filesystem;
template <class T, class Deleter = std::default_delete<T>>
using Unique = std::unique_ptr<T, Deleter>;
template <class T>
using Ref = std::shared_ptr<T>;
} // namespace lightningbolt

86
lib/structs/BoltStructs.hpp Normal file
View File

@ -0,0 +1,86 @@
#include <base/Types.hpp>
#include <cstring>
#include <span>
namespace lightningbolt {
namespace elf {
/// Table entry in the ELF. We use this to create file names.
struct [[gnu::packed]] BoltTableEntry {
/// Pointer to filename. Should be adjusted
u32 filenamePtr;
u16 entryId; // (GID >> 8) | ID
u16 groupId; // (entryId & 0xff00)
};
/// Offsets in the ELF to the table.
struct BoltTableOffsets {
u32 usTable;
};
/// Convert a address to ELF file offset.
constexpr u32 AddressToElfFileOffset(u32 address) {
constexpr u32 LoadAddress = 0x00100000;
constexpr u32 SectionOffset = 0x80;
return (address - LoadAddress) + SectionOffset;
}
static constexpr BoltTableOffsets BoltTableOffsets = {
.usTable = AddressToElfFileOffset(0x0033d400)
};
} // namespace elf
struct [[gnu::packed]] BoltGroupEntry {
u32 unk;
u32 fileSize;
u32 fileOffset;
u32 unk3; // name hash?
};
struct [[gnu::packed]] BoltGroupDescriptor {
u8 unk;
u8 unk2;
u8 unk3;
u8 entryCount;
u32 groupSize;
u32 groupOffset;
u32 EntryCount() {
// Special case: 0x0 == 256 entries.
// I have NO idea why they did it like this,
// this really seems extra when they could have
// used a short field in the same exact space.
if(entryCount == 0x0)
return 256;
return entryCount;
}
std::span<BoltGroupEntry> Entries(u8* base) { return { std::bit_cast<BoltGroupEntry*>(base + groupOffset), EntryCount() }; }
};
struct [[gnu::packed]] BoltLibraryHeader {
static constexpr char VALID_MAGIC[] = "BOLT\r\n";
char magic[6];
u8 unk;
u8 unk2;
u8 unk3;
u8 unk4;
u8 unk5;
u8 groupCount;
u32 libSize;
std::span<BoltGroupDescriptor> GroupDescriptors() {
// The group descriptors are after the primary library header
return { std::bit_cast<BoltGroupDescriptor*>(this + 1), groupCount };
}
inline bool Validate() const { return !std::memcmp(&magic[0], &VALID_MAGIC[0], sizeof(magic)); }
};
} // namespace lightningbolt

80
src/main.cpp Normal file
View File

@ -0,0 +1,80 @@
#include <base/MmapFile.hpp>
#include <filesystem>
#include <structs/BoltStructs.hpp>
#include <vector>
#include <format>
#include <iostream>
struct ParsedTableEntry {
std::string_view filename;
u16 index;
u32 gid;
};
struct BoltReader {
BoltReader() {}
ErrorOr<void> OpenBolt(const lightningbolt::fs::path& path) {
auto p = path;
p.replace_filename("SLUS_201.14");
if(auto error = elfFile.Open(p); error.HasError())
return error;
// Load table entries.
GetTableEntries();
// Load the BOLT file
if(auto error = boltFile.Open(path); error.HasError())
return error;
return {};
}
const std::vector<ParsedTableEntry>& GetTableEntries() {
if(entryTable.empty()) {
auto* base = elfFile.GetMapping();
auto* table = std::bit_cast<lightningbolt::elf::BoltTableEntry*>(base + lightningbolt::elf::BoltTableOffsets.usTable);
while(table->filenamePtr != 0x0) {
auto string_offset = lightningbolt::elf::AddressToElfFileOffset(table->filenamePtr);
ParsedTableEntry te;
te.filename = { std::bit_cast<char*>(base + string_offset) };
te.index = table->entryId;
te.gid = table->groupId;
if(te.filename == "")
break;
std::cout << std::format("te: {} {:04x} {:04x}\n", te.filename, te.index, te.gid);
entryTable.emplace_back(te);
table++;
}
// The ELF file isn't needed after this so unmap it
elfFile.Close();
}
return entryTable;
}
template<class F>
void ForEachFile(F f) {
//for()()
}
private:
std::vector<ParsedTableEntry> entryTable;
lightningbolt::MmapFile elfFile;
lightningbolt::MmapFile boltFile;
};
int main() {
BoltReader reader;
if(auto error = reader.OpenBolt(lightningbolt::fs::current_path() / "ASSETS.BLT"); error.HasError()) {
std::cout << "Error opening Bolt file: " << error.Error();
}
return 0;
}