From 3bcc4d242f0b2569ca6b646f8ca37fbb711e07c4 Mon Sep 17 00:00:00 2001 From: modeco80 Date: Sun, 19 Nov 2023 21:28:06 -0500 Subject: [PATCH] Refactor Bolt I/O code into re-usable "bolt" library We now also wrap the result of the decompression code in a ErrorOr<> and have also added a additional error code for decompression failure (since it *is* something that can fail, after all.). This shouldn't really bother performance, if it does, I can just have it take a vector by ref and then make it ErrorOr. --- CMakeLists.txt | 2 + LICENSE | 4 +- lib/base/Types.hpp | 17 ++- lib/bolt/CMakeLists.txt | 14 +++ lib/bolt/Compression.cpp | 69 +++++++++++++ lib/bolt/Compression.hpp | 13 +++ lib/bolt/Errors.cpp | 28 +++++ lib/bolt/Errors.hpp | 19 ++++ lib/bolt/Reader.cpp | 105 +++++++++++++++++++ lib/bolt/Reader.hpp | 33 ++++++ src/main.cpp | 216 ++++----------------------------------- 11 files changed, 320 insertions(+), 200 deletions(-) create mode 100644 lib/bolt/CMakeLists.txt create mode 100644 lib/bolt/Compression.cpp create mode 100644 lib/bolt/Compression.hpp create mode 100644 lib/bolt/Errors.cpp create mode 100644 lib/bolt/Errors.hpp create mode 100644 lib/bolt/Reader.cpp create mode 100644 lib/bolt/Reader.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f8d488a..2b4eb34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ endif() lb_set_alternate_linker() add_subdirectory(lib/base) +add_subdirectory(lib/bolt) add_executable(lightningbolt src/main.cpp @@ -29,6 +30,7 @@ add_executable(lightningbolt target_link_libraries(lightningbolt PRIVATE lb::base + lb::bolt ) lb_target(lightningbolt) diff --git a/LICENSE b/LICENSE index e5cc25f..f28376b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ -Copyright 2023 The LightningBolt Developers +Copyright 2023 The LightningBolt Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/lib/base/Types.hpp b/lib/base/Types.hpp index af1982d..61caba2 100644 --- a/lib/base/Types.hpp +++ b/lib/base/Types.hpp @@ -5,8 +5,10 @@ #pragma once #include -#include #include +#include +#include +#include // these are in the global namespace since most libraries // won't try defining anything like this in the global namespace @@ -32,3 +34,16 @@ namespace lightningbolt { using Ref = std::shared_ptr; } // namespace lightningbolt + +namespace std { + /// Custom formatter for std::error_code. As far as I know, this is + /// Not going to be added in C++23, despite the fact fmt already has a specialization for it, + /// so I have to implement one myself. At least it's not that awful to. + template <> + struct formatter : formatter { + template + inline constexpr auto format(const error_code& ec, FormatContext& ctx) const { + return std::format_to(ctx.out(), "{} [{}.{}]", ec.message(), ec.category().name(), ec.value()); + } + }; +} // namespace std diff --git a/lib/bolt/CMakeLists.txt b/lib/bolt/CMakeLists.txt new file mode 100644 index 0000000..5a5eba6 --- /dev/null +++ b/lib/bolt/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2023 The LightningBolt Authors +# SPDX-License-Identifier: MIT +add_library(lb_bolt + Errors.cpp + Compression.cpp + Reader.cpp +) + +target_link_libraries(lb_bolt PUBLIC + lb::base +) + +lb_target(lb_bolt) +add_library(lb::bolt ALIAS lb_bolt) diff --git a/lib/bolt/Compression.cpp b/lib/bolt/Compression.cpp new file mode 100644 index 0000000..bd9fb68 --- /dev/null +++ b/lib/bolt/Compression.cpp @@ -0,0 +1,69 @@ +#include +#include "bolt/Errors.hpp" + +namespace lightningbolt { + + ErrorOr> BoltDecompress(u8* input, usize decompressedSize) { + u8* inptr = input; + std::vector res; + + res.resize(decompressedSize); + + u8* pOut = res.data(); + u8* pEnd = res.data() + decompressedSize; + + i32 iVar3 = 0; + i32 iVar6 = 0; + u32 uVar5 = 0; + u8* pbVar4 = nullptr; // outrun + bool bVar1; + + while(pOut < pEnd) { + auto uVar7 = *inptr++; + + if(uVar7 < 128) { // lookback/run? + iVar3 = iVar3 + uVar5 * 8 + ((int)(uVar7 & 0x70) >> 4); + pbVar4 = pOut + -(iVar6 * 0x10 + (uVar7 & 0xf) + 1); + iVar6 = iVar3 + 1; + + if(iVar3 != -2) { + do { + *pOut = *pbVar4; + pbVar4 = pbVar4 + 1; + bVar1 = iVar6 != 0; + pOut = pOut + 1; + iVar6 = iVar6 + -1; + } while(bVar1); + } + iVar6 = 0; + iVar3 = 0; + uVar5 = 0; + } else if(uVar7 < 144) { // literal copy from stream + iVar3 = uVar5 * 0x10 + (uVar7 & 0xf) + 1; + while(iVar3 != 0) { + iVar3 = iVar3 + -1; + *pOut = *inptr++; + pOut = pOut + 1; + } + iVar3 = 0; + uVar5 = 0; + } else if(uVar7 < 160) { + uVar5 = uVar7 & 3; + iVar3 = iVar3 + 1; + iVar6 = (int)(uVar7 & 0xc) >> 2; + } else if(uVar7 < 192) { + uVar5 = uVar5 * 0x20 + (uVar7 & 0x1f); + iVar3 = iVar3 + 1; + } else { + iVar3 = iVar3 + 1; + iVar6 = iVar6 * 0x40 + (uVar7 & 0x3f); + } + } + + if(pOut != pEnd) + return std::make_error_code(BoltErrc::DecompressionError); + + return res; + } + +} // namespace lightningbolt diff --git a/lib/bolt/Compression.hpp b/lib/bolt/Compression.hpp new file mode 100644 index 0000000..5e3ab54 --- /dev/null +++ b/lib/bolt/Compression.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include +#include + +namespace lightningbolt { + + /// Decompress a compressed BOLT file into a new allocated buffer. + /// TODO: this should probably be passed a file object, but for now, + /// just passing bare parameters works. + ErrorOr> BoltDecompress(u8* input, usize decompressedSize); + +} // namespace lightningbolt diff --git a/lib/bolt/Errors.cpp b/lib/bolt/Errors.cpp new file mode 100644 index 0000000..e3003a6 --- /dev/null +++ b/lib/bolt/Errors.cpp @@ -0,0 +1,28 @@ +#include + +namespace lightningbolt { + + namespace { + struct BoltErrorCategory : std::error_category { + const char* name() const noexcept override { return "bolt"; } + std::string message(i32 ev) const override { + switch(static_cast(ev)) { + case BoltErrc::InvalidMagic: return "Invalid bolt library file magic"; + case BoltErrc::DecompressionError: return "Error during BOLT file decompression"; + default: return "Unknown error"; + } + } + }; + + } // namespace + + const BoltErrorCategory boltCategory {}; + +} // namespace lightningbolt + +namespace std { + std::error_code make_error_code(::lightningbolt::BoltErrc errc) { + return { static_cast(errc), ::lightningbolt::boltCategory }; + } + +} // namespace std diff --git a/lib/bolt/Errors.hpp b/lib/bolt/Errors.hpp new file mode 100644 index 0000000..308810b --- /dev/null +++ b/lib/bolt/Errors.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace lightningbolt { + + /// Error codes for BOLT + enum class BoltErrc { + InvalidMagic = 1, + DecompressionError + }; + +} // namespace lightningbolt + +namespace std { + template <> + struct is_error_code_enum<::lightningbolt::BoltErrc> : true_type {}; + std::error_code make_error_code(::lightningbolt::BoltErrc errc); +} // namespace std diff --git a/lib/bolt/Reader.cpp b/lib/bolt/Reader.cpp new file mode 100644 index 0000000..b091b3a --- /dev/null +++ b/lib/bolt/Reader.cpp @@ -0,0 +1,105 @@ + + +#include +#include + +#include +#include "base/ErrorOr.hpp" + +#if 0 + +#endif + +namespace lightningbolt { + + struct BoltReader::Impl { + + ErrorOr OpenBolt(const fs::path& path) { + // Load the BOLT file + if(auto error = boltFile.Open(path); error.HasError()) + return error; + + lib = std::bit_cast(boltFile.GetMapping()); + + if(!lib->Validate()) + return std::make_error_code(BoltErrc::InvalidMagic); + + auto p = path; + p.replace_filename("SLUS_201.14"); + + if(auto error = elfFile.Open(p); error.HasError()) + return error; + + // Load table entries. + GetTableEntries(); + + return {}; + } + + const std::vector& GetTableEntries() { + if(entryTable.empty()) { + auto* base = elfFile.GetMapping(); + auto* table = std::bit_cast(base + elf::BoltTableOffsets.usTable); + while(table->filenamePtr != 0x0) { + auto string_offset = elf::AddressToElfFileOffset(table->filenamePtr); + + BoltReader::File te; + te.filename = { std::bit_cast(base + string_offset) }; + te.index = table->entryId; + te.gid = table->groupId; + + if(te.filename == "") + break; + + entryTable.emplace_back(te); + + table++; + } + } + return entryTable; + } + + template + void ForEachFile(F f) { + for(auto& file : entryTable) { + if(file.uncompressedData == nullptr) { + auto gid = (file.gid >> 8); + auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping()); + auto size = entries[file.index & 0x00ff].fileSize; + auto offset = entries[file.index & 0x00ff].fileOffset; + + file.compressed = !(entries[file.index & 0x00ff].unk & 0x8); + + file.uncompressedData = std::bit_cast(boltFile.GetMapping() + offset); + file.uncompressedSize = size; + } + + if(!f(file)) + break; + } + } + + private: + std::vector entryTable; + lightningbolt::MmapFile elfFile; + lightningbolt::MmapFile boltFile; + + lightningbolt::BoltLibraryHeader* lib; + }; + + BoltReader::BoltReader(): + impl(std::make_unique()) {} + + BoltReader::~BoltReader() = default; + + ErrorOr BoltReader::OpenBolt(const fs::path& path) { + return impl->OpenBolt(path); + } + + void BoltReader::ForEachFile(std::function f) { + impl->ForEachFile([&f](auto& file) { + return f(file); + }); + } + +} // namespace lightningbolt diff --git a/lib/bolt/Reader.hpp b/lib/bolt/Reader.hpp new file mode 100644 index 0000000..033e446 --- /dev/null +++ b/lib/bolt/Reader.hpp @@ -0,0 +1,33 @@ +#pragma once +#include +#include +#include + +namespace lightningbolt { + + struct BoltReader { + struct File { + std::string_view filename; + u16 index; + u16 gid; + + bool compressed; + + u8* uncompressedData { nullptr }; + usize uncompressedSize; + + }; + + BoltReader(); + ~BoltReader(); + + ErrorOr OpenBolt(const fs::path& path); + + void ForEachFile(std::function f); + + private: + struct Impl; + Unique impl; + }; + +} // namespace lightningbolt diff --git a/src/main.cpp b/src/main.cpp index 01ad3e1..845a52f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,220 +1,38 @@ // Copyright 2023 The LightningBolt Authors // SPDX-License-Identifier: MIT -#include #include #include #include #include -#include -#include #include +#include +#include + namespace fs = std::filesystem; -std::vector BoltDecompress(u8* input, usize decompressedSize) { - u8* inptr = input; - std::vector res; +fs::path ConvertBoltPath(const std::string_view filename) { + // Create a mutable copy of the filename + std::string s = { filename.data(), filename.size() }; - res.resize(decompressedSize); + for(auto& c : s) + if(c == '\\') + c = '/'; - u8* pOut = res.data(); - u8* pEnd = res.data() + decompressedSize; - - i32 iVar3 = 0; - i32 iVar6 = 0; - u32 uVar5 = 0; - u8* pbVar4 = nullptr; // outrun - bool bVar1; - - while(pOut < pEnd) { - auto uVar7 = *inptr++; - - if(uVar7 < 128) { // lookback/run? - iVar3 = iVar3 + uVar5 * 8 + ((int)(uVar7 & 0x70) >> 4); - pbVar4 = pOut + -(iVar6 * 0x10 + (uVar7 & 0xf) + 1); - iVar6 = iVar3 + 1; - - if(iVar3 != -2) { - do { - *pOut = *pbVar4; - pbVar4 = pbVar4 + 1; - bVar1 = iVar6 != 0; - pOut = pOut + 1; - iVar6 = iVar6 + -1; - } while(bVar1); - } - iVar6 = 0; - iVar3 = 0; - uVar5 = 0; - } else if(uVar7 < 144) { // literal copy from stream - iVar3 = uVar5 * 0x10 + (uVar7 & 0xf) + 1; - while(iVar3 != 0) { - iVar3 = iVar3 + -1; - *pOut = *inptr++; - pOut = pOut + 1; - } - iVar3 = 0; - uVar5 = 0; - } else if(uVar7 < 160) { - uVar5 = uVar7 & 3; - iVar3 = iVar3 + 1; - iVar6 = (int)(uVar7 & 0xc) >> 2; - } else if(uVar7 < 192) { - uVar5 = uVar5 * 0x20 + (uVar7 & 0x1f); - iVar3 = iVar3 + 1; - } else { - iVar3 = iVar3 + 1; - iVar6 = iVar6 * 0x40 + (uVar7 & 0x3f); - } - } - - return res; + return fs::current_path() / "ASSETS" / fs::path(s); } -// error code boilerplate. This should be moved later -enum class BoltErrc { InvalidMagic = 1 }; - -struct BoltErrorCategory : std::error_category { - const char* name() const noexcept override { return "boltio"; } - std::string message(i32 ev) const override { - switch(static_cast(ev)) { - case BoltErrc::InvalidMagic: return "invalid bolt library file magic"; - default: return "unknown error"; - } - } -}; - -const BoltErrorCategory boltCategory {}; - -namespace std { - template <> - struct is_error_code_enum : true_type {}; - - std::error_code make_error_code(BoltErrc errc) { - return { static_cast(errc), boltCategory }; - } - - /// Custom formatter for std::error_code. As far as I know, this is - /// Not going to be added in C++23, despite the fact fmt already has a specialization for it, - /// so I have to implement one myself. At least it's not that awful to. - template <> - struct formatter : formatter { - template - inline auto format(const error_code& ec, FormatContext& ctx) const { - return std::format_to(ctx.out(), "{} [{}.{}]", ec.message(), ec.category().name(), ec.value()); - } - }; -} // namespace std - -struct BoltReader { - struct ParsedTableEntry { - std::string_view filename; - u16 index; - u16 gid; - - bool compressed; - - u8* uncompressedData { nullptr }; - usize uncompressedSize; - - fs::path pathify() { - // Create a mutable copy of the filename - std::string s = { filename.data(), filename.size() }; - - for(auto& c : s) - if(c == '\\') - c = '/'; - - return fs::current_path() / "ASSETS" / fs::path(s); - } - }; - - BoltReader() {} - - ErrorOr OpenBolt(const fs::path& path) { - // Load the BOLT file - if(auto error = boltFile.Open(path); error.HasError()) - return error; - - lib = std::bit_cast(boltFile.GetMapping()); - - if(!lib->Validate()) - return std::make_error_code(BoltErrc::InvalidMagic); - - auto p = path; - p.replace_filename("SLUS_201.14"); - - if(auto error = elfFile.Open(p); error.HasError()) - return error; - - // Load table entries. - GetTableEntries(); - - return {}; - } - - const std::vector& GetTableEntries() { - if(entryTable.empty()) { - auto* base = elfFile.GetMapping(); - auto* table = std::bit_cast(base + lightningbolt::elf::BoltTableOffsets.usTable); - while(table->filenamePtr != 0x0) { - auto string_offset = lightningbolt::elf::AddressToElfFileOffset(table->filenamePtr); - - ParsedTableEntry te; - te.filename = { std::bit_cast(base + string_offset) }; - te.index = table->entryId; - te.gid = table->groupId; - - if(te.filename == "") - break; - - entryTable.emplace_back(te); - - table++; - } - } - return entryTable; - } - - template - void ForEachFile(F f) { - for(auto& file : entryTable) { - if(file.uncompressedData == nullptr) { - auto gid = (file.gid >> 8); - auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping()); - auto size = entries[file.index & 0x00ff].fileSize; - auto offset = entries[file.index & 0x00ff].fileOffset; - - file.compressed = !(entries[file.index & 0x00ff].unk & 0x8); - - file.uncompressedData = std::bit_cast(boltFile.GetMapping() + offset); - file.uncompressedSize = size; - } - - if(!f(file)) - break; - } - } - - private: - std::vector entryTable; - lightningbolt::MmapFile elfFile; - lightningbolt::MmapFile boltFile; - - lightningbolt::BoltLibraryHeader* lib; -}; - int main() { - BoltReader reader; + lightningbolt::BoltReader reader; if(auto error = reader.OpenBolt(fs::current_path() / "ASSETS.BLT"); error.HasError()) { std::cout << std::format("Error initalizing bolt reader: {}\n", error.Error()); } - reader.ForEachFile([](BoltReader::ParsedTableEntry& ent) { + reader.ForEachFile([](lightningbolt::BoltReader::File& ent) { std::cout << std::format("File: {}", ent.filename); - auto p = ent.pathify(); + auto p = ConvertBoltPath(ent.filename); auto pathonly = p; pathonly.remove_filename(); @@ -232,9 +50,13 @@ int main() { if(ent.compressed) { std::cout << std::format(", compressed ({} bytes uncompressed)", ent.uncompressedSize); - auto decompressed = BoltDecompress(ent.uncompressedData, ent.uncompressedSize); + auto decompressed = lightningbolt::BoltDecompress(ent.uncompressedData, ent.uncompressedSize); + if(decompressed.HasError()) { + std::cout << std::format("Error during decompression: {}\n", decompressed.Error()); + return false; + } - ofs.write((char*)decompressed.data(), ent.uncompressedSize); + ofs.write((char*)decompressed.Value().data(), ent.uncompressedSize); std::cout << std::format(", written to \"{}\"\n", p.string()); } else { std::cout << std::format(", uncompressed ({} bytes)", ent.uncompressedSize);