GameModding
/
lightningbolt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor Bolt I/O code into re-usable "bolt" library 

We now also wrap the result of the decompression code in a
ErrorOr<> and have also added a additional error code for
decompression failure (since it *is* something that can fail,
after all.). This shouldn't really bother performance, if it does,
I can just have it take a vector by ref and then make it ErrorOr<void>.
This commit is contained in:
Lily Tsuru 2023-11-19 21:28:06 -05:00
parent a8b4aef168
commit 3bcc4d242f
11 changed files with 320 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -22,6 +22,7 @@ endif()
lb_set_alternate_linker()
add_subdirectory(lib/base)
add_subdirectory(lib/bolt)
add_executable(lightningbolt
src/main.cpp
@ -29,6 +30,7 @@ add_executable(lightningbolt
target_link_libraries(lightningbolt PRIVATE
lb::base
lb::bolt
)
lb_target(lightningbolt)

2
LICENSE
View File

@ -1,4 +1,4 @@
Copyright 2023 The LightningBolt Developers
Copyright 2023 The LightningBolt Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

17
lib/base/Types.hpp
View File

@ -5,8 +5,10 @@
#pragma once
#include <cstdint>
#include <memory>
#include <filesystem>
#include <memory>
#include <format>
#include <system_error>
// these are in the global namespace since most libraries
// won't try defining anything like this in the global namespace
@ -32,3 +34,16 @@ namespace lightningbolt {
using Ref = std::shared_ptr<T>;
} // namespace lightningbolt
namespace std {
/// Custom formatter for std::error_code. As far as I know, this is
/// Not going to be added in C++23, despite the fact fmt already has a specialization for it,
/// so I have to implement one myself. At least it's not that awful to.
template <>
struct formatter<error_code> : formatter<string_view> {
template <typename FormatContext>
inline constexpr auto format(const error_code& ec, FormatContext& ctx) const {
return std::format_to(ctx.out(), "{} [{}.{}]", ec.message(), ec.category().name(), ec.value());
}
};
} // namespace std

14
lib/bolt/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
# Copyright 2023 The LightningBolt Authors
# SPDX-License-Identifier: MIT
add_library(lb_bolt
Errors.cpp
Compression.cpp
Reader.cpp
)
target_link_libraries(lb_bolt PUBLIC
lb::base
)
lb_target(lb_bolt)
add_library(lb::bolt ALIAS lb_bolt)

69
lib/bolt/Compression.cpp Normal file
View File

@ -0,0 +1,69 @@
#include <bolt/Compression.hpp>
#include "bolt/Errors.hpp"
namespace lightningbolt {
ErrorOr<std::vector<u8>> BoltDecompress(u8* input, usize decompressedSize) {
u8* inptr = input;
std::vector<u8> res;
res.resize(decompressedSize);
u8* pOut = res.data();
u8* pEnd = res.data() + decompressedSize;
i32 iVar3 = 0;
i32 iVar6 = 0;
u32 uVar5 = 0;
u8* pbVar4 = nullptr; // outrun
bool bVar1;
while(pOut < pEnd) {
auto uVar7 = *inptr++;
if(uVar7 < 128) { // lookback/run?
iVar3 = iVar3 + uVar5 * 8 + ((int)(uVar7 & 0x70) >> 4);
pbVar4 = pOut + -(iVar6 * 0x10 + (uVar7 & 0xf) + 1);
iVar6 = iVar3 + 1;
if(iVar3 != -2) {
do {
*pOut = *pbVar4;
pbVar4 = pbVar4 + 1;
bVar1 = iVar6 != 0;
pOut = pOut + 1;
iVar6 = iVar6 + -1;
} while(bVar1);
}
iVar6 = 0;
iVar3 = 0;
uVar5 = 0;
} else if(uVar7 < 144) { // literal copy from stream
iVar3 = uVar5 * 0x10 + (uVar7 & 0xf) + 1;
while(iVar3 != 0) {
iVar3 = iVar3 + -1;
*pOut = *inptr++;
pOut = pOut + 1;
}
iVar3 = 0;
uVar5 = 0;
} else if(uVar7 < 160) {
uVar5 = uVar7 & 3;
iVar3 = iVar3 + 1;
iVar6 = (int)(uVar7 & 0xc) >> 2;
} else if(uVar7 < 192) {
uVar5 = uVar5 * 0x20 + (uVar7 & 0x1f);
iVar3 = iVar3 + 1;
} else {
iVar3 = iVar3 + 1;
iVar6 = iVar6 * 0x40 + (uVar7 & 0x3f);
}
}
if(pOut != pEnd)
return std::make_error_code(BoltErrc::DecompressionError);
return res;
}
} // namespace lightningbolt

13
lib/bolt/Compression.hpp Normal file
View File

@ -0,0 +1,13 @@
#pragma once
#include <base/Types.hpp>
#include <base/ErrorOr.hpp>
#include <vector>
namespace lightningbolt {
/// Decompress a compressed BOLT file into a new allocated buffer.
/// TODO: this should probably be passed a file object, but for now,
/// just passing bare parameters works.
ErrorOr<std::vector<u8>> BoltDecompress(u8* input, usize decompressedSize);
} // namespace lightningbolt

28
lib/bolt/Errors.cpp Normal file
View File

@ -0,0 +1,28 @@
#include <bolt/Errors.hpp>
namespace lightningbolt {
namespace {
struct BoltErrorCategory : std::error_category {
const char* name() const noexcept override { return "bolt"; }
std::string message(i32 ev) const override {
switch(static_cast<BoltErrc>(ev)) {
case BoltErrc::InvalidMagic: return "Invalid bolt library file magic";
case BoltErrc::DecompressionError: return "Error during BOLT file decompression";
default: return "Unknown error";
}
}
};
} // namespace
const BoltErrorCategory boltCategory {};
} // namespace lightningbolt
namespace std {
std::error_code make_error_code(::lightningbolt::BoltErrc errc) {
return { static_cast<i32>(errc), ::lightningbolt::boltCategory };
}
} // namespace std

19
lib/bolt/Errors.hpp Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#include <base/Types.hpp>
namespace lightningbolt {
/// Error codes for BOLT
enum class BoltErrc {
InvalidMagic = 1,
DecompressionError
};
} // namespace lightningbolt
namespace std {
template <>
struct is_error_code_enum<::lightningbolt::BoltErrc> : true_type {};
std::error_code make_error_code(::lightningbolt::BoltErrc errc);
} // namespace std

105
lib/bolt/Reader.cpp Normal file
View File

@ -0,0 +1,105 @@
#include <base/MmapFile.hpp>
#include <bolt/Reader.hpp>
#include <structs/BoltStructs.hpp>
#include "base/ErrorOr.hpp"
#if 0
#endif
namespace lightningbolt {
struct BoltReader::Impl {
ErrorOr<void> OpenBolt(const fs::path& path) {
// Load the BOLT file
if(auto error = boltFile.Open(path); error.HasError())
return error;
lib = std::bit_cast<BoltLibraryHeader*>(boltFile.GetMapping());
if(!lib->Validate())
return std::make_error_code(BoltErrc::InvalidMagic);
auto p = path;
p.replace_filename("SLUS_201.14");
if(auto error = elfFile.Open(p); error.HasError())
return error;
// Load table entries.
GetTableEntries();
return {};
}
const std::vector<BoltReader::File>& GetTableEntries() {
if(entryTable.empty()) {
auto* base = elfFile.GetMapping();
auto* table = std::bit_cast<elf::BoltTableEntry*>(base + elf::BoltTableOffsets.usTable);
while(table->filenamePtr != 0x0) {
auto string_offset = elf::AddressToElfFileOffset(table->filenamePtr);
BoltReader::File te;
te.filename = { std::bit_cast<char*>(base + string_offset) };
te.index = table->entryId;
te.gid = table->groupId;
if(te.filename == "")
break;
entryTable.emplace_back(te);
table++;
}
}
return entryTable;
}
template <class F>
void ForEachFile(F f) {
for(auto& file : entryTable) {
if(file.uncompressedData == nullptr) {
auto gid = (file.gid >> 8);
auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping());
auto size = entries[file.index & 0x00ff].fileSize;
auto offset = entries[file.index & 0x00ff].fileOffset;
file.compressed = !(entries[file.index & 0x00ff].unk & 0x8);
file.uncompressedData = std::bit_cast<u8*>(boltFile.GetMapping() + offset);
file.uncompressedSize = size;
}
if(!f(file))
break;
}
}
private:
std::vector<BoltReader::File> entryTable;
lightningbolt::MmapFile elfFile;
lightningbolt::MmapFile boltFile;
lightningbolt::BoltLibraryHeader* lib;
};
BoltReader::BoltReader():
impl(std::make_unique<Impl>()) {}
BoltReader::~BoltReader() = default;
ErrorOr<void> BoltReader::OpenBolt(const fs::path& path) {
return impl->OpenBolt(path);
}
void BoltReader::ForEachFile(std::function<bool(File&)> f) {
impl->ForEachFile([&f](auto& file) {
return f(file);
});
}
} // namespace lightningbolt

33
lib/bolt/Reader.hpp Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#include <base/ErrorOr.hpp>
#include <bolt/Errors.hpp>
#include <functional>
namespace lightningbolt {
struct BoltReader {
struct File {
std::string_view filename;
u16 index;
u16 gid;
bool compressed;
u8* uncompressedData { nullptr };
usize uncompressedSize;
};
BoltReader();
~BoltReader();
ErrorOr<void> OpenBolt(const fs::path& path);
void ForEachFile(std::function<bool(File&)> f);
private:
struct Impl;
Unique<Impl> impl;
};
} // namespace lightningbolt

206
src/main.cpp
View File

@ -1,124 +1,18 @@
// Copyright 2023 The LightningBolt Authors
// SPDX-License-Identifier: MIT
#include <base/MmapFile.hpp>
#include <filesystem>
#include <format>
#include <fstream>
#include <iostream>
#include <structs/BoltStructs.hpp>
#include <type_traits>
#include <vector>
#include <bolt/Reader.hpp>
#include <bolt/Compression.hpp>
namespace fs = std::filesystem;
std::vector<u8> BoltDecompress(u8* input, usize decompressedSize) {
u8* inptr = input;
std::vector<u8> res;
res.resize(decompressedSize);
u8* pOut = res.data();
u8* pEnd = res.data() + decompressedSize;
i32 iVar3 = 0;
i32 iVar6 = 0;
u32 uVar5 = 0;
u8* pbVar4 = nullptr; // outrun
bool bVar1;
while(pOut < pEnd) {
auto uVar7 = *inptr++;
if(uVar7 < 128) { // lookback/run?
iVar3 = iVar3 + uVar5 * 8 + ((int)(uVar7 & 0x70) >> 4);
pbVar4 = pOut + -(iVar6 * 0x10 + (uVar7 & 0xf) + 1);
iVar6 = iVar3 + 1;
if(iVar3 != -2) {
do {
*pOut = *pbVar4;
pbVar4 = pbVar4 + 1;
bVar1 = iVar6 != 0;
pOut = pOut + 1;
iVar6 = iVar6 + -1;
} while(bVar1);
}
iVar6 = 0;
iVar3 = 0;
uVar5 = 0;
} else if(uVar7 < 144) { // literal copy from stream
iVar3 = uVar5 * 0x10 + (uVar7 & 0xf) + 1;
while(iVar3 != 0) {
iVar3 = iVar3 + -1;
*pOut = *inptr++;
pOut = pOut + 1;
}
iVar3 = 0;
uVar5 = 0;
} else if(uVar7 < 160) {
uVar5 = uVar7 & 3;
iVar3 = iVar3 + 1;
iVar6 = (int)(uVar7 & 0xc) >> 2;
} else if(uVar7 < 192) {
uVar5 = uVar5 * 0x20 + (uVar7 & 0x1f);
iVar3 = iVar3 + 1;
} else {
iVar3 = iVar3 + 1;
iVar6 = iVar6 * 0x40 + (uVar7 & 0x3f);
}
}
return res;
}
// error code boilerplate. This should be moved later
enum class BoltErrc { InvalidMagic = 1 };
struct BoltErrorCategory : std::error_category {
const char* name() const noexcept override { return "boltio"; }
std::string message(i32 ev) const override {
switch(static_cast<BoltErrc>(ev)) {
case BoltErrc::InvalidMagic: return "invalid bolt library file magic";
default: return "unknown error";
}
}
};
const BoltErrorCategory boltCategory {};
namespace std {
template <>
struct is_error_code_enum<BoltErrc> : true_type {};
std::error_code make_error_code(BoltErrc errc) {
return { static_cast<i32>(errc), boltCategory };
}
/// Custom formatter for std::error_code. As far as I know, this is
/// Not going to be added in C++23, despite the fact fmt already has a specialization for it,
/// so I have to implement one myself. At least it's not that awful to.
template <>
struct formatter<error_code> : formatter<string_view> {
template <typename FormatContext>
inline auto format(const error_code& ec, FormatContext& ctx) const {
return std::format_to(ctx.out(), "{} [{}.{}]", ec.message(), ec.category().name(), ec.value());
}
};
} // namespace std
struct BoltReader {
struct ParsedTableEntry {
std::string_view filename;
u16 index;
u16 gid;
bool compressed;
u8* uncompressedData { nullptr };
usize uncompressedSize;
fs::path pathify() {
fs::path ConvertBoltPath(const std::string_view filename) {
// Create a mutable copy of the filename
std::string s = { filename.data(), filename.size() };
@ -127,94 +21,18 @@ struct BoltReader {
c = '/';
return fs::current_path() / "ASSETS" / fs::path(s);
}
};
BoltReader() {}
ErrorOr<void> OpenBolt(const fs::path& path) {
// Load the BOLT file
if(auto error = boltFile.Open(path); error.HasError())
return error;
lib = std::bit_cast<lightningbolt::BoltLibraryHeader*>(boltFile.GetMapping());
if(!lib->Validate())
return std::make_error_code(BoltErrc::InvalidMagic);
auto p = path;
p.replace_filename("SLUS_201.14");
if(auto error = elfFile.Open(p); error.HasError())
return error;
// Load table entries.
GetTableEntries();
return {};
}
const std::vector<ParsedTableEntry>& GetTableEntries() {
if(entryTable.empty()) {
auto* base = elfFile.GetMapping();
auto* table = std::bit_cast<lightningbolt::elf::BoltTableEntry*>(base + lightningbolt::elf::BoltTableOffsets.usTable);
while(table->filenamePtr != 0x0) {
auto string_offset = lightningbolt::elf::AddressToElfFileOffset(table->filenamePtr);
ParsedTableEntry te;
te.filename = { std::bit_cast<char*>(base + string_offset) };
te.index = table->entryId;
te.gid = table->groupId;
if(te.filename == "")
break;
entryTable.emplace_back(te);
table++;
}
}
return entryTable;
}
template <class F>
void ForEachFile(F f) {
for(auto& file : entryTable) {
if(file.uncompressedData == nullptr) {
auto gid = (file.gid >> 8);
auto entries = lib->GroupDescriptors()[gid].Entries(boltFile.GetMapping());
auto size = entries[file.index & 0x00ff].fileSize;
auto offset = entries[file.index & 0x00ff].fileOffset;
file.compressed = !(entries[file.index & 0x00ff].unk & 0x8);
file.uncompressedData = std::bit_cast<u8*>(boltFile.GetMapping() + offset);
file.uncompressedSize = size;
}
if(!f(file))
break;
}
}
private:
std::vector<ParsedTableEntry> entryTable;
lightningbolt::MmapFile elfFile;
lightningbolt::MmapFile boltFile;
lightningbolt::BoltLibraryHeader* lib;
};
}
int main() {
BoltReader reader;
lightningbolt::BoltReader reader;
if(auto error = reader.OpenBolt(fs::current_path() / "ASSETS.BLT"); error.HasError()) {
std::cout << std::format("Error initalizing bolt reader: {}\n", error.Error());
}
reader.ForEachFile([](BoltReader::ParsedTableEntry& ent) {
reader.ForEachFile([](lightningbolt::BoltReader::File& ent) {
std::cout << std::format("File: {}", ent.filename);
auto p = ent.pathify();
auto p = ConvertBoltPath(ent.filename);
auto pathonly = p;
pathonly.remove_filename();
@ -232,9 +50,13 @@ int main() {
if(ent.compressed) {
std::cout << std::format(", compressed ({} bytes uncompressed)", ent.uncompressedSize);
auto decompressed = BoltDecompress(ent.uncompressedData, ent.uncompressedSize);
auto decompressed = lightningbolt::BoltDecompress(ent.uncompressedData, ent.uncompressedSize);
if(decompressed.HasError()) {
std::cout << std::format("Error during decompression: {}\n", decompressed.Error());
return false;
}
ofs.write((char*)decompressed.data(), ent.uncompressedSize);
ofs.write((char*)decompressed.Value().data(), ent.uncompressedSize);
std::cout << std::format(", written to \"{}\"\n", p.string());
} else {
std::cout << std::format(", uncompressed ({} bytes)", ent.uncompressedSize);