Initial support for Ver3/.PMDL archives

................. fuck

This really needs to be cleaned up before I'm willing to call it "good" but ultimately the API changes here needed to be done anyhow
This commit is contained in:
Lily Tsuru 2023-08-01 18:18:40 -04:00
parent 5272175a21
commit 2c0237933c
17 changed files with 366 additions and 153 deletions

3
.gitignore vendored
View File

@ -1,5 +1,4 @@
/.idea
cmake-build-*
/.cache
build/
# swap

View File

@ -21,28 +21,81 @@ namespace europa::io {
struct PakFile {
using DataType = std::vector<std::uint8_t>;
template<class T>
void InitAs(const T& value) {
toc = value;
}
void InitAs(structs::PakVersion version) {
switch(version) {
case structs::PakVersion::Ver3:
toc = structs::PakHeader_V3::TocEntry{};
break;
case structs::PakVersion::Ver4:
toc = structs::PakHeader_V4::TocEntry{};
break;
case structs::PakVersion::Ver5:
toc = structs::PakHeader_V5::TocEntry{};
break;
}
}
/**
* Get the file data.
*/
[[nodiscard]] const DataType& GetData() const;
[[nodiscard]] const DataType& GetData() const {
return data;
}
/**
* Get the TOC entry responsible.
*/
[[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const;
template<class T>
[[nodiscard]] const T& GetTOCEntry() const {
return std::get<T>(toc);
}
void SetData(DataType&& data);
void SetData(DataType&& data) {
this->data = std::move(data);
}
structs::PakTocEntry& GetTOCEntry();
std::uint32_t GetOffset() const {
std::uint32_t size{};
void FillTOCEntry();
std::visit([&](auto& entry) {
size = entry.offset;
}, toc);
return size;
}
std::uint32_t GetSize() const {
std::uint32_t size{};
std::visit([&](auto& entry) {
size = entry.size;
}, toc);
return size;
}
void FillTOCEntry() {
std::visit([&](auto& entry) {
entry.size = static_cast<std::uint32_t>(data.size());
}, toc);
}
template<class Cb>
void Visit(const Cb& cb) {
std::visit(cb, toc);
}
private:
friend PakReader;
friend PakWriter;
std::vector<std::uint8_t> data;
structs::PakTocEntry tocData;
structs::PakTocEntryVariant toc;
};
} // namespace europa::io

View File

@ -15,11 +15,14 @@
#include <string>
#include <unordered_map>
#include <variant>
namespace europa::io {
struct PakReader {
using MapType = std::unordered_map<std::string, PakFile>;
explicit PakReader(std::istream& is);
void ReadData();
@ -39,13 +42,17 @@ namespace europa::io {
const MapType& GetFiles() const;
// implement in cpp later, lazy and just wanna get this out :vvv
const structs::PakHeader& GetHeader() const { return header; }
const structs::PakHeaderVariant& GetHeader() const { return header; }
private:
template<class T>
void ReadData_Impl();
std::istream& stream;
bool invalid { false };
structs::PakHeader header {};
structs::PakVersion version;
structs::PakHeaderVariant header {};
MapType files;
};

View File

@ -14,6 +14,7 @@
#include <iosfwd>
#include <string>
#include <utility>
#include "europa/structs/Pak.hpp"
namespace europa::io {
@ -23,9 +24,11 @@ namespace europa::io {
struct PakWriter {
using FlattenedType = std::pair<std::string, PakFile>;
void Init(structs::PakHeader::Version version);
//void Init(structs::PakHeader::Version version);
const structs::PakHeader& GetHeader() const { return pakHeader; }
//const HeaderType& GetHeader() const { return pakHeader; }
void SetVersion(structs::PakVersion version);
/**
* Write the resulting archive to the given output stream.
@ -33,7 +36,12 @@ namespace europa::io {
void Write(std::ostream& os, std::vector<FlattenedType>&& vec, PakProgressReportSink& sink);
private:
structs::PakHeader pakHeader {};
template<class T>
void WriteImpl(std::ostream& os, std::vector<FlattenedType>&& vec, PakProgressReportSink& sink, bool sectorAligned = true);
structs::PakVersion version{};
//HeaderType pakHeader {};
};
} // namespace europa::io

View File

@ -12,17 +12,23 @@
#include <cstdint>
#include <cstring>
#include <europa/structs/ImHexAdapter.hpp>
#include <optional>
#include <variant>
#include <cstdio>
namespace europa::structs {
struct [[gnu::packed]] PakHeader {
constexpr static const char VALID_MAGIC[16] = "Europa Packfile";
constexpr static const char VALID_MAGIC[16] = "Europa Packfile";
enum class Version : u16 {
Ver4 = 0x4,
Ver5 = 0x5
};
enum class PakVersion : u16 {
Invalid = 0xffff,
Ver3 = 0x3, ///< Typically used for PMDL files
Ver4 = 0x4,
Ver5 = 0x5
};
struct [[gnu::packed]] PakHeader_Common {
char magic[16]; // "Europa Packfile\0"
/**
@ -30,7 +36,91 @@ namespace europa::structs {
*/
u16 headerSize;
Version version;
PakVersion version;
bool Valid() const {
return !std::strcmp(magic, VALID_MAGIC);
}
};
template <class Impl, PakVersion Version>
struct [[gnu::packed]] PakHeader_Impl : PakHeader_Common {
constexpr static auto VERSION = Version;
/**
* Get the real header size (including the magic).
*/
[[nodiscard]] constexpr std::size_t RealHeaderSize() const {
return sizeof(magic) + static_cast<std::size_t>(headerSize);
}
constexpr static u16 HeaderSize() {
return sizeof(Impl) - (sizeof(VALID_MAGIC) - 1);
}
PakHeader_Impl() {
// clear any junk
memset(this, 0, sizeof(PakHeader_Impl));
version = Version;
// Copy important things & set proper header size.
std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC));
headerSize = HeaderSize();
}
explicit PakHeader_Impl(const PakHeader_Common& header) {
memcpy(&magic[0], &header.magic[0], sizeof(header.magic));
version = header.version;
headerSize = header.headerSize;
}
[[nodiscard]] bool Valid() const noexcept {
// Magic must match.
if(!reinterpret_cast<const PakHeader_Common*>(this)->Valid())
return false;
// Check header size.
if(headerSize != HeaderSize() && headerSize != HeaderSize() + 1)
return false;
return version == Version;
}
};
struct [[gnu::packed]] PakHeader_V3 : public PakHeader_Impl<PakHeader_V3, PakVersion::Ver3> {
using PakHeader_Impl<PakHeader_V3, PakVersion::Ver3>::VERSION;
using PakHeader_Impl<PakHeader_V3, PakVersion::Ver3>::PakHeader_Impl;
using PakHeader_Impl<PakHeader_V3, PakVersion::Ver3>::Valid;
struct [[gnu::packed]] TocEntry {
u32 offset;
u32 size;
u32 creationUnixTime; // junk on these v3 files
u16 junk;
};
u32 tocOffset;
u32 tocSize;
u32 fileCount;
u32 creationUnixTime;
// Zeroes.
u32 reservedPad{};
};
struct [[gnu::packed]] PakHeader_V4 : public PakHeader_Impl<PakHeader_V4, PakVersion::Ver4> {
using PakHeader_Impl<PakHeader_V4, PakVersion::Ver4>::PakHeader_Impl;
struct [[gnu::packed]] TocEntry {
u32 offset;
u32 size;
u32 creationUnixTime;
};
u8 pad;
u32 tocOffset;
@ -43,66 +133,53 @@ namespace europa::structs {
// Zeroes.
u32 reservedPad;
/**
* Get the real header size (including the magic).
*/
[[nodiscard]] constexpr std::size_t RealHeaderSize() const {
return sizeof(magic) + static_cast<std::size_t>(headerSize);
}
/**
* Initialize this header (used when writing).
*/
void Init(Version ver) noexcept {
// clear any junk
memset(this, 0, sizeof(PakHeader));
// Copy important things.
std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC));
// Set proper header size.
headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1);
// Set archive version
version = ver;
}
[[nodiscard]] bool Valid() const noexcept {
// Magic must match.
if(std::strcmp(magic, VALID_MAGIC) != 0)
return false;
// Check header size.
if(headerSize != sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1))
return false;
using enum Version;
// Version must match ones we support,
// otherwise it's invalid.
switch(version) {
case Ver4:
case Ver5:
return true;
default:
return false;
}
}
};
// A Toc entry (without string. Needs to be read in separately)
struct [[gnu::packed]] PakTocEntry {
u32 offset;
u32 size;
struct [[gnu::packed]] PakHeader_V5 : public PakHeader_Impl<PakHeader_V5, PakVersion::Ver5> {
using PakHeader_Impl<PakHeader_V5, PakVersion::Ver5>::PakHeader_Impl;
struct [[gnu::packed]] TocEntry {
u32 offset;
u32 size;
u32 creationUnixTime;
};
u8 pad;
u32 tocOffset;
u32 tocSize;
u32 fileCount;
u32 creationUnixTime;
// Zeroes.
u32 reservedPad;
};
using PakHeaderVariant = std::variant<
structs::PakHeader_V3,
structs::PakHeader_V4,
structs::PakHeader_V5>;
static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!");
static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid when writing archives.");
static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!");
using PakTocEntryVariant = std::variant<
structs::PakHeader_V3::TocEntry,
structs::PakHeader_V4::TocEntry,
structs::PakHeader_V5::TocEntry>;
static_assert(sizeof(PakHeader_V3) == 0x28, "PakHeader_V3 wrong size");
// TODO: their format really seems to be wrong, 0x19 is proper, but some v3 archives have 0x1a header size
// ??? very weird
//static_assert(sizeof(PakHeader_V3) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V3::headerSize will be invalid when writing archives.");
static_assert(sizeof(PakHeader_V4) == 0x29, "PakHeader_V4 wrong size!!");
static_assert(sizeof(PakHeader_V4) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V4::headerSize will be invalid when writing archives.");
static_assert(sizeof(PakHeader_V5) == 0x29, "PakHeader_V5 wrong size!!");
static_assert(sizeof(PakHeader_V5) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V5::headerSize will be invalid when writing archives.");
static_assert(sizeof(PakHeader_V3::TocEntry) == 0xe, "V3 TocEntry wrong size!");
static_assert(sizeof(PakHeader_V4::TocEntry) == 0xc, "V4 PakTocEntry wrong size!");
static_assert(sizeof(PakHeader_V5::TocEntry) == 0xc, "V5 PakTocEntry wrong size!");
} // namespace europa::structs

View File

@ -28,7 +28,7 @@ namespace europa::structs {
*/
constexpr static u32 TextureFlag_UsesAlpha = 0x1000000;
constexpr static auto ValidMagic = util::FourCC<"YATF", std::endian::big>();
constexpr static auto ValidMagic = util::FourCC<"YATF", std::endian::little>();
u32 magic;

View File

@ -11,7 +11,6 @@ add_library(europa
io/StreamUtils.cpp
# Pak IO
io/PakFile.cpp
io/PakReader.cpp
io/PakWriter.cpp

View File

@ -1,33 +0,0 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#include <europa/io/PakFile.hpp>
namespace europa::io {
const PakFile::DataType& PakFile::GetData() const {
return data;
}
const structs::PakTocEntry& PakFile::GetTOCEntry() const {
return tocData;
}
structs::PakTocEntry& PakFile::GetTOCEntry() {
return tocData;
}
void PakFile::SetData(PakFile::DataType&& newData) {
data = std::move(newData);
}
void PakFile::FillTOCEntry() {
tocData.size = static_cast<std::uint32_t>(data.size());
}
} // namespace europa::io

View File

@ -14,27 +14,83 @@
namespace europa::io {
/*
inline std::optional<PakHeader> GetPakHeader(const PakHeader_Common& common_header) {
switch(common_header.version) {
case PakVersion::Ver3:
return PakHeader_V3(common_header);
case PakVersion::Ver4:
return PakHeader_V4(common_header);
case PakVersion::Ver5:
return PakHeader_V5(common_header);
case PakVersion::Invalid:
default:
return std::nullopt;
}
}
*/
PakReader::PakReader(std::istream& is)
: stream(is) {
}
void PakReader::ReadData() {
header = impl::ReadStreamType<structs::PakHeader>(stream);
template<class T>
void PakReader::ReadData_Impl() {
auto header_type = impl::ReadStreamType<T>(stream);
if(!header.Valid()) {
if(!header_type.Valid()) {
invalid = true;
return;
}
bool isStreams{false};
if(header_type.tocOffset > 0x17000000)
isStreams = true;
// Read the archive TOC
stream.seekg(header.tocOffset, std::istream::beg);
for(auto i = 0; i < header.fileCount; ++i) {
// The first part of the TOC entry is a VLE string,
stream.seekg(header_type.tocOffset, std::istream::beg);
for(auto i = 0; i < header_type.fileCount; ++i) {
// The first part of the TOC entry is always a VLE string,
// which we don't store inside the type (because we can't)
//
// Read this in first.
auto filename = impl::ReadPString(stream);
files[filename].GetTOCEntry() = impl::ReadStreamType<structs::PakTocEntry>(stream);
files[filename].InitAs(impl::ReadStreamType<typename T::TocEntry>(stream));
if(isStreams)
files[filename].Visit([&](auto& tocEntry) {
tocEntry.creationUnixTime = impl::ReadStreamType<structs::u32>(stream);
});
}
header = header_type;
}
void PakReader::ReadData() {
auto commonHeader = impl::ReadStreamType<structs::PakHeader_Common>(stream);
stream.seekg(0, std::istream::beg);
std::cout << "picking version " << (int)commonHeader.version << '\n';
switch(commonHeader.version) {
case structs::PakVersion::Ver3:
ReadData_Impl<structs::PakHeader_V3>();
break;
case structs::PakVersion::Ver4:
ReadData_Impl<structs::PakHeader_V4>();
break;
case structs::PakVersion::Ver5:
ReadData_Impl<structs::PakHeader_V5>();
break;
default:
return;
}
}
@ -51,11 +107,10 @@ namespace europa::io {
if(!fileObject.data.empty())
return;
auto& toc = fileObject.GetTOCEntry();
fileObject.data.resize(toc.size);
fileObject.data.resize(fileObject.GetSize());
stream.seekg(toc.offset, std::istream::beg);
stream.read(reinterpret_cast<char*>(&fileObject.data[0]), toc.size);
stream.seekg(fileObject.GetOffset(), std::istream::beg);
stream.read(reinterpret_cast<char*>(&fileObject.data[0]), fileObject.GetSize());
}
PakReader::MapType& PakReader::GetFiles() {

View File

@ -12,12 +12,13 @@
#include <iostream>
#include "StreamUtils.h"
#include "europa/structs/Pak.hpp"
namespace europa::io {
void PakWriter::Init(structs::PakHeader::Version version) {
void PakWriter::SetVersion(structs::PakVersion version) {
// for now.
pakHeader.Init(version);
this->version = version;
}
// move to a util/ header
@ -27,28 +28,47 @@ namespace europa::io {
return (-value) & alignment - 1;
}
// TODO:
// - Composable operations (WriteTOC, WriteFile, WriteHeader)
void PakWriter::Write(std::ostream &os, std::vector<FlattenedType> &&vec, PakProgressReportSink &sink) {
switch(version) {
case structs::PakVersion::Ver3:
WriteImpl<structs::PakHeader_V3>(os, std::move(vec), sink);
break;
case structs::PakVersion::Ver4:
WriteImpl<structs::PakHeader_V3>(os, std::move(vec), sink);
break;
case structs::PakVersion::Ver5:
WriteImpl<structs::PakHeader_V3>(os, std::move(vec), sink);
break;
}
}
void PakWriter::Write(std::ostream& os, std::vector<FlattenedType>&& vec, PakProgressReportSink& sink) {
template<class T>
void PakWriter::WriteImpl(std::ostream& os, std::vector<FlattenedType>&& vec, PakProgressReportSink& sink, bool sectorAligned) {
std::vector<FlattenedType> sortedFiles = std::move(vec);
T pakHeader{};
// Sort the flattened array by file size, the biggest first.
// Doesn't seem to help (neither does name length)
std::ranges::sort(sortedFiles, std::greater{}, [](const FlattenedType& elem) {
return elem.second.GetTOCEntry().size;
return elem.second.GetSize();
});
// Leave space for the header
os.seekp(sizeof(structs::PakHeader), std::ostream::beg);
os.seekp(sizeof(T), std::ostream::beg);
// Version 5 paks seem to have an additional bit of reserved data
// (which is all zeros.)
if(pakHeader.version == structs::PakHeader::Version::Ver5) {
if(T::VERSION == structs::PakVersion::Ver5) {
os.seekp(6, std::ostream::cur);
}
//os.seekp(
// AlignBy(os.tellp(), 2048),
// std::istream::beg
//);
// Write file data
for(auto& [filename, file] : sortedFiles) {
sink.OnEvent({
@ -56,8 +76,17 @@ namespace europa::io {
filename
});
file.GetTOCEntry().offset = os.tellp();
os.write(reinterpret_cast<const char*>(file.GetData().data()), file.GetData().size());
file.Visit([&](auto& tocEntry) {
tocEntry.offset = os.tellp();
});
os.write(reinterpret_cast<const char*>(file.GetData().data()), file.GetSize());
//os.seekp(
// AlignBy(os.tellp(), 2048),
// std::istream::beg
//);
// Flush on file writing
os.flush();
@ -84,7 +113,10 @@ namespace europa::io {
os.put(c);
os.put('\0');
impl::WriteStreamType(os, file.GetTOCEntry());
file.Visit([&](auto& tocEntry) {
impl::WriteStreamType(os, tocEntry);
});
}
@ -92,7 +124,6 @@ namespace europa::io {
PakProgressReportSink::PakEvent::Type::FillInHeader
});
// Fill out the rest of the header.
pakHeader.fileCount = sortedFiles.size();
pakHeader.tocSize = static_cast<std::uint32_t>(os.tellp()) - (pakHeader.tocOffset - 1);

View File

@ -7,6 +7,7 @@
//
#include "StreamUtils.h"
#include <cstdint>
namespace europa::io::impl {
@ -64,4 +65,4 @@ namespace europa::io::impl {
}
}
} // namespace europa::io::impl
} // namespace europa::io::impl

View File

@ -11,6 +11,7 @@
#include <string>
#include <ctime>
#include <cstdint>
namespace eupak {

View File

@ -53,7 +53,7 @@ int main(int argc, char** argv) {
createParser.add_argument("-V","--archive-version")
.default_value("starfighter")
.help(R"(Output archive version. Either "starfighter" or "jedistarfighter".)")
.help(R"(Output archive version. Either "pmdl", "starfighter" or "jedistarfighter".)")
.metavar("VERSION");
createParser.add_argument("output")
@ -123,16 +123,18 @@ int main(int argc, char** argv) {
if(createParser.is_used("--archive-version")) {
const auto& versionStr = createParser.get("--archive-version");
if(versionStr == "starfighter") {
args.pakVersion = europa::structs::PakHeader::Version::Ver4;
if(versionStr == "pmdl") {
args.pakVersion = europa::structs::PakVersion::Ver3;
} else if(versionStr == "starfighter") {
args.pakVersion = europa::structs::PakVersion::Ver4;
} else if(versionStr == "jedistarfighter") {
args.pakVersion = europa::structs::PakHeader::Version::Ver5;
args.pakVersion = europa::structs::PakVersion::Ver5;
} else {
std::cout << "Error: Invalid version \"" << versionStr << "\"\n" << createParser;
return 1;
}
} else {
args.pakVersion = europa::structs::PakHeader::Version::Ver4;
args.pakVersion = europa::structs::PakVersion::Ver4;
}

View File

@ -78,7 +78,7 @@ namespace eupak::tasks {
int CreateTask::Run(Arguments&& args) {
europa::io::PakWriter writer;
writer.Init(args.pakVersion);
writer.SetVersion(args.pakVersion);
auto currFile = 0;
auto fileCount = 0;
@ -143,9 +143,10 @@ namespace eupak::tasks {
ifs.read(reinterpret_cast<char*>(&pakData[0]), pakData.size());
file.SetData(std::move(pakData));
file.FillTOCEntry();
file.GetTOCEntry().creationUnixTime = static_cast<std::uint32_t>(lastModified.time_since_epoch().count());
file.InitAs(args.pakVersion);
//file.GetTOCEntry().creationUnixTime = static_cast<std::uint32_t>(lastModified.time_since_epoch().count());
files.emplace_back(std::make_pair(relativePathName, std::move(file)));
progress.tick();

View File

@ -22,7 +22,7 @@ namespace eupak::tasks {
fs::path outputFile;
bool verbose;
europa::structs::PakHeader::Version pakVersion;
europa::structs::PakVersion pakVersion;
};
int Run(Arguments&& args);

View File

@ -80,7 +80,7 @@ namespace eupak::tasks {
std::cerr << "Extracting file \"" << filename << "\"...\n";
}
ofs.write(reinterpret_cast<const char*>(file.GetData().data()), static_cast<std::streampos>(file.GetTOCEntry().size));
ofs.write(reinterpret_cast<const char*>(file.GetData().data()), static_cast<std::streampos>(file.GetSize()));
ofs.flush();
progress.tick();
}

View File

@ -36,23 +36,35 @@ namespace eupak::tasks {
return 1;
}
std::string version = "Version 4 (Starfighter)";
std::visit([&](auto& header){
std::string version;
if constexpr(std::decay_t<decltype(header)>::VERSION == europa::structs::PakVersion::Ver3)
version = "Version 3 (PMDL)";
else if constexpr(std::decay_t<decltype(header)>::VERSION == europa::structs::PakVersion::Ver4)
version = "Version 4 (Starfighter)";
else if constexpr(std::decay_t<decltype(header)>::VERSION == europa::structs::PakVersion::Ver5)
version = "Version 5 (Jedi Starfighter)";
if(reader.GetHeader().version == europa::structs::PakHeader::Version::Ver5)
version = "Version 5 (Jedi Starfighter)";
std::cout << "Archive " << args.inputPath << ":\n";
std::cout << " Created: " << FormatUnixTimestamp(reader.GetHeader().creationUnixTime, DATE_FORMAT) << '\n';
std::cout << " Version: " << version << '\n';
std::cout << " Size: " << FormatUnit(reader.GetHeader().tocOffset + reader.GetHeader().tocSize) << '\n';
std::cout << " File Count: " << reader.GetHeader().fileCount << " files\n";
std::cout << "Archive " << args.inputPath << ":\n";
std::cout << " Created: " << FormatUnixTimestamp(header.creationUnixTime, DATE_FORMAT) << '\n';
std::cout << " Version: " << version << '\n';
std::cout << " Size: " << FormatUnit(header.tocOffset + header.tocSize) << '\n';
std::cout << " File Count: " << header.fileCount << " files\n";
}, reader.GetHeader());
// Print a detailed file list if verbose.
if(args.verbose) {
for(auto& [ filename, file ] : reader.GetFiles()) {
std::cout << "File \"" << filename << "\":\n";
std::cout << " Created: " << FormatUnixTimestamp(file.GetTOCEntry().creationUnixTime, DATE_FORMAT) << '\n';
std::cout << " Size: " << FormatUnit(file.GetTOCEntry().size) << '\n';
file.Visit([&](auto& tocEntry) {
std::cout << " Created: " << FormatUnixTimestamp(tocEntry.creationUnixTime, DATE_FORMAT) << '\n';
std::cout << " Size: " << FormatUnit(tocEntry.size) << '\n';
});
}
}