GameModding
/
EuropaTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add possibly not WIP package writer 

Don't have a tool for building "fresh" packages yet, but the "paktest" tests regurgitating a pak into a new pak file.

Usage is:

./paktest [pak file]

it will write to "new.pak" always, which meh. It's just a test utility.
This commit is contained in:
Lily Tsuru 2022-09-05 19:59:46 -05:00
parent 1782fef0dc
commit 5d03f49e21
20 changed files with 351 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
include/europa/io/PakFile.h Normal file
View File

@ -0,0 +1,55 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#ifndef EUROPA_IO_PAKFILE_H
#define EUROPA_IO_PAKFILE_H
#include <vector>
#include <cstdint>
#include <europa/structs/Pak.h>
namespace europa::io {
struct PakReader;
struct PakWriter;
struct PakFile {
using DataType = std::vector<std::uint8_t>;
/**
* Get the file data.
*/
[[nodiscard]] const DataType& GetData() const;
/**
* Get the TOC entry responsible.
*/
[[nodiscard]] const structs::PakTocEntry& GetTOCEntry() const;
void SetData(DataType&& data);
structs::PakTocEntry& GetTOCEntry();
void FillTOCEntry();
private:
friend PakReader;
friend PakWriter;
std::vector<std::uint8_t> data;
structs::PakTocEntry tocData;
};
}
#endif //EUROPA_IO_PAKFILE_H

36
include/europa/io/PakReader.h
View File

@ -9,53 +9,37 @@
#ifndef EUROPA_IO_PAKREADER_H
#define EUROPA_IO_PAKREADER_H
#include <iosfwd>
#include <string>
#include <unordered_map>
#include <vector>
#include <europa/io/PakFile.h>
#include <europa/structs/Pak.h>
#include <iosfwd>
#include <string>
#include <unordered_map>
namespace europa::io {
struct PakReader {
struct File {
File(std::vector<std::uint8_t>&& data, structs::PakTocEntry& tocData);
const std::vector<std::uint8_t>& GetData() const;
const structs::PakTocEntry& GetTOCEntry() const;
private:
std::vector<std::uint8_t> data;
structs::PakTocEntry tocData;
};
explicit PakReader(std::istream& is);
void ReadData();
bool Invalid() const {
return invalid;
}
const std::unordered_map<std::string, File>& GetFiles() const;
const std::unordered_map<std::string, PakFile>& GetFiles() const;
private:
std::istream& stream;
bool invalid{false};
bool invalid { false };
structs::PakHeader header{};
structs::PakHeader header {};
std::unordered_map<std::string, structs::PakTocEntry> tocData;
std::unordered_map<std::string, File> files;
std::unordered_map<std::string, PakFile> files;
};
}
} // namespace europa::io
#endif // EUROPA_IO_PAKREADER_H

47
include/europa/io/PakWriter.h Normal file
View File

@ -0,0 +1,47 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#ifndef EUROPA_IO_PAKWRITER_H
#define EUROPA_IO_PAKWRITER_H
#include <europa/io/PakFile.h>
#include <iosfwd>
#include <string>
#include <unordered_map>
namespace europa::io {
/**
* Writer for package files.
*/
struct PakWriter {
void Init();
void AddFile(const std::string& path, const PakFile& data);
void RemoveFile(const std::string& path);
/**
* Write the resulting archive to the given output stream.
*/
void Write(std::ostream& os);
private:
structs::PakHeader pakHeader{};
std::unordered_map<std::string, PakFile> archiveFiles;
};
}
#endif // EUROPA_IO_PAKWRITER_H

8
include/europa/io/YatfReader.h
View File

@ -9,10 +9,11 @@
#ifndef EUROPA_IO_YATFREADER_H
#define EUROPA_IO_YATFREADER_H
#include <iosfwd>
#include <europa/structs/Yatf.h>
#include <pixel/RgbaImage.h>
#include <iosfwd>
namespace europa::io {
/**
@ -35,8 +36,7 @@ namespace europa::io {
private:
std::istream& stream;
bool invalid {false};
bool invalid { false };
structs::YatfHeader header;
@ -46,6 +46,6 @@ namespace europa::io {
pixel::RgbaImage image;
};
}
} // namespace europa::io
#endif // EUROPA_IO_YATFREADER_H

18
include/europa/structs/ImHexAdapter.h
View File

@ -14,17 +14,17 @@
#include <cstdint>
namespace europa::structs {
using u8 = std::uint8_t;
using s8 = std::int8_t;
using u8 = std::uint8_t;
using s8 = std::int8_t;
using u16 = std::uint16_t;
using s16 = std::int16_t;
using u16 = std::uint16_t;
using s16 = std::int16_t;
using u32 = std::uint32_t;
using s32 = std::int32_t;
using u32 = std::uint32_t;
using s32 = std::int32_t;
using u64 = std::uint64_t;
using s64 = std::int64_t;
}
using u64 = std::uint64_t;
using s64 = std::int64_t;
} // namespace europa::structs
#endif // EUROPA_STRUCTS_IMHEXADAPTER_H

25
include/europa/structs/Pak.h
View File

@ -9,10 +9,11 @@
#ifndef EUROPA_STRUCTS_PAK_H
#define EUROPA_STRUCTS_PAK_H
#include <cstdint>
#include <europa/structs/ImHexAdapter.h>
#include <cstdint>
#include <cstring>
namespace europa::structs {
enum class PakVersion : u16 {
@ -20,7 +21,7 @@ namespace europa::structs {
Ver2 = 0x5
};
struct [[gnu::packed]] PakHeader {
struct [[gnu::packed]] PakHeader {
constexpr static const char VALID_MAGIC[16] = "Europa Packfile";
char magic[16]; // "Europa Packfile\0"
@ -51,6 +52,18 @@ namespace europa::structs {
[[nodiscard]] constexpr std::size_t RealHeaderSize() const {
return sizeof(magic) + static_cast<std::size_t>(headerSize);
}
void Init(PakVersion ver) {
// clear any junk
memset(this, 0, sizeof(PakHeader));
// Copy important things.
std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC));
headerSize = sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1);
// Set archive version
version = ver;
}
};
// A Toc entry (without string. Needs to be read in seperately)
@ -63,10 +76,10 @@ namespace europa::structs {
u32 unk3;
};
static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!");
static_assert(sizeof(PakHeader) == 0x29, "PakHeader wrong size!!");
static_assert(sizeof(PakHeader) - (sizeof(PakHeader::VALID_MAGIC) - 1) == 0x1a, "PakHeader::headerSize will be invalid");
static_assert(sizeof(PakTocEntry) == 0xc, "PakTocEntry wrong size!");
}
} // namespace europa::structs
#endif // EUROPA_STRUCTS_PAK_H

5
include/europa/structs/Yatf.h
View File

@ -9,15 +9,14 @@
#ifndef EUROPA_STRUCTS_YATF_H
#define EUROPA_STRUCTS_YATF_H
#include <europa/structs/ImHexAdapter.h>
#include <europa/util/FourCC.h>
#include <europa/structs/ImHexAdapter.h>
#include <type_traits>
namespace europa::structs {
struct [[gnu::packed]] YatfHeader {
constexpr static u32 TextureFlag_Unknown = 0x1;
/**
@ -47,6 +46,6 @@ namespace europa::structs {
}
};
}
} // namespace europa::structs
#endif // EUROPA_STRUCTS_YATF_H

8
include/europa/util/FixedString.h
View File

@ -1,10 +1,9 @@
//
// SSX 3 Lobby Server
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// This file is licensed under the GNU General Public License Version 3.
// Text is provided in LICENSE.
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#ifndef EUROPA_UTIL_FIXEDSTRING_H
@ -14,6 +13,9 @@
namespace europa::util {
/**
* A compile-time string. Usable as a C++20 cNTTP.
*/
template<std::size_t N>
struct FixedString {
char buf[N + 1]{};

5
include/europa/util/FourCC.h
View File

@ -1,10 +1,9 @@
//
// SSX 3 Lobby Server
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// This file is licensed under the GNU General Public License Version 3.
// Text is provided in LICENSE.
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#ifndef EUROPA_FOURCC_H

8
src/libeuropa/CMakeLists.txt
View File

@ -1,8 +1,15 @@
add_library(libeuropa
# Implementation details
io/StreamUtils.cpp
# Pak IO
io/PakFile.cpp
io/PakReader.cpp
io/PakWriter.cpp
# Yatf IO
io/YatfReader.cpp
)
@ -15,6 +22,5 @@ set_target_properties(libeuropa PROPERTIES
# Projects which libeuropa depends on
target_link_libraries(libeuropa PUBLIC
pixel::libpixel
)

34
src/libeuropa/io/PakFile.cpp Normal file
View File

@ -0,0 +1,34 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#include <europa/io/PakFile.h>
namespace europa::io {
const PakFile::DataType& PakFile::GetData() const {
return data;
}
const structs::PakTocEntry& PakFile::GetTOCEntry() const {
return tocData;
}
structs::PakTocEntry& PakFile::GetTOCEntry() {
return tocData;
}
void PakFile::SetData(PakFile::DataType&& newData) {
data = std::move(newData);
}
void PakFile::FillTOCEntry() {
tocData.size = static_cast<std::uint32_t>(data.size());
}
}

49
src/libeuropa/io/PakReader.cpp
View File

@ -31,19 +31,26 @@ namespace europa::io {
// Read this in first.
auto filename = impl::ReadPString(stream);
// Then read in the rest.
tocData[filename] = impl::ReadStreamType<structs::PakTocEntry>(stream);
files[filename].GetTOCEntry() = impl::ReadStreamType<structs::PakTocEntry>(stream);
};
ReadHeader();
// Validate the archive magic
// Validate the archive header
if(std::strcmp(header.magic, structs::PakHeader::VALID_MAGIC) != 0) {
invalid = true;
return;
}
// std::cout << (int)header.headerSize << " version " << (int)header.version << '\n';
switch(header.version) {
case structs::PakVersion::Starfighter:
case structs::PakVersion::Ver2:
break;
default:
invalid = true;
return;
}
stream.seekg(header.tocOffset, std::istream::beg);
@ -51,39 +58,19 @@ namespace europa::io {
for(auto i = 0; i < header.fileCount; ++i)
ReadTocEntry();
// for(auto& [ filename, data ] : tocData) {
// std::cout << filename << " offset " << data.offset << " size " << data.size << '\n';
// }
// Read all file data in
for(auto& [filename, file] : files) {
auto& toc = file.GetTOCEntry();
file.data.resize(toc.size);
// Read all files in
for(auto& [filename, data] : tocData) {
std::vector<std::uint8_t> dataBuffer;
dataBuffer.resize(data.size);
stream.seekg(data.offset, std::istream::beg);
stream.read(reinterpret_cast<char*>(&dataBuffer[0]), data.size);
File file(std::move(dataBuffer), data);
files.insert_or_assign(filename, file);
stream.seekg(toc.offset, std::istream::beg);
stream.read(reinterpret_cast<char*>(&file.data[0]), toc.size);
}
}
const std::unordered_map<std::string, PakReader::File>& PakReader::GetFiles() const {
const std::unordered_map<std::string, PakFile>& PakReader::GetFiles() const {
return files;
}
PakReader::File::File(std::vector<std::uint8_t>&& data, structs::PakTocEntry& tocData)
: data(std::move(data)),
tocData(tocData) {
}
const std::vector<std::uint8_t>& PakReader::File::GetData() const {
return data;
}
const structs::PakTocEntry& PakReader::File::GetTOCEntry() const {
return tocData;
}
} // namespace europa::io

62
src/libeuropa/io/PakWriter.cpp Normal file
View File

@ -0,0 +1,62 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#include <europa/io/PakWriter.h>
#include <iostream>
#include "StreamUtils.h"
namespace europa::io {
void PakWriter::Init() {
// for now.
pakHeader.Init(structs::PakVersion::Starfighter);
}
void PakWriter::AddFile(const std::string &path, const PakFile& data) {
archiveFiles[path] = data;
}
void PakWriter::RemoveFile(const std::string &path) {
archiveFiles.erase(path);
}
void PakWriter::Write(std::ostream &os) {
// Set up the header a bit more...
pakHeader.fileCount = archiveFiles.size();
// Leave space for the header
os.seekp(sizeof(structs::PakHeader), std::ostream::beg);
// Write file data
for (auto &[filename, file]: archiveFiles) {
file.GetTOCEntry().offset = os.tellp();
os.write(reinterpret_cast<const char *>(file.GetData().data()), file.GetData().size());
}
pakHeader.tocOffset = os.tellp();
// Write the TOC
for (auto &[filename, file]: archiveFiles) {
file.FillTOCEntry();
// Write the pstring
os.put(static_cast<char>(filename.length() + 1));
for (const auto c: filename)
os.put(c);
os.put('\0');
impl::WriteStreamType(os, file.GetTOCEntry());
}
os.seekp(0, std::ostream::beg);
impl::WriteStreamType(os, pakHeader);
}
}

5
src/libeuropa/io/StreamUtils.cpp
View File

@ -19,6 +19,11 @@ namespace europa::io::impl {
is.read(&buffer[0], size);
}
void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size) {
os.write(&buffer[0], buffer_size);
}
} // namespace detail
std::string ReadZeroTerminatedString(std::istream& is) {

15
src/libeuropa/io/StreamUtils.h
View File

@ -16,9 +16,9 @@ namespace europa::io::impl {
namespace detail {
void ReadStreamTypeImpl(std::istream& is, char* buffer, std::size_t size);
void WriteStreamTypeImpl(std::ostream& os, const char* buffer, std::size_t buffer_size);
}
// This is lame. But it works :)
template <class T>
constexpr T ReadStreamType(std::istream& is) {
@ -37,6 +37,19 @@ namespace europa::io::impl {
return object;
}
template <class T>
constexpr void WriteStreamType(std::ostream& os, const T& object) {
// Absolutely UB.
union Hack {
const T* t;
const char* c;
} address {
.t = &object
};
detail::WriteStreamTypeImpl(os, address.c, sizeof(T));
}
std::string ReadZeroTerminatedString(std::istream& is);
std::string ReadPString(std::istream& is);

2
src/libeuropa/io/YatfReader.cpp
View File

@ -29,7 +29,6 @@ namespace europa::io {
void YatfReader::ReadImage() {
if(header.flags & structs::YatfHeader::TextureFlag_NoPalette) {
image.Resize({ static_cast<std::uint16_t>(header.width), static_cast<std::uint16_t>(header.height) });
stream.read(reinterpret_cast<char*>(image.GetBuffer()), (header.width * header.height) * sizeof(pixel::RgbaColor));
} else {
pixel::RgbaColor palette[256];
@ -37,7 +36,6 @@ namespace europa::io {
// NB: sizeof() does pre-multiplication, so it's 100% ok for us to do this.
stream.read(reinterpret_cast<char*>(&palette[0]), sizeof(palette));
stream.read(reinterpret_cast<char*>(&tempBuffer[0]), tempBuffer.size());
image.Resize({ static_cast<std::uint16_t>(header.width), static_cast<std::uint16_t>(header.height) });

11
src/tools/CMakeLists.txt
View File

@ -15,4 +15,13 @@ target_link_libraries(texdump PUBLIC libeuropa)
set_target_properties(texdump PROPERTIES
CXX_STANDARD 20
CXX_STANDARD_REQUIRED ON
)
)
add_executable(paktest paktest.cpp)
target_link_libraries(paktest PUBLIC libeuropa)
set_target_properties(paktest PROPERTIES
CXX_STANDARD 20
CXX_STANDARD_REQUIRED ON
)

6
src/tools/europa_pack_extractor.cpp
View File

@ -6,12 +6,12 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#include <europa/io/PakReader.h>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <europa/io/PakReader.h>
namespace fs = std::filesystem;
int main(int argc, char** argv) {
@ -38,7 +38,7 @@ int main(int argc, char** argv) {
return 1;
}
for(auto& [ filename, file ] : reader.GetFiles()) {
for(auto& [filename, file] : reader.GetFiles()) {
auto nameCopy = filename;
#ifndef _WIN32

42
src/tools/paktest.cpp Normal file
View File

@ -0,0 +1,42 @@
//
// EuropaTools
//
// (C) 2021-2022 modeco80 <lily.modeco80@protonmail.ch>
//
// SPDX-License-Identifier: LGPL-3.0-or-later
//
// A test utility to regurgitate a pak.
#include <europa/io/PakReader.h>
#include <europa/io/PakWriter.h>
#include <iostream>
#include <fstream>
int main(int argc, char** argv) {
std::ifstream ifs(argv[1], std::ifstream::binary);
std::ofstream ofs("new_archive.pak", std::ofstream::binary);
europa::io::PakWriter writer;
writer.Init();
// Read pak data and vomit it into the writer.
// This will temporarily consume 2x the memory (so about 240mb for the biggest paks I've seen),
// but the writer will contain the first copy,
// until it's cleared.
{
europa::io::PakReader reader(ifs);
reader.ReadData();
for (auto &[filename, file]: reader.GetFiles()) {
writer.AddFile(filename, file);
}
}
writer.Write(ofs);
std::cout << "Wrote regurgitated archive to new.pak!\n";
return 0;
}

9
src/tools/texdump.cpp
View File

@ -6,15 +6,13 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
//
#include <europa/io/YatfReader.h>
#include <pixel/ImageWriter.h>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <europa/io/YatfReader.h>
#include <pixel/ImageWriter.h>
namespace fs = std::filesystem;
int main(int argc, char** argv) {
if(argc != 2) {
@ -38,8 +36,7 @@ int main(int argc, char** argv) {
reader.ReadImage();
pixel::ImageWriter writer{};
pixel::ImageWriter writer {};
auto outPath = fs::path(argv[1]).replace_extension(".png");