From 2c060e8f4c5a42ac51cef9b00e87c502a07c30eb Mon Sep 17 00:00:00 2001 From: modeco80 Date: Mon, 26 Feb 2024 01:50:21 -0500 Subject: [PATCH] init There's still LOTS of work to do with the ziff stream... I hate IFF, and this weird crusty Z-Axis deriviative is NO better. --- .clang-format | 44 ++++++++++++++ .editorconfig | 11 ++++ .gitignore | 6 ++ AUTHORS | 1 + CMakeLists.txt | 25 ++++++++ LICENSE | 7 +++ cmake/CompilerFlags.cmake | 72 +++++++++++++++++++++++ cmake/Policies.cmake | 22 +++++++ cmake/ProjectFuncs.cmake | 24 ++++++++ lib/base/CMakeLists.txt | 7 +++ lib/base/ErrorOr.hpp | 104 ++++++++++++++++++++++++++++++++++ lib/base/FixedString.hpp | 35 ++++++++++++ lib/base/FourCC.hpp | 24 ++++++++ lib/base/OffsetPtr.hpp | 88 ++++++++++++++++++++++++++++ lib/base/Types.hpp | 19 +++++++ lib/base/io/FileStream.cpp | 90 +++++++++++++++++++++++++++++ lib/base/io/FileStream.hpp | 41 ++++++++++++++ lib/base/io/MemoryStream.cpp | 56 ++++++++++++++++++ lib/base/io/MemoryStream.hpp | 30 ++++++++++ lib/base/io/Stream.cpp | 13 +++++ lib/base/io/Stream.hpp | 74 ++++++++++++++++++++++++ lib/dmbmx/CMakeLists.txt | 9 +++ lib/dmbmx/ZHash.hpp | 42 ++++++++++++++ lib/dmbmx/ZiffChunk.hpp | 20 +++++++ lib/dmbmx/ZiffInputStream.cpp | 85 +++++++++++++++++++++++++++ lib/dmbmx/ZiffInputStream.hpp | 44 ++++++++++++++ progs/zalgo/CMakeLists.txt | 10 ++++ progs/zalgo/main.cpp | 34 +++++++++++ 28 files changed, 1037 insertions(+) create mode 100755 .clang-format create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 cmake/CompilerFlags.cmake create mode 100644 cmake/Policies.cmake create mode 100644 cmake/ProjectFuncs.cmake create mode 100644 lib/base/CMakeLists.txt create mode 100644 lib/base/ErrorOr.hpp create mode 100644 lib/base/FixedString.hpp create mode 100644 lib/base/FourCC.hpp create mode 100644 lib/base/OffsetPtr.hpp create mode 100644 lib/base/Types.hpp create mode 100644 lib/base/io/FileStream.cpp create mode 100644 lib/base/io/FileStream.hpp create mode 100644 lib/base/io/MemoryStream.cpp create mode 100644 lib/base/io/MemoryStream.hpp create mode 100644 lib/base/io/Stream.cpp create mode 100644 lib/base/io/Stream.hpp create mode 100644 lib/dmbmx/CMakeLists.txt create mode 100644 lib/dmbmx/ZHash.hpp create mode 100644 lib/dmbmx/ZiffChunk.hpp create mode 100644 lib/dmbmx/ZiffInputStream.cpp create mode 100644 lib/dmbmx/ZiffInputStream.hpp create mode 100644 progs/zalgo/CMakeLists.txt create mode 100644 progs/zalgo/main.cpp diff --git a/.clang-format b/.clang-format new file mode 100755 index 0000000..6142558 --- /dev/null +++ b/.clang-format @@ -0,0 +1,44 @@ +BasedOnStyle: Google + +# force T* or T& +DerivePointerAlignment: false +PointerAlignment: Left + +TabWidth: 4 +IndentWidth: 4 +UseTab: Always +IndentPPDirectives: BeforeHash + +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true + +BinPackArguments: true +BinPackParameters: true +BreakConstructorInitializers: BeforeColon +BreakStringLiterals: false + +ColumnLimit: 0 +CompactNamespaces: false + +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ContinuationIndentWidth: 0 + +# turning this on causes major issues with initializer lists +Cpp11BracedListStyle: false +SpaceBeforeCpp11BracedList: true + +FixNamespaceComments: true + +NamespaceIndentation: All +ReflowComments: true + +SortIncludes: CaseInsensitive +SortUsingDeclarations: true + +SpacesInSquareBrackets: false +SpaceBeforeParens: Never +SpacesBeforeTrailingComments: 1 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0b53eb8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 + +# specifically for YAML +[{yml, yaml}] +indent_style = space diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..397d47e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +build/ +/out +cmake-build-*/ +/.idea +.cache/ +/compile_commands.json diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..92c9561 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Lily Tsuru diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..93188d9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.15) + +project(dmtools + LANGUAGES CXX +) + +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + +include(Policies) +include(CompilerFlags) +include(ProjectFuncs) + +# required system dependencies +find_package(PkgConfig REQUIRED) +find_package(Threads REQUIRED) + + +dmtools_set_alternate_linker() + +# libraries +add_subdirectory(lib/base) +add_subdirectory(lib/dmbmx) + +# programs +add_subdirectory(progs/zalgo) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c62a4df --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2023 The DMBMX2Tools Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cmake/CompilerFlags.cmake b/cmake/CompilerFlags.cmake new file mode 100644 index 0000000..c80b948 --- /dev/null +++ b/cmake/CompilerFlags.cmake @@ -0,0 +1,72 @@ +# Core compile arguments used for dmtools +set(DMTOOLS_CORE_COMPILE_ARGS "-Wall -Wformat=2 -fstrict-flex-arrays=3 -fstack-clash-protection -fstack-protector-strong ") +set(DMTOOLS_CORE_LINKER_ARGS "-fuse-ld=${DMTOOLS_LINKER}") + +if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + set(DMTOOLS_CORE_COMPILE_ARGS "${DMTOOLS_CORE_COMPILE_ARGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3") + + # If on Release use link-time optimizations. + # On clang we use ThinLTO for even better build performance. + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(DMTOOLS_CORE_COMPILE_ARGS "${DMTOOLS_CORE_COMPILE_ARGS} -flto=thin") + set(DMTOOLS_CORE_LINKER_ARGS "${DMTOOLS_CORE_LINKER_ARGS} -flto=thin") + else() + set(DMTOOLS_CORE_COMPILE_ARGS "${DMTOOLS_CORE_COMPILE_ARGS} -flto") + set(DMTOOLS_CORE_LINKER_ARGS "${DMTOOLS_CORE_LINKER_ARGS} -flto") + endif() +else() + # TODO: This currently assumes libstdc++, later on we should *probably* set this with some detection + # to check the current c++ standard library (but we only will compile and run with libstdc++ anyways since libc++ is a bit lacking with even c++17, so..) + set(DMTOOLS_CORE_COMPILE_ARGS "${DMTOOLS_CORE_COMPILE_ARGS} -D_GLIBCXX_ASSERTIONS") +endif() + +set(_DMTOOLS_CORE_WANTED_SANITIZERS "") + +if("asan" IN_LIST DMTOOLS_BUILD_FEATURES) + # Error if someone's trying to mix asan and tsan together, + # they aren't compatible. + if("tsan" IN_LIST DMTOOLS_BUILD_FEATURES) + message(FATAL_ERROR "ASAN and TSAN cannot be used together.") + endif() + + message(STATUS "Enabling ASAN because it was in DMTOOLS_BUILD_FEATURES") + list(APPEND _DMTOOLS_CORE_WANTED_SANITIZERS "address") +endif() + +if("tsan" IN_LIST DMTOOLS_BUILD_FEATURES) + if("asan" IN_LIST DMTOOLS_BUILD_FEATURES) + message(FATAL_ERROR "ASAN and TSAN cannot be used together.") + endif() + + message(STATUS "Enabling TSAN because it was in DMTOOLS_BUILD_FEATURES") + list(APPEND _DMTOOLS_CORE_WANTED_SANITIZERS "thread") +endif() + +if("ubsan" IN_LIST DMTOOLS_BUILD_FEATURES) + message(STATUS "Enabling UBSAN because it was in DMTOOLS_BUILD_FEATURES") + list(APPEND _DMTOOLS_CORE_WANTED_SANITIZERS "undefined") +endif() + +list(LENGTH _DMTOOLS_CORE_WANTED_SANITIZERS _DMTOOLS_CORE_WANTED_SANITIZERS_LENGTH) +if(NOT _DMTOOLS_CORE_WANTED_SANITIZERS_LENGTH EQUAL 0) + list(JOIN _DMTOOLS_CORE_WANTED_SANITIZERS "," _DMTOOLS_CORE_WANTED_SANITIZERS_ARG) + message(STATUS "Enabled sanitizers: ${_DMTOOLS_CORE_WANTED_SANITIZERS_ARG}") + set(DMTOOLS_CORE_COMPILE_ARGS "${DMTOOLS_CORE_COMPILE_ARGS} -fsanitize=${_COLLABVM_CORE_WANTED_SANITIZERS_ARG}") + set(DMTOOLS_CORE_LINKER_ARGS "${DMTOOLS_CORE_LINKER_ARGS} -fsanitize=${_COLLABVM_CORE_WANTED_SANITIZERS_ARG}") +endif() + +# Set core CMake toolchain variables so that they get applied to all projects. +# A bit nasty, but /shrug, this way our third party libraries can be mostly sanitized/etc as well. + +set(CMAKE_C_FLAGS "${DMTOOLS_CORE_COMPILE_ARGS}") +set(CMAKE_CXX_FLAGS "${DMTOOLS_CORE_COMPILE_ARGS}") + +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} -O0 -g3") +set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS} -O3 -g3") +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} -O3") + +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -O0 -g3") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS} -O3 -g3") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O3") + +set(CMAKE_EXE_LINKER_FLAGS "${DMTOOLS_CORE_LINKER_ARGS} -Wl,-z,noexecstack,-z,relro,-z,now") diff --git a/cmake/Policies.cmake b/cmake/Policies.cmake new file mode 100644 index 0000000..d7a084b --- /dev/null +++ b/cmake/Policies.cmake @@ -0,0 +1,22 @@ +# CMake policy configuration + +# Macro to enable new CMake policy. +# Makes this file a *LOT* shorter. +macro (_new_cmake_policy policy) + if(POLICY ${policy}) + #message(STATUS "Enabling new policy ${policy}") + cmake_policy(SET ${policy} NEW) + endif() +endmacro() + +_new_cmake_policy(CMP0026) # CMake 3.0: Disallow use of the LOCATION property for build targets. +_new_cmake_policy(CMP0042) # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name +_new_cmake_policy(CMP0046) # warn about non-existent dependencies +_new_cmake_policy(CMP0048) # CMake 3.0+: project() command now maintains VERSION +_new_cmake_policy(CMP0054) # CMake 3.1: Only interpret if() arguments as variables or keywords when unquoted. +_new_cmake_policy(CMP0056) # try_compile() linker flags +_new_cmake_policy(CMP0066) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE +_new_cmake_policy(CMP0067) # CMake 3.8: try_compile(): honor language standard variables (like C++11) +_new_cmake_policy(CMP0068) # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`. +_new_cmake_policy(CMP0075) # CMake 3.12+: Include file check macros honor `CMAKE_REQUIRED_LIBRARIES` +_new_cmake_policy(CMP0077) # CMake 3.13+: option() honors normal variables. diff --git a/cmake/ProjectFuncs.cmake b/cmake/ProjectFuncs.cmake new file mode 100644 index 0000000..62e89cd --- /dev/null +++ b/cmake/ProjectFuncs.cmake @@ -0,0 +1,24 @@ +function(dmtools_target target) + target_compile_definitions(${target} PRIVATE "$<$:DMTOOLS_DEBUG>") + #target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}) + target_compile_features(${target} PRIVATE cxx_std_20) + target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/lib ${CMAKE_CURRENT_BINARY_DIR}) +endfunction() + +function(dmtools_set_alternate_linker) + find_program(LINKER_EXECUTABLE ld.${DMTOOLS_LINKER} ${COLLABVM_LINKER}) + if(LINKER_EXECUTABLE) + message(STATUS "Using ${DMTOOLS_LINKER} as linker") + else() + message(FATAL_ERROR "Linker ${DMTOOLS_LINKER} does not exist on your system. Please specify one which does or omit this option from your configure command.") + endif() +endfunction() + + +# Set a default linker if the user never provided one. +# This defaults based on the detected compiler to the "best" linker possible +if(NOT DMTOOLS_LINKER AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(DMTOOLS_LINKER "lld") +elseif(NOT DMTOOLS_LINKER) + set(DMTOOLS_LINKER "bfd") +endif() diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt new file mode 100644 index 0000000..92f6dac --- /dev/null +++ b/lib/base/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(dmtoolsBase + io/Stream.cpp + io/FileStream.cpp + io/MemoryStream.cpp +) + +dmtools_target(dmtoolsBase) diff --git a/lib/base/ErrorOr.hpp b/lib/base/ErrorOr.hpp new file mode 100644 index 0000000..f48bafa --- /dev/null +++ b/lib/base/ErrorOr.hpp @@ -0,0 +1,104 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once +#include +#include +#include + +namespace dmtools { + + template + struct ErrorOr { + private: + using Variant = std::variant; + + public: + constexpr static auto IsVoid = false; + ErrorOr() = default; + ErrorOr(const ErrorOr&) = default; + ErrorOr(ErrorOr&&) = default; + + ErrorOr(const T& v) + : maybeVariant(v) {} + ErrorOr(const std::error_code& v) + : maybeVariant(v) {} + + ErrorOr& operator=(const T& value) { + maybeVariant = value; + return *this; + } + + ErrorOr& operator=(const std::error_code& ec) { + maybeVariant = ec; + return *this; + } + + bool HasValue() const { + if(maybeVariant) + return std::holds_alternative(*maybeVariant); + + // No value. + return false; + } + + bool HasError() const { + if(maybeVariant) + return std::holds_alternative(*maybeVariant); + + // No error. Possibly in the default constructed state + return false; + } + + // ASSERT TODO + + T& Value() { return std::get(maybeVariant.value()); } + + const T& Value() const { return std::get(maybeVariant.value()); } + + operator T&() { return Value(); } + operator const T&() { return Value(); } + + std::error_code& Error() { return std::get(maybeVariant.value()); } + + const std::error_code& Error() const { return std::get(maybeVariant.value()); } + + operator std::error_code&() { return Error(); } + operator const std::error_code&() const { return Error(); } + + private: + std::optional maybeVariant; + }; + + template <> + struct ErrorOr { + constexpr static auto IsVoid = true; + ErrorOr() = default; + ErrorOr(const ErrorOr&) = default; + ErrorOr(ErrorOr&&) = default; + ErrorOr(const std::error_code& v) + : maybeEc(v) {} + + ErrorOr& operator=(const std::error_code& ec) { + maybeEc = ec; + return *this; + } + + bool HasError() const { return maybeEc.has_value(); } + + std::error_code& Error() { return maybeEc.value(); } + + const std::error_code& Error() const { return maybeEc.value(); } + + operator std::error_code&() { return Error(); } + operator const std::error_code&() const { return Error(); } + + private: + std::optional maybeEc; + }; + + inline std::error_code ThrowErrno() { + return { errno, std::system_category() }; + } + +} // namespace dmtools diff --git a/lib/base/FixedString.hpp b/lib/base/FixedString.hpp new file mode 100644 index 0000000..26b92b2 --- /dev/null +++ b/lib/base/FixedString.hpp @@ -0,0 +1,35 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include + +namespace dmtools { + + /// A compile-time string. Usable as a C++20 cNTTP. + template + struct FixedString { + char buf[N + 1] {}; + + constexpr FixedString(const char* s) { + for(usize i = 0; i != N; ++i) + buf[i] = s[i]; + } + + constexpr operator const char*() const { + return buf; + } + + constexpr operator std::string_view() const { + return { buf, N }; + } + + [[nodiscard]] constexpr usize Length() const { return N; } + }; + + template + FixedString(char const (&)[N]) -> FixedString; + +} // namespace ssxtools::core diff --git a/lib/base/FourCC.hpp b/lib/base/FourCC.hpp new file mode 100644 index 0000000..01cb9a9 --- /dev/null +++ b/lib/base/FourCC.hpp @@ -0,0 +1,24 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once +#include +#include +#include + +namespace dmtools { + + /// strong type for FourCCs + enum class FourCCT : u32 {}; + + /// Compile-time FourCC generation + template + consteval FourCCT FourCC() { + static_assert(fccString.Length() == 4, "Provided string is not a FourCC"); + switch(Endian) { + case std::endian::little: return static_cast((fccString[0] << 24) | (fccString[1] << 16) | (fccString[2] << 8) | fccString[3]); + case std::endian::big: return static_cast((fccString[0]) | (fccString[1] << 8) | (fccString[2] << 16) | (fccString[3] << 24)); + } + } + +} // namespace ssxtools::core diff --git a/lib/base/OffsetPtr.hpp b/lib/base/OffsetPtr.hpp new file mode 100644 index 0000000..a0f4f08 --- /dev/null +++ b/lib/base/OffsetPtr.hpp @@ -0,0 +1,88 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once +#include +#include +#include +#include + +namespace dmtools { + + namespace detail { + + template + constexpr NativeT* CreatePointerFromAddend(void* BasePointer, OffsetType addend) noexcept { + return std::bit_cast(static_cast(BasePointer) + addend); + } + + } // namespace detail + + /// An "auto-resolving" semi-sweet (/s) pointer type. + /// This is designed to allow resolving offsets in data for + /// games written before 64-bit pointers were common/used at all. + /// This allows looking up data a lot easier :) + /// + /// [NativeT] is the type of data this would point to + /// [OffsetType] is the type of data the "pointer" is repressented as + template + struct OffsetPtr final { + using Type = std::remove_cvref_t; + using Pointer = Type*; + using ConstPointer = const Type*; + + /// Set the offset. Duh! + constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; } + + [[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; } + + [[nodiscard]] constexpr Pointer operator()(void* baseAddr) const noexcept { + // While yucky, it should show problem areas which aren't checking things + // immediately rather than read invalid data that might do much worse. + if(rawOffset == 0) + return nullptr; + + return detail::CreatePointerFromAddend(baseAddr, rawOffset); + } + + template + constexpr OffsetPtr& PtrCast() { + // Safety: The data layout of OffsetPtr<> stays + // the exact same regardless of the result type, therefore + // even though this is *techinically* UB (? using bit_cast it shouldn't be ?), + // this isn't problematic + return *std::bit_cast*>(this); + } + + private: + OffsetType rawOffset; + }; + + /// Like OffsetPtr but for arrays of data + template + struct OffsetArrayPtr final { + using Type = std::remove_cvref_t; + using Pointer = Type*; + using ConstPointer = const Type*; + + using Span = std::span; + + /// Set the offset. Duh! + constexpr void Set(OffsetType newOffset) noexcept { rawOffset = newOffset; } + + [[nodiscard]] constexpr OffsetType Raw() const noexcept { return rawOffset; } + + [[nodiscard]] constexpr Span operator()(void* baseAddr, OffsetType length) const noexcept { + // While yucky, it should show problem areas which aren't checking things + // immediately rather than read invalid data that might do much worse. + if(rawOffset == 0 || length == 0) + return {}; + + return { detail::CreatePointerFromAddend(baseAddr, rawOffset), length }; + } + + private: + OffsetType rawOffset; + }; + +} // namespace ssxtools::core diff --git a/lib/base/Types.hpp b/lib/base/Types.hpp new file mode 100644 index 0000000..6083ce0 --- /dev/null +++ b/lib/base/Types.hpp @@ -0,0 +1,19 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include + +using u8 = std::uint8_t; +using i8 = std::int8_t; +using u16 = std::uint16_t; +using i16 = std::int16_t; +using u32 = std::uint32_t; +using i32 = std::int32_t; +using u64 = std::uint64_t; +using i64 = std::int64_t; +using usize = std::size_t; +using isize = std::intptr_t; + diff --git a/lib/base/io/FileStream.cpp b/lib/base/io/FileStream.cpp new file mode 100644 index 0000000..5f1a85d --- /dev/null +++ b/lib/base/io/FileStream.cpp @@ -0,0 +1,90 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#include +#include + +namespace dmtools::io { + + constexpr auto DmToLibc(StreamSeekDirection dir) { + using enum StreamSeekDirection; + switch(dir) { + case Beg: return SEEK_SET; + case Cur: return SEEK_CUR; + case End: return SEEK_END; + default: return SEEK_SET; + } + } + + constexpr auto DmOpenMode(FileStream::OpenMode mode) { + using enum FileStream::OpenMode; + switch(mode) { + case Read: return "rb"; + case ReadWrite: return "wb+"; + } + } + + FileStream::~FileStream() { + Close(); + } + + bool FileStream::Open(const std::string& filename, OpenMode mode) { + file = std::fopen(filename.c_str(), DmOpenMode(mode)); + return Ok(); + } + + void FileStream::Flush() { + fflush(static_cast(file)); + } + + void FileStream::Close() { + if(Ok()) { + fclose(static_cast(file)); + } + } + + StreamDiff FileStream::ReadSome(u8* buffer, usize length) { + if(Ok()) + return std::fread(static_cast(buffer), 1, length, static_cast(file)); + return -1; + } + + StreamDiff FileStream::WriteSome(const u8* buffer, usize length) { + if(Ok()) + return std::fwrite(static_cast(buffer), length, 1, static_cast(file)); + return -1; + } + + StreamDiff FileStream::Seek(StreamDiff where, StreamSeekDirection dir) { + if(!Ok()) + return -1; + +#ifdef __GLIBC__ + return fseeko64(static_cast(file), where, DmToLibc(dir)); +#else + return fseek(static_cast(file), where, DmToLibc(dir)); +#endif + } + + StreamDiff FileStream::Tell() const { + if(!Ok()) + return -1; +#ifdef __GLIBC__ + return ftello64(static_cast(file)); +#else + return ftell(static_cast(file)); +#endif + } + + bool FileStream::Ok() const { + return file != nullptr; + } + + bool FileStream::Eof() const { + if(!Ok()) + return true; + + return feof(static_cast(file)); + } + +} // namespace dmtools::io diff --git a/lib/base/io/FileStream.hpp b/lib/base/io/FileStream.hpp new file mode 100644 index 0000000..40b2cc7 --- /dev/null +++ b/lib/base/io/FileStream.hpp @@ -0,0 +1,41 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once +#include + +namespace dmtools::io { + + /// A I/O stream using libc FILE*. + struct FileStream : public Stream { + enum class OpenMode : u8 { + Read, + ReadWrite + }; + + virtual ~FileStream(); + + bool Open(const std::string& path, OpenMode mode); + + void Flush() override; + + void Close(); + + StreamDiff ReadSome(u8* buffer, usize length) override; + + StreamDiff WriteSome(const u8* buffer, usize length) override; + + StreamDiff Seek(StreamDiff where, StreamSeekDirection dir) override; + + StreamDiff Tell() const override; + + bool Ok() const override; + + bool Eof() const override; + + + private: + void* file{}; + }; + +} diff --git a/lib/base/io/MemoryStream.cpp b/lib/base/io/MemoryStream.cpp new file mode 100644 index 0000000..f4621ca --- /dev/null +++ b/lib/base/io/MemoryStream.cpp @@ -0,0 +1,56 @@ +#include +#include + +namespace dmtools::io { + MemoryStream::MemoryStream(u8* memory, usize size) { + beg = memory; + cur = beg; + end = memory + size; + this->size = size; + } + + MemoryStream::~MemoryStream() { + } + + StreamDiff MemoryStream::ReadSome(u8* buffer, usize length) { + if(Eof()) + return -1; + memcpy(&buffer[0], cur, length); + cur += length; + return length; + } + + StreamDiff MemoryStream::WriteSome(const u8* buffer, usize length) { + return -1; + }; + + StreamDiff MemoryStream::Seek(StreamDiff where, StreamSeekDirection dir) { + using enum StreamSeekDirection; + auto last = Tell(); + switch(dir) { + case Beg: + cur = beg + where; + break; + case Cur: + cur += where; + break; + case End: // TODO + cur = end; + break; + } + return last; + } + + StreamDiff MemoryStream::Tell() const { + return (cur - beg); + } + + bool MemoryStream::Ok() const { + return !Eof(); + } + + bool MemoryStream::Eof() const { + return cur == end; + }; + +} // namespace dmtools::io diff --git a/lib/base/io/MemoryStream.hpp b/lib/base/io/MemoryStream.hpp new file mode 100644 index 0000000..1a355f4 --- /dev/null +++ b/lib/base/io/MemoryStream.hpp @@ -0,0 +1,30 @@ +#include + +namespace dmtools::io { + + struct MemoryStream : public Stream { + MemoryStream(u8* memory, usize size); + + virtual ~MemoryStream(); + + StreamDiff ReadSome(u8* buffer, usize length) override; + + StreamDiff WriteSome(const u8* buffer, usize length) override; + + StreamDiff Seek(StreamDiff where, StreamSeekDirection dir) override; + + StreamDiff Tell() const override; + + bool Ok() const override; + + bool Eof() const override; + + + private: + u8* beg; + u8* cur; + u8* end; + usize size; + }; + +} diff --git a/lib/base/io/Stream.cpp b/lib/base/io/Stream.cpp new file mode 100644 index 0000000..3354051 --- /dev/null +++ b/lib/base/io/Stream.cpp @@ -0,0 +1,13 @@ +#include + +namespace dmtools::io { + + StreamDiff Stream::ReadSome(std::vector& buffer) { + return ReadSome(buffer.data(), buffer.size()); + } + + StreamDiff Stream::WriteSome(const std::vector& buffer) { + return WriteSome(buffer.data(), buffer.size()); + } + +} // namespace dmtools::io diff --git a/lib/base/io/Stream.hpp b/lib/base/io/Stream.hpp new file mode 100644 index 0000000..86cfadb --- /dev/null +++ b/lib/base/io/Stream.hpp @@ -0,0 +1,74 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#pragma once +#include +#include + +namespace dmtools::io { + + enum class StreamSeekDirection : u8 { + Beg, + Cur, + End + }; + + using StreamDiff = i64; + + // TODO: Communicate error conditions using ErrorOr<> instead of + // -1 values. + + /// A abstract stream of bytes. We use a custom implementation + /// instead of C++ IO streams mostly for control reasons, and + /// also to avoid exceptions (we'll use ErrorOr for errors later on) + struct Stream { + Stream() = default; + virtual ~Stream() = default; + + Stream(const Stream&) = delete; + Stream(Stream&&) = delete; + + virtual void Flush() { return; } + + virtual StreamDiff ReadSome(u8* buffer, usize length) = 0; + + StreamDiff ReadSome(std::vector& buffer); + + virtual StreamDiff WriteSome(const u8* buffer, usize length) { + // Default implementation disallows writes. Can be changed + return -1; + } + + StreamDiff WriteSome(const std::vector& buffer); + + /// Reads a single POD type from stream + template + inline bool ReadObject(T& obj) { + auto sz = ReadSome(std::bit_cast(&obj), sizeof(T)); + if(sz != sizeof(T)) + return false; + return true; + } + + template + inline bool WriteObject(const T& obj) { + auto sz = WriteSome(std::bit_cast(&obj), sizeof(T)); + if(sz != sizeof(T)) + return false; + return true; + } + + virtual StreamDiff Seek(StreamDiff where, StreamSeekDirection dir) = 0; + + virtual StreamDiff Tell() const = 0; + + virtual bool Ok() const = 0; + + virtual bool Eof() const = 0; + + inline operator bool() const { + return Ok() && !Eof(); + } + }; + +} // namespace dmtools diff --git a/lib/dmbmx/CMakeLists.txt b/lib/dmbmx/CMakeLists.txt new file mode 100644 index 0000000..b8b2f82 --- /dev/null +++ b/lib/dmbmx/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(dmtoolsDmBmx + ZiffInputStream.cpp +) + +dmtools_target(dmtoolsDmBmx) + +target_link_libraries(dmtoolsDmBmx + dmtoolsBase +) diff --git a/lib/dmbmx/ZHash.hpp b/lib/dmbmx/ZHash.hpp new file mode 100644 index 0000000..03ae20a --- /dev/null +++ b/lib/dmbmx/ZHash.hpp @@ -0,0 +1,42 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#include + +namespace dmtools::dmbmx { + + /// Z-Axis' variant of the PJW hash algorithm. + /// + /// # Safety + /// This function should *not* be given any input other than ASCII. + constexpr u32 zStringGenerateHash(const char* szString) { + const char* cursor = szString; + u32 hashTally {}; + + while(*cursor) { + u32 signExtendedVal = *cursor; + + // If the character is in the ASCII range for lowercase alphabetical letters + // make it uppercase, by XORing with 0x20 (-= 0x20 works too). If not, don't modify the value. + if((signExtendedVal - 'a') < 26) + signExtendedVal ^= 0x20; + + // Do PJW hash + signExtendedVal += (hashTally << 4); + hashTally = signExtendedVal & 0xf0000000; + + if(hashTally != 0) + signExtendedVal ^= (hashTally >> 24); + + hashTally = signExtendedVal & ~hashTally; + + // Advance to the next byte + cursor++; + } + + return hashTally; + } + + static_assert(zStringGenerateHash("Park.zsc") == 0x066e3a33, "zStringGenerateHash is broken"); + +} // namespace dmtools::dmbmx diff --git a/lib/dmbmx/ZiffChunk.hpp b/lib/dmbmx/ZiffChunk.hpp new file mode 100644 index 0000000..7fc4c7f --- /dev/null +++ b/lib/dmbmx/ZiffChunk.hpp @@ -0,0 +1,20 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#include + +namespace dmtools::dmbmx { + + /// Z-Axis IFF chunk header. + struct ZiffChunkHeader { + FourCCT ckId; + u32 ckSz; + + /// returns true if the chunk id matches the specific FourCC + template + constexpr bool Is() { + return ckId == FourCC(); + } + }; + +} diff --git a/lib/dmbmx/ZiffInputStream.cpp b/lib/dmbmx/ZiffInputStream.cpp new file mode 100644 index 0000000..86d239d --- /dev/null +++ b/lib/dmbmx/ZiffInputStream.cpp @@ -0,0 +1,85 @@ +#include +#include +#include + +namespace dmtools::dmbmx { + + ZiffInputStream::ZiffInputStream(io::Stream& stream, FourCCT expectedFormCkId) + : stream(stream), expectedFormCkId(expectedFormCkId) { + } + + void ZiffInputStream::SetChunkCallback(ChunkCallback&& chunkCallback) { + this->callback = std::move(chunkCallback); + } + + void ZiffInputStream::Process() { + //if(!NextChunkImpl(true)) + // return; + + while(!stream.Eof()) { + if(!NextChunkImpl(false)) + return; + } + } + + void ZiffInputStream::Rewind() { + stream.Seek(0, io::StreamSeekDirection::Beg); + } + + bool ZiffInputStream::NextChunkImpl(bool isForm) { +#if 0 + if(!stream.ReadObject(currentChunk.chunkHeader)) + return false; + + printf("FUCK: %08x %d\n", currentChunk.chunkHeader.ckId, currentChunk.chunkHeader.ckSz); + + if(isForm) { + if(currentChunk.chunkHeader.ckId != expectedFormCkId) { + auto* expectedBytes = std::bit_cast(&expectedFormCkId); + auto* gotBytes = std::bit_cast(¤tChunk.chunkHeader.ckId); + std::printf("WARNING: Expected Form ckId '%c%c%c%c', got '%c%c%c%c'\n", + expectedBytes[0], expectedBytes[1], expectedBytes[2], expectedBytes[3], + gotBytes[0], gotBytes[1], gotBytes[2], gotBytes[3]); + } + } else { + + currentChunk.chunkData.resize(sizeof(currentChunk.chunkHeader) + currentChunk.chunkHeader.ckSz); + + memcpy(currentChunk.chunkData.data(), ¤tChunk.chunkHeader, sizeof(currentChunk.chunkHeader)); + + if(stream.ReadSome(currentChunk.chunkData.data() + sizeof(currentChunk.chunkHeader), currentChunk.chunkHeader.ckSz) != currentChunk.chunkHeader.ckSz) + return false; + } + + if(callback) + callback(currentChunk); + return true; +#endif + + if(!stream.ReadObject(currentChunk.chunkHeader)) + return false; + + printf("FUCK: %08x %d\n", currentChunk.chunkHeader.ckId, currentChunk.chunkHeader.ckSz); + + if(isForm) { + if(currentChunk.chunkHeader.ckId != expectedFormCkId) { + auto* expectedBytes = std::bit_cast(&expectedFormCkId); + auto* gotBytes = std::bit_cast(¤tChunk.chunkHeader.ckId); + std::printf("WARNING: Expected Form ckId '%c%c%c%c', got '%c%c%c%c'\n", + expectedBytes[0], expectedBytes[1], expectedBytes[2], expectedBytes[3], + gotBytes[0], gotBytes[1], gotBytes[2], gotBytes[3]); + } + }// else { + + currentChunk.chunkData.resize(currentChunk.chunkHeader.ckSz); + + if(stream.ReadSome(currentChunk.chunkData) != currentChunk.chunkHeader.ckSz) + return false; + //} + + if(callback) + callback(currentChunk); + return true; + } + +} // namespace dmtools::dmbmx diff --git a/lib/dmbmx/ZiffInputStream.hpp b/lib/dmbmx/ZiffInputStream.hpp new file mode 100644 index 0000000..2c64bae --- /dev/null +++ b/lib/dmbmx/ZiffInputStream.hpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include + +#include "base/FourCC.hpp" + +namespace dmtools::dmbmx { + + /// A input stream that reads a ZIFF stream and posts chunks. + struct ZiffInputStream { + /// A ZIFF chunk. + struct Chunk { + ZiffChunkHeader chunkHeader; + std::vector chunkData; + + /// Casts the data to some other type. + template + T* As() { + return std::bit_cast(chunkData.data()); + } + }; + + using ChunkCallback = std::function; + + explicit ZiffInputStream(io::Stream& stream, FourCCT expectedFormCkId); + + void SetChunkCallback(ChunkCallback&& chunkCallback); + + void Process(); + + void Rewind(); + + private: + + bool NextChunkImpl(bool isForm); + + io::Stream& stream; + FourCCT expectedFormCkId {}; + Chunk currentChunk {}; + ChunkCallback callback {}; + }; + +} // namespace dmtools::dmbmx diff --git a/progs/zalgo/CMakeLists.txt b/progs/zalgo/CMakeLists.txt new file mode 100644 index 0000000..afa2643 --- /dev/null +++ b/progs/zalgo/CMakeLists.txt @@ -0,0 +1,10 @@ +add_executable(zalgo + main.cpp +) + +dmtools_target(zalgo) + +target_link_libraries(zalgo PRIVATE + dmtoolsBase + dmtoolsDmBmx +) diff --git a/progs/zalgo/main.cpp b/progs/zalgo/main.cpp new file mode 100644 index 0000000..0125ac4 --- /dev/null +++ b/progs/zalgo/main.cpp @@ -0,0 +1,34 @@ +// Copyright 2024 The DMBMX2Tools Authors +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include + +int main(int argc, char** argv) { + auto fs = dmtools::io::FileStream {}; + + if(fs.Open("/data/sda/lily/games/dmbmx2/ps2/ASSETS/LEVELS/WOODWARD/WOODWARD.ZIT", dmtools::io::FileStream::OpenMode::Read)) { + auto ziffStream = dmtools::dmbmx::ZiffInputStream { fs, dmtools::FourCC<"FSCN">() }; + + ziffStream.SetChunkCallback([&](dmtools::dmbmx::ZiffInputStream::Chunk& chunk) { + auto stream = dmtools::io::MemoryStream { chunk.chunkData.data(), chunk.chunkData.size() }; + auto terrStream = dmtools::dmbmx::ZiffInputStream { stream, dmtools::FourCC<"TERR">() }; + + terrStream.SetChunkCallback([&](auto& chunk) { + auto* gotBytes = std::bit_cast(&chunk.chunkHeader.ckId); + std::printf("inner stream ckId '%c%c%c%c' with size %d\n", + gotBytes[0], gotBytes[1], gotBytes[2], gotBytes[3], chunk.chunkHeader.ckSz); + }); + + terrStream.Process(); + }); + + ziffStream.Process(); + } else { + printf("Failed to open io::FileStream\n"); + } + + return 0; +}