From 8af28318aa41e80eafb0ea5ff1574ded247ce917 Mon Sep 17 00:00:00 2001 From: Judas Drekonym Date: Sat, 7 Mar 2026 14:04:47 -0500 Subject: [PATCH] Add PHCL, PHNM, NVT support This also adds a standard Linux CMake build path. --- CMakeLists.txt | 25 + aamp_hashes.txt | 61 + source/fivex/AampFile.cpp | 877 ++++++++++++ source/fivex/AampFile.hpp | 101 ++ source/fivex/ClothInfo.cpp | 158 +++ source/fivex/ClothInfo.hpp | 32 + source/fivex/MeshShapeBuilder.cpp | 4 + source/fivex/NavMeshInfo.cpp | 345 +++++ source/fivex/NavMeshInfo.hpp | 37 + source/fivex/NvtInfo.cpp | 283 ++++ source/fivex/NvtInfo.hpp | 44 + source/havok/hkTagfile.cpp | 2108 +++++++++++++++++++++++++++++ source/havok/hkTagfile.hpp | 253 ++++ source/main.cpp | 325 ++++- 14 files changed, 4648 insertions(+), 5 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 aamp_hashes.txt create mode 100644 source/fivex/AampFile.cpp create mode 100644 source/fivex/AampFile.hpp create mode 100644 source/fivex/ClothInfo.cpp create mode 100644 source/fivex/ClothInfo.hpp create mode 100644 source/fivex/NavMeshInfo.cpp create mode 100644 source/fivex/NavMeshInfo.hpp create mode 100644 source/fivex/NvtInfo.cpp create mode 100644 source/fivex/NvtInfo.hpp create mode 100644 source/havok/hkTagfile.cpp create mode 100644 source/havok/hkTagfile.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d7a3fcf --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.14) +project(PhiveConverter CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(yaml-cpp REQUIRED) + +add_executable(PhiveConverter + source/main.cpp + source/fivex/CollisionInfo.cpp + source/fivex/PhiveWriter.cpp + source/fivex/MeshShapeBuilder.cpp + source/fivex/ClothInfo.cpp + source/fivex/NavMeshInfo.cpp + source/fivex/NvtInfo.cpp + source/havok/hkTagfile.cpp + source/fivex/AampFile.cpp +) + +target_include_directories(PhiveConverter PRIVATE source) +target_link_libraries(PhiveConverter PRIVATE yaml-cpp) + +# Copy hash dictionary next to executable +configure_file(${CMAKE_SOURCE_DIR}/aamp_hashes.txt ${CMAKE_BINARY_DIR}/aamp_hashes.txt COPYONLY) diff --git a/aamp_hashes.txt b/aamp_hashes.txt new file mode 100644 index 0000000..45a5b3b --- /dev/null +++ b/aamp_hashes.txt @@ -0,0 +1,61 @@ +# AAMP CRC32 Hash Dictionary +# One name per line. Lines starting with # are comments. +# The CRC32 hash is computed automatically from each name. +# Add new names here to resolve unknown hashes in AAMP YAML output. + +# Structural names (lists/root) +param_root +cloth_mesh_list +collidable_list + +# cloth_mesh object indexed names +cloth_mesh_0 +cloth_mesh_1 +cloth_mesh_2 +cloth_mesh_3 +cloth_mesh_4 +cloth_mesh_5 +cloth_mesh_6 +cloth_mesh_7 +cloth_mesh_8 +cloth_mesh_9 +cloth_mesh_10 +cloth_mesh_11 +cloth_mesh_12 +cloth_mesh_13 +cloth_mesh_14 +cloth_mesh_15 + +# collidable object indexed names +collidable_0 +collidable_1 +collidable_2 +collidable_3 +collidable_4 +collidable_5 +collidable_6 +collidable_7 +collidable_8 +collidable_9 +collidable_10 +collidable_11 +collidable_12 +collidable_13 +collidable_14 +collidable_15 + +# cloth_mesh object parameters +Name +BaseBone +WindPreset +BoneCorrection +BoneCorrectionAxisOrder +Twist +TwistSwingAxis +TwistAngleCoef +TwistMaxAngle +KeepBoneLength +Preset + +# collidable object parameters +ForReplace diff --git a/source/fivex/AampFile.cpp b/source/fivex/AampFile.cpp new file mode 100644 index 0000000..1974d47 --- /dev/null +++ b/source/fivex/AampFile.cpp @@ -0,0 +1,877 @@ +#include "fivex/AampFile.hpp" +#include +#include +#include +#include +#include + +namespace FiveX { + +// CRC32 lookup table +static u32 sCrc32Table[256]; +static bool sCrc32Init = false; + +static void initCrc32() { + if (sCrc32Init) return; + for (u32 i = 0; i < 256; i++) { + u32 crc = i; + for (int j = 0; j < 8; j++) + crc = (crc >> 1) ^ (crc & 1 ? 0xEDB88320 : 0); + sCrc32Table[i] = crc; + } + sCrc32Init = true; +} + +static u32 calcCrc32(const void* data, size_t size) { + initCrc32(); + u32 crc = 0xFFFFFFFF; + const u8* p = (const u8*)data; + for (size_t i = 0; i < size; i++) + crc = sCrc32Table[(crc ^ p[i]) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFF; +} + +// Known hash dictionary for phcl AAMP parameters +static std::map& getDictionary() { + static std::map dict; + static bool initialized = false; + if (!initialized) { + const char* names[] = { + // Structural names + "param_root", "cloth_mesh_list", "collidable_list", + // Indexed object names (cloth_mesh_N, collidable_N) + "cloth_mesh_0", "cloth_mesh_1", "cloth_mesh_2", "cloth_mesh_3", + "cloth_mesh_4", "cloth_mesh_5", "cloth_mesh_6", "cloth_mesh_7", + "cloth_mesh_8", "cloth_mesh_9", + "collidable_0", "collidable_1", "collidable_2", "collidable_3", + "collidable_4", "collidable_5", "collidable_6", "collidable_7", + "collidable_8", "collidable_9", + // cloth_mesh object params + "Name", "BaseBone", "WindPreset", "BoneCorrection", + "BoneCorrectionAxisOrder", "Twist", "TwistSwingAxis", + "TwistAngleCoef", "TwistMaxAngle", "KeepBoneLength", "Preset", + // collidable object params + "ForReplace", + nullptr + }; + for (int i = 0; names[i]; i++) { + u32 h = calcCrc32(names[i], strlen(names[i])); + dict[h] = names[i]; + } + initialized = true; + } + return dict; +} + +std::map& AampFile::getHashDictionary() { + return getDictionary(); +} + +void AampFile::loadHashDictionary(const std::string& path) { + initCrc32(); + auto& dict = getDictionary(); + FILE* f = fopen(path.c_str(), "r"); + if (!f) return; + char line[512]; + while (fgets(line, sizeof(line), f)) { + size_t len = strlen(line); + while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r')) line[--len] = 0; + if (len == 0 || line[0] == '#') continue; + u32 h = calcCrc32(line, len); + dict[h] = std::string(line, len); + } + fclose(f); +} + +std::string AampFile::hashToName(u32 hash) { + auto& dict = getHashDictionary(); + auto it = dict.find(hash); + if (it != dict.end()) return it->second; + char buf[16]; + snprintf(buf, sizeof(buf), "0x%08x", hash); + return buf; +} + +u32 AampFile::nameToHash(const std::string& name) { + if (name.size() > 2 && name[0] == '0' && name[1] == 'x') { + return (u32)strtoul(name.c_str(), nullptr, 16); + } + return calcCrc32(name.data(), name.size()); +} + +const char* AampFile::paramTypeName(AampParamType type) { + switch (type) { + case AampParamType::Bool: return "bool"; + case AampParamType::F32: return "f32"; + case AampParamType::Int: return "int"; + case AampParamType::Vec2: return "vec2"; + case AampParamType::Vec3: return "vec3"; + case AampParamType::Vec4: return "vec4"; + case AampParamType::Color: return "color"; + case AampParamType::String32: return "string32"; + case AampParamType::String64: return "string64"; + case AampParamType::Curve1: return "curve1"; + case AampParamType::Curve2: return "curve2"; + case AampParamType::Curve3: return "curve3"; + case AampParamType::Curve4: return "curve4"; + case AampParamType::BufferInt: return "buffer_int"; + case AampParamType::BufferF32: return "buffer_f32"; + case AampParamType::String256: return "string256"; + case AampParamType::Quat: return "quat"; + case AampParamType::U32: return "u32"; + case AampParamType::BufferU32: return "buffer_u32"; + case AampParamType::BufferBinary: return "buffer_binary"; + case AampParamType::StringRef: return "string_ref"; + case AampParamType::Special: return "special"; + default: return "unknown"; + } +} + +AampParamType AampFile::paramTypeFromName(const std::string& name) { + if (name == "bool") return AampParamType::Bool; + if (name == "f32") return AampParamType::F32; + if (name == "int") return AampParamType::Int; + if (name == "vec2") return AampParamType::Vec2; + if (name == "vec3") return AampParamType::Vec3; + if (name == "vec4") return AampParamType::Vec4; + if (name == "color") return AampParamType::Color; + if (name == "string32") return AampParamType::String32; + if (name == "string64") return AampParamType::String64; + if (name == "curve1") return AampParamType::Curve1; + if (name == "curve2") return AampParamType::Curve2; + if (name == "curve3") return AampParamType::Curve3; + if (name == "curve4") return AampParamType::Curve4; + if (name == "buffer_int") return AampParamType::BufferInt; + if (name == "buffer_f32") return AampParamType::BufferF32; + if (name == "string256") return AampParamType::String256; + if (name == "quat") return AampParamType::Quat; + if (name == "u32") return AampParamType::U32; + if (name == "buffer_u32") return AampParamType::BufferU32; + if (name == "buffer_binary") return AampParamType::BufferBinary; + if (name == "string_ref") return AampParamType::StringRef; + if (name == "special") return AampParamType::Special; + return AampParamType::Special; +} + +// ─── Binary Parsing ────────────────────────────────────────────── + +bool AampFile::loadFromBinary(const u8* data, size_t size) { + if (size < 0x30) return false; + if (memcmp(data, "AAMP", 4) != 0) return false; + + mVersion = *(const u32*)(data + 4); + mFlags = *(const u32*)(data + 8); + u32 fileSize = *(const u32*)(data + 12); + mPioVersion = *(const u32*)(data + 0x10); + mOffsetToPio = *(const u32*)(data + 0x14); + + // Store PIO type string (bytes between header and root list) + if (mOffsetToPio > 0 && 0x30 + mOffsetToPio <= size) { + mPioTypeString.assign(data + 0x30, data + 0x30 + mOffsetToPio); + } + + const u8* rootBase = data + 0x30 + mOffsetToPio; + parseList(mRootList, rootBase, 0); + return true; +} + +void AampFile::parseList(AampList& list, const u8* base, size_t listOffset) const { + const u8* p = base + listOffset; + list.nameHash = *(const u32*)(p); + u32 listON = *(const u32*)(p + 4); + u32 objON = *(const u32*)(p + 8); + + u32 numLists = listON >> 16; + u32 listsOff = (listON & 0xFFFF) * 4; + u32 numObjs = objON >> 16; + u32 objsOff = (objON & 0xFFFF) * 4; + + for (u32 i = 0; i < numLists; i++) { + AampList child; + parseList(child, p, listsOff + i * 12); + list.childLists.push_back(std::move(child)); + } + + for (u32 i = 0; i < numObjs; i++) { + AampObject obj; + parseObject(obj, p, objsOff + i * 8); + list.objects.push_back(std::move(obj)); + } +} + +void AampFile::parseObject(AampObject& obj, const u8* base, size_t objOffset) const { + const u8* p = base + objOffset; + obj.nameHash = *(const u32*)(p); + u32 paramON = *(const u32*)(p + 4); + u32 numParams = paramON >> 16; + u32 paramsOff = (paramON & 0xFFFF) * 4; + + for (u32 i = 0; i < numParams; i++) { + obj.parameters.push_back(parseParameter(p, paramsOff + i * 8)); + } +} + +AampParameter AampFile::parseParameter(const u8* base, size_t paramOffset) const { + const u8* p = base + paramOffset; + AampParameter param; + param.nameHash = *(const u32*)(p); + u32 offsetAndType = *(const u32*)(p + 4); + param.type = (AampParamType)(offsetAndType >> 24); + u32 dataOff = (offsetAndType & 0xFFFFFF) * 4; + + const u8* dataPtr = p + dataOff; + u8 typeId = (u8)param.type; + + switch (param.type) { + case AampParamType::Bool: + param.boolVal = *(const u32*)dataPtr != 0; + break; + case AampParamType::F32: + param.f32Val = *(const float*)dataPtr; + break; + case AampParamType::Int: + param.intVal = *(const s32*)dataPtr; + break; + case AampParamType::Vec2: + memcpy(param.vec, dataPtr, 8); + break; + case AampParamType::Vec3: + memcpy(param.vec, dataPtr, 12); + break; + case AampParamType::Vec4: + case AampParamType::Color: + case AampParamType::Quat: + memcpy(param.vec, dataPtr, 16); + break; + case AampParamType::String32: + case AampParamType::String64: + case AampParamType::String256: + case AampParamType::StringRef: { + size_t len = strlen((const char*)dataPtr); + param.stringVal.assign((const char*)dataPtr, len); + break; + } + case AampParamType::U32: + param.u32Val = *(const u32*)dataPtr; + break; + case AampParamType::Curve1: + case AampParamType::Curve2: + case AampParamType::Curve3: + case AampParamType::Curve4: { + u32 n = typeId - (u32)AampParamType::Curve1 + 1; + u32 sz = n * 0x80; + param.bufferData.assign(dataPtr, dataPtr + sz); + break; + } + case AampParamType::BufferInt: + case AampParamType::BufferF32: + case AampParamType::BufferU32: { + u32 count = *(const u32*)(dataPtr - 4); + u32 sz = count * 4; + param.bufferData.assign(dataPtr, dataPtr + sz); + break; + } + case AampParamType::BufferBinary: { + u32 count = *(const u32*)(dataPtr - 4); + param.bufferData.assign(dataPtr, dataPtr + count); + break; + } + default: + break; + } + + return param; +} + +// ─── Binary Writing ────────────────────────────────────────────── + +static u32 alignUp4(u32 v) { return (v + 3) & ~3u; } + +static bool isStringParam(AampParamType type) { + return type == AampParamType::String32 || type == AampParamType::String64 || + type == AampParamType::String256 || type == AampParamType::StringRef; +} + +void AampFile::writeToBinary(std::vector& out) const { + // Layout: Header(0x30) | PioTypeString(mOffsetToPio) | StructSection | DataSection | StringSection + // file_size = total output size + + // Phase 1: Count lists, objects, params + u32 numLists = 0, numObjects = 0, numParams = 0; + std::function countEntries; + countEntries = [&](const AampList& list) { + numLists++; + for (auto& child : list.childLists) countEntries(child); + for (auto& obj : list.objects) { + numObjects++; + numParams += (u32)obj.parameters.size(); + } + }; + countEntries(mRootList); + + // Phase 2: Build structural section - all lists first, then all objects, then all params + // Original AAMP layout: [lists in DFS][objects in DFS][params in DFS] + std::vector structBuf; + struct PendingParam { + u32 structOffset; + const AampParameter* param; + }; + std::vector pendingParams; + + // Collect lists, objects, params in DFS order, tracking indices + struct ListInfo { + const AampList* list; + u32 pos; + u32 firstChildIdx; // index in allLists of first child (or next sibling) + u32 firstObjIdx; // index in allObjs of first object in this list + }; + struct ObjInfo { + const AampObject* obj; + u32 pos; + u32 firstParamIdx; // index in allParams of first param in this object + }; + std::vector allLists; + std::vector allObjs; + + struct ParamCollect { const AampParameter* param; u32 pos; }; + std::vector allParams; + + // DFS to collect all entries in order (pre-order: list entries, then objects before children) + std::function collectDFS; + collectDFS = [&](const AampList& list) { + u32 myIdx = (u32)allLists.size(); + u32 firstObjIdx = (u32)allObjs.size(); + allLists.push_back({&list, 0, myIdx + 1, firstObjIdx}); + + // Add this list's objects BEFORE recursing into children (pre-order) + for (auto& obj : list.objects) { + u32 firstParamIdx = (u32)allParams.size(); + allObjs.push_back({&obj, 0, firstParamIdx}); + for (auto& p : obj.parameters) + allParams.push_back({&p, 0}); + } + + // Recurse into children + for (auto& child : list.childLists) collectDFS(child); + }; + collectDFS(mRootList); + + // Assign positions: lists at 0, objects after, params after objects + u32 listsSize = numLists * 12; + u32 objsSize = numObjects * 8; + u32 paramsSize = numParams * 8; + structBuf.resize(listsSize + objsSize + paramsSize, 0); + + for (u32 i = 0; i < allLists.size(); i++) + allLists[i].pos = i * 12; + for (u32 i = 0; i < allObjs.size(); i++) + allObjs[i].pos = listsSize + i * 8; + for (u32 i = 0; i < allParams.size(); i++) + allParams[i].pos = listsSize + objsSize + i * 8; + + // Fill list entries + for (u32 i = 0; i < allLists.size(); i++) { + auto& li = allLists[i]; + u32 listPos = li.pos; + u32 nChildLists = (u32)li.list->childLists.size(); + u32 nObjs = (u32)li.list->objects.size(); + + // childListsOff: first child position, or listsSize when count=0 + u32 listsOff; + if (nChildLists > 0) + listsOff = (allLists[li.firstChildIdx].pos - listPos) / 4; + else + listsOff = (listsSize - listPos) / 4; + + // objectsOff: first object position, or listsSize when count=0 + u32 objsOff; + if (nObjs > 0) + objsOff = (listsSize + li.firstObjIdx * 8 - listPos) / 4; + else + objsOff = (listsSize - listPos) / 4; + + *(u32*)(structBuf.data() + listPos + 0) = li.list->nameHash; + *(u32*)(structBuf.data() + listPos + 4) = (nChildLists << 16) | (listsOff & 0xFFFF); + *(u32*)(structBuf.data() + listPos + 8) = (nObjs << 16) | (objsOff & 0xFFFF); + } + + // Fill object entries + for (u32 i = 0; i < allObjs.size(); i++) { + u32 objPos = allObjs[i].pos; + auto& obj = *allObjs[i].obj; + u32 np = (u32)obj.parameters.size(); + + u32 paramAbsPos = listsSize + objsSize + allObjs[i].firstParamIdx * 8; + u32 paramsOff = (paramAbsPos - objPos) / 4; + + *(u32*)(structBuf.data() + objPos + 0) = obj.nameHash; + *(u32*)(structBuf.data() + objPos + 4) = (np << 16) | (paramsOff & 0xFFFF); + } + + // Store param positions for data offset computation + for (auto& pi : allParams) + pendingParams.push_back({pi.pos, pi.param}); + + // Phase 3: Build data section (non-strings) and string section (strings) separately + std::vector dataSec; + std::vector stringSec; + + // Entry dedup tracking: (data, offset) + std::vector, u32>> dataEntries; + std::vector, u32>> stringEntries; + + // Collect param data, building both sections + struct ParamDataInfo { + bool isString; + bool isBuffer; + u32 sectionOffset; // offset within dataSec or stringSec + u32 dataSize; + }; + std::vector paramInfos; + + for (auto& pp : pendingParams) { + auto& param = *pp.param; + ParamDataInfo info = {false, false, 0, 0}; + std::vector paramData; + + switch (param.type) { + case AampParamType::Bool: { + u32 v = param.boolVal ? 1 : 0; + paramData.resize(4); + memcpy(paramData.data(), &v, 4); + break; + } + case AampParamType::F32: + paramData.resize(4); + memcpy(paramData.data(), ¶m.f32Val, 4); + break; + case AampParamType::Int: + paramData.resize(4); + memcpy(paramData.data(), ¶m.intVal, 4); + break; + case AampParamType::U32: + paramData.resize(4); + memcpy(paramData.data(), ¶m.u32Val, 4); + break; + case AampParamType::Vec2: + paramData.resize(8); + memcpy(paramData.data(), param.vec, 8); + break; + case AampParamType::Vec3: + paramData.resize(12); + memcpy(paramData.data(), param.vec, 12); + break; + case AampParamType::Vec4: + case AampParamType::Color: + case AampParamType::Quat: + paramData.resize(16); + memcpy(paramData.data(), param.vec, 16); + break; + case AampParamType::String32: + case AampParamType::String64: + case AampParamType::String256: + case AampParamType::StringRef: + info.isString = true; + paramData.insert(paramData.end(), param.stringVal.begin(), param.stringVal.end()); + paramData.push_back(0); + while (paramData.size() % 4 != 0) paramData.push_back(0); + break; + case AampParamType::Curve1: + case AampParamType::Curve2: + case AampParamType::Curve3: + case AampParamType::Curve4: + paramData = param.bufferData; + break; + case AampParamType::BufferInt: + case AampParamType::BufferF32: + case AampParamType::BufferU32: { + info.isBuffer = true; + u32 count = (u32)param.bufferData.size() / 4; + paramData.resize(4); + memcpy(paramData.data(), &count, 4); + paramData.insert(paramData.end(), param.bufferData.begin(), param.bufferData.end()); + break; + } + case AampParamType::BufferBinary: { + info.isBuffer = true; + u32 count = (u32)param.bufferData.size(); + paramData.resize(4); + memcpy(paramData.data(), &count, 4); + paramData.insert(paramData.end(), param.bufferData.begin(), param.bufferData.end()); + while (paramData.size() % 4 != 0) paramData.push_back(0); + break; + } + default: break; + } + + info.dataSize = (u32)paramData.size(); + if (info.isString) { + // Deduplicate: only at entry boundaries (tracked start offsets) + bool found = false; + for (auto& [existData, existOff] : stringEntries) { + if (existData == paramData) { + info.sectionOffset = existOff; + found = true; + break; + } + } + if (!found) { + info.sectionOffset = (u32)stringSec.size(); + stringEntries.push_back({paramData, info.sectionOffset}); + stringSec.insert(stringSec.end(), paramData.begin(), paramData.end()); + } + } else { + // Deduplicate: only at entry boundaries (tracked start offsets) + bool found = false; + for (auto& [existData, existOff] : dataEntries) { + if (existData == paramData) { + info.sectionOffset = existOff; + found = true; + break; + } + } + if (!found) { + info.sectionOffset = (u32)dataSec.size(); + dataEntries.push_back({paramData, info.sectionOffset}); + dataSec.insert(dataSec.end(), paramData.begin(), paramData.end()); + } + } + paramInfos.push_back(info); + } + + // Phase 4: Compute relative offsets and write param entries + u32 pioTypeSize = mOffsetToPio; + u32 structSize = (u32)structBuf.size(); + u32 dataSecSize = (u32)dataSec.size(); + + for (size_t i = 0; i < pendingParams.size(); i++) { + auto& pp = pendingParams[i]; + auto& param = *pp.param; + auto& info = paramInfos[i]; + + // Param entry absolute position in the body (after header, relative to PIO type string start) + u32 paramAbsInBody = pioTypeSize + pp.structOffset; + + // Data absolute position in the body + u32 dataAbsInBody; + if (info.isString) { + dataAbsInBody = pioTypeSize + structSize + dataSecSize + info.sectionOffset; + } else { + dataAbsInBody = pioTypeSize + structSize + info.sectionOffset; + } + if (info.isBuffer) dataAbsInBody += 4; // skip count prefix + + u32 relOffset = (dataAbsInBody - paramAbsInBody) / 4; + *(u32*)(structBuf.data() + pp.structOffset + 0) = param.nameHash; + *(u32*)(structBuf.data() + pp.structOffset + 4) = ((u32)param.type << 24) | (relOffset & 0xFFFFFF); + } + + // Phase 5: Assemble final output + u32 bodySize = pioTypeSize + structSize + dataSecSize + (u32)stringSec.size(); + u32 totalSize = 0x30 + bodySize; + out.resize(totalSize, 0); + + // Write header + memcpy(out.data(), "AAMP", 4); + *(u32*)(out.data() + 4) = mVersion; + *(u32*)(out.data() + 8) = mFlags; + *(u32*)(out.data() + 12) = totalSize; + *(u32*)(out.data() + 0x10) = mPioVersion; + *(u32*)(out.data() + 0x14) = pioTypeSize; + *(u32*)(out.data() + 0x18) = numLists; + *(u32*)(out.data() + 0x1C) = numObjects; + *(u32*)(out.data() + 0x20) = numParams; + *(u32*)(out.data() + 0x24) = dataSecSize; + *(u32*)(out.data() + 0x28) = (u32)stringSec.size(); + *(u32*)(out.data() + 0x2C) = 0; + + // Write body + u32 pos = 0x30; + if (!mPioTypeString.empty()) { + memcpy(out.data() + pos, mPioTypeString.data(), mPioTypeString.size()); + } + pos += pioTypeSize; + memcpy(out.data() + pos, structBuf.data(), structBuf.size()); + pos += structSize; + if (!dataSec.empty()) + memcpy(out.data() + pos, dataSec.data(), dataSec.size()); + pos += dataSecSize; + if (!stringSec.empty()) + memcpy(out.data() + pos, stringSec.data(), stringSec.size()); +} + +// ─── YAML Output ───────────────────────────────────────────────── + +YAML::Node AampFile::toYaml() const { + YAML::Node root; + root["version"] = mVersion; + root["flags"] = mFlags; + root["pio_version"] = mPioVersion; + // PIO type string (e.g. "phcl") + if (!mPioTypeString.empty()) { + size_t len = 0; + while (len < mPioTypeString.size() && mPioTypeString[len] != 0) len++; + root["pio_type"] = std::string((const char*)mPioTypeString.data(), len); + } + root["root"] = listToYaml(mRootList); + return root; +} + +YAML::Node AampFile::listToYaml(const AampList& list) const { + YAML::Node node; + node["hash"] = hashToName(list.nameHash); + + if (!list.childLists.empty()) { + YAML::Node listsNode; + for (auto& child : list.childLists) + listsNode.push_back(listToYaml(child)); + node["lists"] = listsNode; + } + + if (!list.objects.empty()) { + YAML::Node objsNode; + for (auto& obj : list.objects) + objsNode.push_back(objectToYaml(obj)); + node["objects"] = objsNode; + } + + return node; +} + +YAML::Node AampFile::objectToYaml(const AampObject& obj) const { + YAML::Node node; + node["hash"] = hashToName(obj.nameHash); + + YAML::Node params; + for (auto& param : obj.parameters) + params.push_back(parameterToYaml(param)); + node["params"] = params; + + return node; +} + +YAML::Node AampFile::parameterToYaml(const AampParameter& param) const { + YAML::Node node; + node["hash"] = hashToName(param.nameHash); + node["type"] = paramTypeName(param.type); + + switch (param.type) { + case AampParamType::Bool: + node["value"] = param.boolVal; + break; + case AampParamType::F32: + node["value"] = param.f32Val; + break; + case AampParamType::Int: + node["value"] = param.intVal; + break; + case AampParamType::U32: + node["value"] = param.u32Val; + break; + case AampParamType::Vec2: { + YAML::Node v; + v.push_back(param.vec[0]); v.push_back(param.vec[1]); + node["value"] = v; + break; + } + case AampParamType::Vec3: { + YAML::Node v; + v.push_back(param.vec[0]); v.push_back(param.vec[1]); v.push_back(param.vec[2]); + node["value"] = v; + break; + } + case AampParamType::Vec4: + case AampParamType::Color: + case AampParamType::Quat: { + YAML::Node v; + for (int i = 0; i < 4; i++) v.push_back(param.vec[i]); + node["value"] = v; + break; + } + case AampParamType::String32: + case AampParamType::String64: + case AampParamType::String256: + case AampParamType::StringRef: + node["value"] = param.stringVal; + break; + case AampParamType::Curve1: + case AampParamType::Curve2: + case AampParamType::Curve3: + case AampParamType::Curve4: { + // Curves: array of floats from the buffer + YAML::Node v; + const float* fp = (const float*)param.bufferData.data(); + size_t count = param.bufferData.size() / 4; + for (size_t i = 0; i < count; i++) v.push_back(fp[i]); + node["value"] = v; + break; + } + case AampParamType::BufferInt: { + YAML::Node v; + const s32* ip = (const s32*)param.bufferData.data(); + size_t count = param.bufferData.size() / 4; + for (size_t i = 0; i < count; i++) v.push_back(ip[i]); + node["value"] = v; + break; + } + case AampParamType::BufferF32: { + YAML::Node v; + const float* fp = (const float*)param.bufferData.data(); + size_t count = param.bufferData.size() / 4; + for (size_t i = 0; i < count; i++) v.push_back(fp[i]); + node["value"] = v; + break; + } + case AampParamType::BufferU32: { + YAML::Node v; + const u32* up = (const u32*)param.bufferData.data(); + size_t count = param.bufferData.size() / 4; + for (size_t i = 0; i < count; i++) v.push_back(up[i]); + node["value"] = v; + break; + } + case AampParamType::BufferBinary: { + YAML::Node v; + for (size_t i = 0; i < param.bufferData.size(); i++) + v.push_back((int)param.bufferData[i]); + node["value"] = v; + break; + } + default: + break; + } + + return node; +} + +// ─── YAML Input ────────────────────────────────────────────────── + +bool AampFile::fromYaml(const YAML::Node& node) { + if (!node["version"] || !node["root"]) return false; + mVersion = node["version"].as(); + mFlags = node["flags"].as(3); + mPioVersion = node["pio_version"].as(0); + + // Restore PIO type string + if (node["pio_type"]) { + std::string pioType = node["pio_type"].as(); + mPioTypeString.resize(8, 0); + memcpy(mPioTypeString.data(), pioType.data(), std::min(pioType.size(), (size_t)8)); + mOffsetToPio = 8; + } + + mRootList = listFromYaml(node["root"]); + return true; +} + +AampList AampFile::listFromYaml(const YAML::Node& node) const { + AampList list; + list.nameHash = nameToHash(node["hash"].as()); + + if (node["lists"]) { + for (const auto& child : node["lists"]) + list.childLists.push_back(listFromYaml(child)); + } + if (node["objects"]) { + for (const auto& obj : node["objects"]) + list.objects.push_back(objectFromYaml(obj)); + } + return list; +} + +AampObject AampFile::objectFromYaml(const YAML::Node& node) const { + AampObject obj; + obj.nameHash = nameToHash(node["hash"].as()); + + if (node["params"]) { + for (const auto& param : node["params"]) + obj.parameters.push_back(parameterFromYaml(param)); + } + return obj; +} + +AampParameter AampFile::parameterFromYaml(const YAML::Node& node) const { + AampParameter param; + param.nameHash = nameToHash(node["hash"].as()); + param.type = paramTypeFromName(node["type"].as()); + + const YAML::Node& val = node["value"]; + + switch (param.type) { + case AampParamType::Bool: + param.boolVal = val.as(); + break; + case AampParamType::F32: + param.f32Val = val.as(); + break; + case AampParamType::Int: + param.intVal = val.as(); + break; + case AampParamType::U32: + param.u32Val = val.as(); + break; + case AampParamType::Vec2: + param.vec[0] = val[0].as(); + param.vec[1] = val[1].as(); + break; + case AampParamType::Vec3: + param.vec[0] = val[0].as(); + param.vec[1] = val[1].as(); + param.vec[2] = val[2].as(); + break; + case AampParamType::Vec4: + case AampParamType::Color: + case AampParamType::Quat: + for (int i = 0; i < 4; i++) param.vec[i] = val[i].as(); + break; + case AampParamType::String32: + case AampParamType::String64: + case AampParamType::String256: + case AampParamType::StringRef: + param.stringVal = val.as(); + break; + case AampParamType::Curve1: + case AampParamType::Curve2: + case AampParamType::Curve3: + case AampParamType::Curve4: { + size_t count = val.size(); + param.bufferData.resize(count * 4); + float* fp = (float*)param.bufferData.data(); + for (size_t i = 0; i < count; i++) fp[i] = val[i].as(); + break; + } + case AampParamType::BufferInt: { + size_t count = val.size(); + param.bufferData.resize(count * 4); + s32* ip = (s32*)param.bufferData.data(); + for (size_t i = 0; i < count; i++) ip[i] = val[i].as(); + break; + } + case AampParamType::BufferF32: { + size_t count = val.size(); + param.bufferData.resize(count * 4); + float* fp = (float*)param.bufferData.data(); + for (size_t i = 0; i < count; i++) fp[i] = val[i].as(); + break; + } + case AampParamType::BufferU32: { + size_t count = val.size(); + param.bufferData.resize(count * 4); + u32* up = (u32*)param.bufferData.data(); + for (size_t i = 0; i < count; i++) up[i] = val[i].as(); + break; + } + case AampParamType::BufferBinary: { + size_t count = val.size(); + param.bufferData.resize(count); + for (size_t i = 0; i < count; i++) param.bufferData[i] = (u8)val[i].as(); + break; + } + default: + break; + } + + return param; +} + +} // namespace FiveX diff --git a/source/fivex/AampFile.hpp b/source/fivex/AampFile.hpp new file mode 100644 index 0000000..624fbb9 --- /dev/null +++ b/source/fivex/AampFile.hpp @@ -0,0 +1,101 @@ +#pragma once + +#include "types.h" +#include +#include +#include +#include + +namespace FiveX { + +// AAMP parameter types (from agl::utl::ParameterBase::ParameterType) +enum class AampParamType : u8 { + Bool = 0, + F32 = 1, + Int = 2, + Vec2 = 3, + Vec3 = 4, + Vec4 = 5, + Color = 6, + String32 = 7, + String64 = 8, + Curve1 = 9, + Curve2 = 10, + Curve3 = 11, + Curve4 = 12, + BufferInt = 13, + BufferF32 = 14, + String256 = 15, + Quat = 16, + U32 = 17, + BufferU32 = 18, + BufferBinary = 19, + StringRef = 20, + Special = 21, +}; + +struct AampParameter { + u32 nameHash; + AampParamType type; + // Typed value storage + bool boolVal = false; + float f32Val = 0; + s32 intVal = 0; + u32 u32Val = 0; + float vec[4] = {}; + std::string stringVal; + std::vector bufferData; + // Curve data: N * 0x80 bytes stored in bufferData +}; + +struct AampObject { + u32 nameHash; + std::vector parameters; +}; + +struct AampList { + u32 nameHash; + std::vector childLists; + std::vector objects; +}; + +class AampFile { +public: + bool loadFromBinary(const u8* data, size_t size); + void writeToBinary(std::vector& out) const; + + YAML::Node toYaml() const; + bool fromYaml(const YAML::Node& node); + + // Load a hash dictionary from file (one "name" per line) + static void loadHashDictionary(const std::string& path); + + u32 mVersion = 2; + u32 mFlags = 3; // LE + UTF8 + u32 mPioVersion = 0; + u32 mOffsetToPio = 8; + std::vector mPioTypeString; // e.g. "phcl\0\0\0\0" + AampList mRootList; + +private: + void parseList(AampList& list, const u8* base, size_t listOffset) const; + void parseObject(AampObject& obj, const u8* base, size_t objOffset) const; + AampParameter parseParameter(const u8* base, size_t paramOffset) const; + + YAML::Node listToYaml(const AampList& list) const; + YAML::Node objectToYaml(const AampObject& obj) const; + YAML::Node parameterToYaml(const AampParameter& param) const; + + AampList listFromYaml(const YAML::Node& node) const; + AampObject objectFromYaml(const YAML::Node& node) const; + AampParameter parameterFromYaml(const YAML::Node& node) const; + + static std::string hashToName(u32 hash); + static u32 nameToHash(const std::string& name); + static const char* paramTypeName(AampParamType type); + static AampParamType paramTypeFromName(const std::string& name); + + static std::map& getHashDictionary(); +}; + +} // namespace FiveX diff --git a/source/fivex/ClothInfo.cpp b/source/fivex/ClothInfo.cpp new file mode 100644 index 0000000..303965e --- /dev/null +++ b/source/fivex/ClothInfo.cpp @@ -0,0 +1,158 @@ +#include "fivex/ClothInfo.hpp" +#include +#include + +static constexpr u32 byteswap32(u32 x) { + return ((x << 24) & 0xFF000000) | ((x << 8) & 0x00FF0000) | + ((x >> 8) & 0x0000FF00) | ((x >> 24) & 0x000000FF); +} + +namespace FiveX { + +ClothInfo::ClothInfo() {} +ClothInfo::~ClothInfo() {} + +void ClothInfo::loadFromBphcl(std::vector& in) { + Phive::PhiveHeader& header = *reinterpret_cast(in.data()); + + // Section 0: TAG file + u32 tagOffset = header.HktOffset; // Section0 offset (always 0x30) + u32 tagSize = header.FileSize; // Section0 size (TAG + alignment padding) + + // Section 1: AAMP data + u32 aampOffset = header.TableOffset0; // Section1 offset + u32 aampSize = header.HktSize; // Section1 size + + // Section 2 + u32 section2Offset = header.TableOffset1; + u32 section2Size = header.TableSize0; + + // Parse TAG file (pass actual Section0 size, parser reads TAG0 internal size) + mTagFile.loadFromBinary(in.data() + tagOffset, tagSize); + + // Load AAMP section if present + if (aampSize > 0 && aampOffset + aampSize <= in.size()) { + mAampSection.assign(in.data() + aampOffset, in.data() + aampOffset + aampSize); + mHasAamp = true; + } else { + mAampSection.clear(); + mHasAamp = false; + } + + // Load section 2 data if present + if (section2Size > 0 && section2Offset > 0 && section2Offset + section2Size <= in.size()) { + mSection2Data.assign(in.data() + section2Offset, in.data() + section2Offset + section2Size); + mHasSection2 = true; + } else { + mSection2Data.clear(); + mHasSection2 = false; + } +} + +void ClothInfo::serializeToBphcl(std::vector& out) { + // Build TAG file binary + std::vector tagData; + mTagFile.writeToBinary(tagData); + + // Calculate layout - all sizes computed from data + u32 phiveHeaderSize = 0x30; + u32 aampSize = (u32)mAampSection.size(); + + // Section0 size = TAG data aligned to 16 bytes + u32 section0Size = ((u32)tagData.size() + 15) & ~15u; + + // Section1 (AAMP) size = AAMP data aligned to 16 bytes + u32 section1Size = (aampSize > 0) ? ((aampSize + 15) & ~15u) : 0; + + u32 aampOffset = phiveHeaderSize + section0Size; + u32 section2Off = aampOffset + section1Size; + u32 section2Size = (u32)mSection2Data.size(); + + u32 totalSize = section2Off + section2Size; + + out.resize(totalSize, 0); + + // Write Phive header + Phive::PhiveHeader header; + memset(&header, 0, sizeof(header)); + memcpy(header.Magic, "Phive", 6); + header.Reserve1 = 1; + header.BOM = 0xFEFF; + header.MajorVersion = 3; // bphcl version + header.MinorVersion = 3; + header.HktOffset = phiveHeaderSize; // Section0 offset + header.TableOffset0 = aampOffset; // Section1 offset + header.TableOffset1 = section2Off; // Section2 offset + header.FileSize = section0Size; // Section0 size + header.HktSize = section1Size; // Section1 size + header.TableSize0 = section2Size; // Section2 size + + memcpy(out.data(), &header, sizeof(header)); + + // Write TAG file + memcpy(out.data() + phiveHeaderSize, tagData.data(), tagData.size()); + + // Write AAMP + if (mHasAamp && !mAampSection.empty()) { + memcpy(out.data() + aampOffset, mAampSection.data(), mAampSection.size()); + } + + // Write section2 data + if (mHasSection2 && !mSection2Data.empty()) { + memcpy(out.data() + section2Off, mSection2Data.data(), mSection2Data.size()); + } +} + +void ClothInfo::serializeToYaml(std::vector& outTagYaml, std::vector& outAampYaml) { + // Main TAG YAML with decoded items + YAML::Node root = mTagFile.toYaml(); + root["format"] = "bphcl"; + + YAML::Emitter tagEmit; + tagEmit << root; + std::string tagStr = tagEmit.c_str(); + outTagYaml.assign(tagStr.begin(), tagStr.end()); + + // AAMP YAML - human-readable decoded parameters + outAampYaml.clear(); + if (mHasAamp && !mAampSection.empty()) { + AampFile aamp; + if (aamp.loadFromBinary(mAampSection.data(), mAampSection.size())) { + YAML::Node aampRoot = aamp.toYaml(); + YAML::Emitter aampEmit; + aampEmit << aampRoot; + std::string aampStr = aampEmit.c_str(); + outAampYaml.assign(aampStr.begin(), aampStr.end()); + } + } +} + +void ClothInfo::loadFromYaml(const std::vector& tagYaml, const std::vector& aampYaml) { + std::string tagStr(tagYaml.begin(), tagYaml.end()); + YAML::Node root = YAML::Load(tagStr); + + mTagFile.fromYaml(root); + + // Rebuild AAMP from decoded YAML + if (!aampYaml.empty()) { + std::string aampStr(aampYaml.begin(), aampYaml.end()); + YAML::Node aampRoot = YAML::Load(aampStr); + AampFile aamp; + if (aamp.fromYaml(aampRoot)) { + aamp.writeToBinary(mAampSection); + mHasAamp = true; + } else { + mAampSection.clear(); + mHasAamp = false; + } + } else { + mAampSection.clear(); + mHasAamp = false; + } + + // Section2 is empty in all bphcl files + mSection2Data.clear(); + mHasSection2 = false; +} + +} diff --git a/source/fivex/ClothInfo.hpp b/source/fivex/ClothInfo.hpp new file mode 100644 index 0000000..01167b5 --- /dev/null +++ b/source/fivex/ClothInfo.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "types.h" +#include "havok/hkTagfile.hpp" +#include "fivex/AampFile.hpp" +#include "phive/phive.h" +#include +#include + +namespace FiveX { + +class ClothInfo { +public: + ClothInfo(); + ~ClothInfo(); + + void loadFromBphcl(std::vector& in); + void serializeToBphcl(std::vector& out); + + // YAML serialization (human-readable output) + void serializeToYaml(std::vector& outTagYaml, std::vector& outAampYaml); + void loadFromYaml(const std::vector& tagYaml, const std::vector& aampYaml); + +private: + Havok::hkTagfile mTagFile; + std::vector mAampSection; // Section1 data (AAMP) + std::vector mSection2Data; // Section2 data (if any) + bool mHasAamp = false; + bool mHasSection2 = false; +}; + +} diff --git a/source/fivex/MeshShapeBuilder.cpp b/source/fivex/MeshShapeBuilder.cpp index cc10db2..1cf9520 100644 --- a/source/fivex/MeshShapeBuilder.cpp +++ b/source/fivex/MeshShapeBuilder.cpp @@ -342,9 +342,13 @@ static void sortLsb2Hsb(RadixEntry* data, hkUint32 numObjects, RadixEntry* buffe static int countLeadingZeros32(hkUint32 v) { if (v == 0) return 32; +#ifdef _MSC_VER unsigned long idx; _BitScanReverse(&idx, v); return 31 - (int)idx; +#else + return __builtin_clz(v); +#endif } static void sortRadix(RadixEntry* data, int n, RadixEntry* buffer) { diff --git a/source/fivex/NavMeshInfo.cpp b/source/fivex/NavMeshInfo.cpp new file mode 100644 index 0000000..ca6e0ad --- /dev/null +++ b/source/fivex/NavMeshInfo.cpp @@ -0,0 +1,345 @@ +#include "fivex/NavMeshInfo.hpp" +#include +#include +#include +#include + +static constexpr u32 byteswap32(u32 x) { + return ((x << 24) & 0xFF000000) | ((x << 8) & 0x00FF0000) | + ((x >> 8) & 0x0000FF00) | ((x >> 24) & 0x000000FF); +} + +namespace FiveX { + +NavMeshInfo::NavMeshInfo() {} +NavMeshInfo::~NavMeshInfo() {} + +void NavMeshInfo::loadFromBphnm(std::vector& in) { + Phive::PhiveHeader& header = *reinterpret_cast(in.data()); + + // Section 0: TAG file + u32 tagOffset = header.HktOffset; // Section0 offset (always 0x30) + u32 tagSize = header.FileSize; // Section0 size = TAG file size + + // Section 1: ReferenceRotation data (52 bytes: u32 sectionUid + 12 floats) + u32 extraOffset = header.TableOffset0; // Section1 offset + u32 extraSize = header.HktSize; // Section1 size + + // Section 2 + u32 section2Offset = header.TableOffset1; + u32 section2Size = header.TableSize0; + + // Parse TAG file + mTagFile.loadFromBinary(in.data() + tagOffset, tagSize); + + // Load extra section (ReferenceRotation) if present + if (extraSize > 0 && extraOffset + extraSize <= in.size()) { + mExtraSection.assign(in.data() + extraOffset, in.data() + extraOffset + extraSize); + mHasExtra = true; + } else { + mExtraSection.clear(); + mHasExtra = false; + } + + // Load section 2 data if present + // Track whether section2 concept exists (s2off > 0) separately from having data + mHasSection2 = (section2Offset > 0); + if (section2Size > 0 && section2Offset > 0 && section2Offset + section2Size <= in.size()) { + mSection2Data.assign(in.data() + section2Offset, in.data() + section2Offset + section2Size); + } else { + mSection2Data.clear(); + } +} + +void NavMeshInfo::serializeToBphnm(std::vector& out) { + // Build TAG file binary + std::vector tagData; + mTagFile.writeToBinary(tagData); + + // Calculate layout - all sizes computed from data + u32 phiveHeaderSize = 0x30; + u32 tagSizeAligned = (u32)ALIGN_UP(tagData.size(), 8); + u32 extraSize = (u32)mExtraSection.size(); + u32 extraOffset = phiveHeaderSize + tagSizeAligned; + u32 section2Size = (u32)mSection2Data.size(); + u32 section2Off = mHasSection2 ? (u32)ALIGN_UP(extraOffset + extraSize, 8) : 0; + + // File size only extends to s2off when section2 has actual data + u32 totalSize = (section2Size > 0) ? (section2Off + section2Size) : (extraOffset + extraSize); + + out.resize(totalSize, 0); + + // Write Phive header + Phive::PhiveHeader header; + memset(&header, 0, sizeof(header)); + memcpy(header.Magic, "Phive", 6); + header.Reserve1 = 1; + header.BOM = 0xFEFF; + header.MajorVersion = 1; // bphnm version + header.MinorVersion = 3; + header.HktOffset = phiveHeaderSize; // Section0 offset + header.TableOffset0 = extraOffset; // Section1 offset + header.TableOffset1 = section2Off; // Section2 offset + header.FileSize = (u32)tagData.size(); // Section0 size = TAG size (not aligned) + header.HktSize = extraSize; // Section1 size + header.TableSize0 = section2Size; // Section2 size + + memcpy(out.data(), &header, sizeof(header)); + + // Write TAG file + memcpy(out.data() + phiveHeaderSize, tagData.data(), tagData.size()); + + // Write extra section (ReferenceRotation) + if (mHasExtra && !mExtraSection.empty()) { + memcpy(out.data() + extraOffset, mExtraSection.data(), mExtraSection.size()); + } + + // Write section2 data + if (mHasSection2 && !mSection2Data.empty()) { + memcpy(out.data() + section2Off, mSection2Data.data(), mSection2Data.size()); + } +} + +void NavMeshInfo::serializeToYaml(std::vector& outYaml) { + YAML::Node root = mTagFile.toYaml(); + root["format"] = "bphnm"; + + // Decode ReferenceRotation (Section1) for human readability + if (mHasExtra && mExtraSection.size() >= 52) { + YAML::Node refRot(YAML::NodeType::Map); + u32 sectionUid; + memcpy(§ionUid, mExtraSection.data(), 4); + refRot["sectionUid"] = sectionUid; + + YAML::Node rotation(YAML::NodeType::Sequence); + for (int i = 0; i < 12; i++) { + float f; + memcpy(&f, mExtraSection.data() + 4 + i * 4, 4); + rotation.push_back(f); + } + refRot["rotation"] = rotation; + root["referenceRotation"] = refRot; + } + + // Decode Section2 as array of u32 entries (NavTable hashes) + // Include empty navTable when section2 concept exists but has no data + if (mHasSection2) { + YAML::Node navTable(YAML::NodeType::Sequence); + for (size_t i = 0; i + 3 < mSection2Data.size(); i += 4) { + u32 val; + memcpy(&val, mSection2Data.data() + i, 4); + char hexBuf[16]; + snprintf(hexBuf, sizeof(hexBuf), "0x%08X", val); + navTable.push_back(std::string(hexBuf)); + } + root["navTable"] = navTable; + } + + YAML::Emitter emit; + emit << root; + std::string yamlStr = emit.c_str(); + outYaml.assign(yamlStr.begin(), yamlStr.end()); +} + +void NavMeshInfo::loadFromYaml(const std::vector& yamlData) { + std::string yamlStr(yamlData.begin(), yamlData.end()); + YAML::Node root = YAML::Load(yamlStr); + + mTagFile.fromYaml(root); + + // Rebuild ReferenceRotation from decoded values + if (root["referenceRotation"]) { + const YAML::Node& refRot = root["referenceRotation"]; + mExtraSection.resize(52, 0); + u32 uid = refRot["sectionUid"].as(); + memcpy(mExtraSection.data(), &uid, 4); + for (int i = 0; i < 12; i++) { + float f = refRot["rotation"][i].as(); + memcpy(mExtraSection.data() + 4 + i * 4, &f, 4); + } + mHasExtra = true; + } else { + mExtraSection.clear(); + mHasExtra = false; + } + + // Rebuild Section2 from navTable entries + // navTable presence indicates section2 concept exists (even if empty) + if (root["navTable"]) { + mHasSection2 = true; + const YAML::Node& navTable = root["navTable"]; + mSection2Data.resize(navTable.size() * 4); + for (size_t i = 0; i < navTable.size(); i++) { + std::string valStr = navTable[i].as(); + u32 val = (u32)strtoul(valStr.c_str(), nullptr, 0); + memcpy(mSection2Data.data() + i * 4, &val, 4); + } + } else { + mHasSection2 = false; + mSection2Data.clear(); + } +} + +bool NavMeshInfo::serializeToObj(std::vector& outObj) { + const auto& items = mTagFile.mItems; + const auto& types = mTagFile.mTypes; + const auto& data = mTagFile.mDataSection; + + // Find the hkaiNavMesh item to get its field layout + int navMeshItemIdx = -1; + int faceItemIdx = -1; + int edgeItemIdx = -1; + int vertexItemIdx = -1; + + for (size_t i = 0; i < items.size(); i++) { + if (items[i].typeIndex == 0) continue; + u32 typeId = items[i].typeIndex; + if (typeId >= types.size()) continue; + const std::string& name = types[typeId].name; + if (name == "hkaiNavMesh" && navMeshItemIdx < 0) { + navMeshItemIdx = (int)i; + } else if (name == "hkaiNavMesh::Face" && faceItemIdx < 0) { + faceItemIdx = (int)i; + } else if (name == "hkaiNavMesh::Edge" && edgeItemIdx < 0) { + edgeItemIdx = (int)i; + } else if (name == "hkVector4" && vertexItemIdx < 0) { + // First hkVector4 array after faces/edges is the vertex array + if (faceItemIdx >= 0 && edgeItemIdx >= 0) + vertexItemIdx = (int)i; + } + } + + if (faceItemIdx < 0 || edgeItemIdx < 0 || vertexItemIdx < 0) + return false; + + // Parse Face type to find field offsets + const auto& faceType = types[items[faceItemIdx].typeIndex]; + u32 faceSize = faceType.size; + u32 faceStartEdgeOff = 0, faceNumEdgesOff = 0; + for (const auto& f : faceType.fields) { + if (f.name == "startEdgeIndex") faceStartEdgeOff = f.offset; + else if (f.name == "numEdges") faceNumEdgesOff = f.offset; + } + + // Parse Edge type to find field offsets + const auto& edgeType = types[items[edgeItemIdx].typeIndex]; + u32 edgeSize = edgeType.size; + u32 edgeAOff = 0; + for (const auto& f : edgeType.fields) { + if (f.name == "a") edgeAOff = f.offset; + } + + u32 faceCount = items[faceItemIdx].count; + u32 faceBase = items[faceItemIdx].offset; + u32 edgeCount = items[edgeItemIdx].count; + u32 edgeBase = items[edgeItemIdx].offset; + u32 vertexCount = items[vertexItemIdx].count; + u32 vertexBase = items[vertexItemIdx].offset; + + // Build OBJ content + std::string obj; + obj += "# NavMesh exported from bphnm by PhiveConverter\n"; + obj += "# Vertices: " + std::to_string(vertexCount) + "\n"; + obj += "# Faces: " + std::to_string(faceCount) + "\n\n"; + + // Write vertices (hkVector4 = 16 bytes per vertex: x, y, z, w as f32 LE) + for (u32 i = 0; i < vertexCount; i++) { + u32 off = vertexBase + i * 16; + if (off + 16 > data.size()) break; + float x, y, z; + memcpy(&x, &data[off + 0], 4); + memcpy(&y, &data[off + 4], 4); + memcpy(&z, &data[off + 8], 4); + char buf[128]; + snprintf(buf, sizeof(buf), "v %.9g %.9g %.9g\n", x, y, z); + obj += buf; + } + + obj += "\n"; + + // Write faces (each face references edges by startEdgeIndex + numEdges) + for (u32 fi = 0; fi < faceCount; fi++) { + u32 faceOff = faceBase + fi * faceSize; + if (faceOff + faceSize > data.size()) break; + + u32 startEdge; + memcpy(&startEdge, &data[faceOff + faceStartEdgeOff], 4); + + u16 numEdges; + memcpy(&numEdges, &data[faceOff + faceNumEdgesOff], 2); + + obj += "f"; + for (u16 ei = 0; ei < numEdges; ei++) { + u32 edgeIdx = startEdge + ei; + if (edgeIdx >= edgeCount) break; + u32 edgeOff = edgeBase + edgeIdx * edgeSize; + if (edgeOff + edgeSize > data.size()) break; + u32 vertA; + memcpy(&vertA, &data[edgeOff + edgeAOff], 4); + obj += " " + std::to_string(vertA + 1); // OBJ is 1-indexed + } + obj += "\n"; + } + + outObj.assign(obj.begin(), obj.end()); + return true; +} + +bool NavMeshInfo::loadFromObj(const std::vector& objData) { + const auto& items = mTagFile.mItems; + const auto& types = mTagFile.mTypes; + auto& data = mTagFile.mDataSection; + + // Find the vertex item (hkVector4 array after Face and Edge items) + int faceItemIdx = -1, edgeItemIdx = -1, vertexItemIdx = -1; + for (size_t i = 0; i < items.size(); i++) { + if (items[i].typeIndex == 0 || items[i].typeIndex >= types.size()) continue; + const std::string& name = types[items[i].typeIndex].name; + if (name == "hkaiNavMesh::Face" && faceItemIdx < 0) faceItemIdx = (int)i; + else if (name == "hkaiNavMesh::Edge" && edgeItemIdx < 0) edgeItemIdx = (int)i; + else if (name == "hkVector4" && vertexItemIdx < 0) { + if (faceItemIdx >= 0 && edgeItemIdx >= 0) vertexItemIdx = (int)i; + } + } + if (vertexItemIdx < 0) return false; + + u32 vertexCount = items[vertexItemIdx].count; + u32 vertexBase = items[vertexItemIdx].offset; + + // Parse OBJ vertices + std::vector> objVerts; + std::string objStr(objData.begin(), objData.end()); + std::istringstream iss(objStr); + std::string line; + while (std::getline(iss, line)) { + if (line.size() < 2 || line[0] != 'v' || line[1] != ' ') continue; + float x, y, z; + if (sscanf(line.c_str(), "v %f %f %f", &x, &y, &z) == 3) { + objVerts.push_back({x, y, z}); + } + } + + if (objVerts.size() != vertexCount) { + std::cerr << "OBJ vertex count (" << objVerts.size() + << ") does not match navmesh vertex count (" << vertexCount << ")\n"; + return false; + } + + // Patch vertex positions into the TAG data section + for (u32 i = 0; i < vertexCount; i++) { + u32 off = vertexBase + i * 16; + if (off + 16 > data.size()) break; + memcpy(&data[off + 0], &objVerts[i][0], 4); + memcpy(&data[off + 4], &objVerts[i][1], 4); + memcpy(&data[off + 8], &objVerts[i][2], 4); + // w component preserved from original data + } + + // Also update the stored base64 data_section in metadata if present + // (Not needed for bphnm serialization which reads mDataSection directly, + // but ensures consistency if the data is later exported to YAML again) + + return true; +} + +} diff --git a/source/fivex/NavMeshInfo.hpp b/source/fivex/NavMeshInfo.hpp new file mode 100644 index 0000000..fe0d7a7 --- /dev/null +++ b/source/fivex/NavMeshInfo.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "types.h" +#include "havok/hkTagfile.hpp" +#include "phive/phive.h" +#include +#include + +namespace FiveX { + +class NavMeshInfo { +public: + NavMeshInfo(); + ~NavMeshInfo(); + + void loadFromBphnm(std::vector& in); + void serializeToBphnm(std::vector& out); + + // YAML serialization + void serializeToYaml(std::vector& outYaml); + void loadFromYaml(const std::vector& yamlData); + + // OBJ export (navmesh visualization) + bool serializeToObj(std::vector& outObj); + + // OBJ import (update vertex positions from edited OBJ) + bool loadFromObj(const std::vector& objData); + +private: + Havok::hkTagfile mTagFile; + std::vector mExtraSection; // Section1 data (ReferenceRotation) + std::vector mSection2Data; // Section2 data (NavTable hashes) + bool mHasExtra = false; + bool mHasSection2 = false; +}; + +} diff --git a/source/fivex/NvtInfo.cpp b/source/fivex/NvtInfo.cpp new file mode 100644 index 0000000..498eb77 --- /dev/null +++ b/source/fivex/NvtInfo.cpp @@ -0,0 +1,283 @@ +#include "fivex/NvtInfo.hpp" +#include +#include +#include +#include +#include +#include + +namespace FiveX { + +NvtWaypoint NvtWaypoint::encode(float x, float y, float z) { + NvtWaypoint wp; + wp.xSign = std::signbit(x) ? 1 : 0; + wp.ySign = std::signbit(y) ? 1 : 0; + wp.zSign = std::signbit(z) ? 1 : 0; + wp.xVal = (u16)std::round(std::fabs(x) / 256.0f * 65535.0f); + wp.yVal = (u16)std::round(std::fabs(y) / 256.0f * 65535.0f); + wp.zVal = (u16)std::round(std::fabs(z) / 256.0f * 65535.0f); + return wp; +} + +NvtInfo::NvtInfo() {} +NvtInfo::~NvtInfo() {} + +void NvtInfo::loadFromNvt(std::vector& in) { + if (in.size() < 8) return; + + memcpy(&mMagic, in.data(), 4); + memcpy(&mCount, in.data() + 4, 4); + + u32 count = mCount; + size_t offset = 8; + + // Face indices: count × u16 + mFaceIndices.resize(count); + for (u32 i = 0; i < count; i++) { + memcpy(&mFaceIndices[i], in.data() + offset, 2); + offset += 2; + } + + // Cost matrix: count × count × f32 (stride = count, no padding before matrix) + u32 matrixEntries = count * count; + mCostMatrix.resize(matrixEntries); + for (u32 i = 0; i < matrixEntries; i++) { + memcpy(&mCostMatrix[i], in.data() + offset, 4); + offset += 4; + } + + // Hop table: count × count × u8 + mHopTable.resize(matrixEntries); + memcpy(mHopTable.data(), in.data() + offset, matrixEntries); + offset += matrixEntries; + + // Path data: 9 bytes per waypoint + size_t remaining = in.size() - offset; + size_t waypointCount = remaining / 9; + mWaypoints.resize(waypointCount); + for (size_t i = 0; i < waypointCount; i++) { + const u8* p = in.data() + offset + i * 9; + memcpy(&mWaypoints[i].xVal, p + 0, 2); + mWaypoints[i].xSign = p[2]; + memcpy(&mWaypoints[i].yVal, p + 3, 2); + mWaypoints[i].ySign = p[5]; + memcpy(&mWaypoints[i].zVal, p + 6, 2); + mWaypoints[i].zSign = p[8]; + } +} + +void NvtInfo::serializeToYaml(std::vector& outYaml) { + YAML::Node root; + root["format"] = "nvt"; + root["magic"] = mMagic; + root["count"] = mCount; + + // Face indices + YAML::Node indices(YAML::NodeType::Sequence); + for (u32 i = 0; i < mCount; i++) { + indices.push_back((int)mFaceIndices[i]); + } + root["face_indices"] = indices; + + // Cost matrix as nested sequences [row][col] + YAML::Node matrix(YAML::NodeType::Sequence); + for (u32 i = 0; i < mCount; i++) { + YAML::Node row(YAML::NodeType::Sequence); + for (u32 j = 0; j < mCount; j++) { + row.push_back(mCostMatrix[i * mCount + j]); + } + matrix.push_back(row); + } + root["cost_matrix"] = matrix; + + // Hop table as nested sequences [row][col] + YAML::Node hops(YAML::NodeType::Sequence); + for (u32 i = 0; i < mCount; i++) { + YAML::Node row(YAML::NodeType::Sequence); + for (u32 j = 0; j < mCount; j++) { + row.push_back((int)mHopTable[i * mCount + j]); + } + hops.push_back(row); + } + root["hop_table"] = hops; + + // Path waypoints as flat list of [x, y, z] + // Organized sequentially in row-major order matching hop_table. + // For each (src, dst) pair where hop_table[src][dst] > 0, + // that many consecutive waypoints define the path. + if (!mWaypoints.empty()) { + YAML::Node waypoints(YAML::NodeType::Sequence); + for (size_t i = 0; i < mWaypoints.size(); i++) { + YAML::Node pt(YAML::NodeType::Sequence); + pt.SetStyle(YAML::EmitterStyle::Flow); + char buf[64]; + snprintf(buf, sizeof(buf), "%.9g", mWaypoints[i].decodeX()); + pt.push_back(YAML::Load(buf)); + snprintf(buf, sizeof(buf), "%.9g", mWaypoints[i].decodeY()); + pt.push_back(YAML::Load(buf)); + snprintf(buf, sizeof(buf), "%.9g", mWaypoints[i].decodeZ()); + pt.push_back(YAML::Load(buf)); + waypoints.push_back(pt); + } + root["path_waypoints"] = waypoints; + } + + YAML::Emitter emit; + emit << root; + std::string yamlStr = emit.c_str(); + outYaml.assign(yamlStr.begin(), yamlStr.end()); +} + +void NvtInfo::loadFromYaml(const std::vector& yamlData) { + std::string yamlStr(yamlData.begin(), yamlData.end()); + YAML::Node root = YAML::Load(yamlStr); + + mMagic = root["magic"].as(); + mCount = root["count"].as(); + + // Face indices + const YAML::Node& indices = root["face_indices"]; + mFaceIndices.resize(mCount); + for (u32 i = 0; i < mCount; i++) { + mFaceIndices[i] = indices[i].as(); + } + + // Cost matrix + const YAML::Node& matrix = root["cost_matrix"]; + mCostMatrix.resize(mCount * mCount); + for (u32 i = 0; i < mCount; i++) { + for (u32 j = 0; j < mCount; j++) { + mCostMatrix[i * mCount + j] = matrix[i][j].as(); + } + } + + // Hop table + const YAML::Node& hops = root["hop_table"]; + mHopTable.resize(mCount * mCount); + for (u32 i = 0; i < mCount; i++) { + for (u32 j = 0; j < mCount; j++) { + mHopTable[i * mCount + j] = hops[i][j].as(); + } + } + + // Path waypoints + mWaypoints.clear(); + if (root["path_waypoints"]) { + const YAML::Node& wps = root["path_waypoints"]; + mWaypoints.resize(wps.size()); + for (size_t i = 0; i < wps.size(); i++) { + float x = wps[i][0].as(); + float y = wps[i][1].as(); + float z = wps[i][2].as(); + mWaypoints[i] = NvtWaypoint::encode(x, y, z); + } + } +} + +void NvtInfo::serializeToNvt(std::vector& out) { + u32 count = mCount; + u32 matrixEntries = count * count; + + size_t totalSize = 8 // header + + count * 2 // face indices + + matrixEntries * 4 // cost matrix + + matrixEntries // hop table + + mWaypoints.size() * 9; // path waypoints + + out.resize(totalSize, 0); + size_t offset = 0; + + // Header + memcpy(out.data() + offset, &mMagic, 4); offset += 4; + memcpy(out.data() + offset, &mCount, 4); offset += 4; + + // Face indices + for (u32 i = 0; i < count; i++) { + memcpy(out.data() + offset, &mFaceIndices[i], 2); + offset += 2; + } + + // Cost matrix + for (u32 i = 0; i < matrixEntries; i++) { + memcpy(out.data() + offset, &mCostMatrix[i], 4); + offset += 4; + } + + // Hop table + memcpy(out.data() + offset, mHopTable.data(), matrixEntries); + offset += matrixEntries; + + // Path waypoints + for (size_t i = 0; i < mWaypoints.size(); i++) { + u8* p = out.data() + offset + i * 9; + memcpy(p + 0, &mWaypoints[i].xVal, 2); + p[2] = mWaypoints[i].xSign; + memcpy(p + 3, &mWaypoints[i].yVal, 2); + p[5] = mWaypoints[i].ySign; + memcpy(p + 6, &mWaypoints[i].zVal, 2); + p[8] = mWaypoints[i].zSign; + } +} + +void NvtInfo::serializeToObj(std::vector& outObj) { + std::ostringstream oss; + oss << "# NVT Path Waypoints\n"; + oss << "# " << mWaypoints.size() << " waypoints\n"; + oss << "# Paths organized by hop_table: for each (src, dst) with hops > 0,\n"; + oss << "# that many consecutive vertices form the path.\n\n"; + + // Write all waypoints as vertices + for (size_t i = 0; i < mWaypoints.size(); i++) { + char buf[128]; + snprintf(buf, sizeof(buf), "v %.9g %.9g %.9g\n", + mWaypoints[i].decodeX(), + mWaypoints[i].decodeY(), + mWaypoints[i].decodeZ()); + oss << buf; + } + + // Write path polylines using the hop table + size_t wpIdx = 1; // OBJ is 1-indexed + for (u32 src = 0; src < mCount; src++) { + for (u32 dst = 0; dst < mCount; dst++) { + u8 hops = mHopTable[src * mCount + dst]; + if (hops > 0) { + oss << "g path_" << src << "_" << dst << "\nl"; + for (u8 h = 0; h < hops; h++) { + oss << " " << (wpIdx + h); + } + oss << "\n"; + wpIdx += hops; + } + } + } + + std::string s = oss.str(); + outObj.assign(s.begin(), s.end()); +} + +bool NvtInfo::loadFromObj(const std::vector& objData) { + std::istringstream iss(std::string(objData.begin(), objData.end())); + std::string line; + std::vector newWaypoints; + + while (std::getline(iss, line)) { + if (line.size() >= 2 && line[0] == 'v' && line[1] == ' ') { + float x, y, z; + if (sscanf(line.c_str(), "v %f %f %f", &x, &y, &z) == 3) { + newWaypoints.push_back(NvtWaypoint::encode(x, y, z)); + } + } + } + + if (newWaypoints.size() != mWaypoints.size()) { + std::cerr << "NVT OBJ vertex count mismatch: expected " + << mWaypoints.size() << ", got " << newWaypoints.size() << std::endl; + return false; + } + + mWaypoints = std::move(newWaypoints); + return true; +} + +} diff --git a/source/fivex/NvtInfo.hpp b/source/fivex/NvtInfo.hpp new file mode 100644 index 0000000..51ae586 --- /dev/null +++ b/source/fivex/NvtInfo.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "types.h" +#include +#include + +namespace FiveX { + +// A packed 3D waypoint used in nvt path data. +// Each coordinate is encoded as: (u16 / 65535.0) * 256.0 * (sign ? -1 : 1) +struct NvtWaypoint { + u16 xVal; u8 xSign; + u16 yVal; u8 ySign; + u16 zVal; u8 zSign; + + float decodeX() const { return (xVal / 65535.0f) * 256.0f * (xSign ? -1.0f : 1.0f); } + float decodeY() const { return (yVal / 65535.0f) * 256.0f * (ySign ? -1.0f : 1.0f); } + float decodeZ() const { return (zVal / 65535.0f) * 256.0f * (zSign ? -1.0f : 1.0f); } + + static NvtWaypoint encode(float x, float y, float z); +}; + +class NvtInfo { +public: + NvtInfo(); + ~NvtInfo(); + + void loadFromNvt(std::vector& in); + void serializeToYaml(std::vector& outYaml); + void loadFromYaml(const std::vector& yamlData); + void serializeToNvt(std::vector& out); + void serializeToObj(std::vector& outObj); + bool loadFromObj(const std::vector& objData); + +private: + u32 mMagic = 0; + u32 mCount = 0; + std::vector mFaceIndices; // count × u16 face indices + std::vector mCostMatrix; // count × count float cost matrix + std::vector mHopTable; // count × count u8 hop counts + std::vector mWaypoints; // decoded path waypoints (9 bytes each) +}; + +} diff --git a/source/havok/hkTagfile.cpp b/source/havok/hkTagfile.cpp new file mode 100644 index 0000000..a5ec5d8 --- /dev/null +++ b/source/havok/hkTagfile.cpp @@ -0,0 +1,2108 @@ +#include "havok/hkTagfile.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +// Base64 encoding/decoding for binary metadata storage +static const char b64chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +std::string base64Encode(const std::vector& data) { + std::string out; + out.reserve((data.size() + 2) / 3 * 4); + for (size_t i = 0; i < data.size(); i += 3) { + u32 n = (u32)data[i] << 16; + if (i + 1 < data.size()) n |= (u32)data[i+1] << 8; + if (i + 2 < data.size()) n |= data[i+2]; + out += b64chars[(n >> 18) & 0x3F]; + out += b64chars[(n >> 12) & 0x3F]; + out += (i + 1 < data.size()) ? b64chars[(n >> 6) & 0x3F] : '='; + out += (i + 2 < data.size()) ? b64chars[n & 0x3F] : '='; + } + return out; +} + +std::vector base64Decode(const std::string& str) { + static int lut[256] = {-1}; + if (lut[0] == -1) { + for (int i = 0; i < 256; i++) lut[i] = -1; + for (int i = 0; i < 64; i++) lut[(u8)b64chars[i]] = i; + } + std::vector out; + out.reserve(str.size() * 3 / 4); + u32 buf = 0; + int bits = 0; + for (char c : str) { + if (lut[(u8)c] < 0) continue; + buf = (buf << 6) | lut[(u8)c]; + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back((u8)(buf >> bits)); + } + } + return out; +} + +} // anonymous namespace + +namespace Havok { + +static constexpr u32 byteswap32(u32 x) { + return ((x << 24) & 0xFF000000) | ((x << 8) & 0x00FF0000) | + ((x >> 8) & 0x0000FF00) | ((x >> 24) & 0x000000FF); +} + +// NaN-preserving float serialization: YAML's .nan loses the specific NaN payload, +// so we store NaN as a tagged hex string to preserve the exact bit pattern. +YAML::Node floatToYaml(float f) { + if (std::isnan(f)) { + u32 bits; + memcpy(&bits, &f, 4); + char buf[16]; + snprintf(buf, sizeof(buf), "0x%08X", bits); + YAML::Node n; + n.SetTag("!nan"); + n = std::string(buf); + return n; + } + return YAML::Node(f); +} + +float yamlToFloat(const YAML::Node& node) { + if (node.Tag() == "!nan" || node.Tag() == "nan") { + u32 bits = (u32)std::stoul(node.as(), nullptr, 16); + float f; + memcpy(&f, &bits, 4); + return f; + } + return node.as(); +} + +// ============================================================================ +// Binary Loading +// ============================================================================ + +bool hkTagfile::loadFromBinary(const u8* data, size_t size) { + if (size < 8) return false; + + // TAG0 header + u32 tag0Size = readBE32(data) & 0x3FFFFFFF; + if (memcmp(data + 4, "TAG0", 4) != 0) return false; + + const u8* pos = data + 8; + const u8* tagEnd = data + tag0Size; + + // Parse sections within TAG0 + const u8* dataPayload = nullptr; + size_t dataPayloadSize = 0; + const u8* typePayload = nullptr; + size_t typePayloadSize = 0; + const u8* indxPayload = nullptr; + size_t indxPayloadSize = 0; + + while (pos < tagEnd) { + u32 secSizeRaw = readBE32(pos); + u32 secSize = secSizeRaw & 0x3FFFFFFF; + if (secSize < 8) { + // Skip zero padding (occurs between TYPE and INDX sections) + pos += 4; + continue; + } + char secMagic[5] = {}; + memcpy(secMagic, pos + 4, 4); + + if (strcmp(secMagic, "SDKV") == 0) { + mSdkvSizeRaw = secSizeRaw; + mSdkVersion = std::string((const char*)pos + 8, secSize - 8); + // Trim any trailing nulls + while (!mSdkVersion.empty() && mSdkVersion.back() == '\0') + mSdkVersion.pop_back(); + } else if (strcmp(secMagic, "DATA") == 0) { + mDataSizeRaw = secSizeRaw; + dataPayload = pos + 8; + dataPayloadSize = secSize - 8; + } else if (strcmp(secMagic, "TYPE") == 0) { + typePayload = pos; + typePayloadSize = secSize; + } else if (strcmp(secMagic, "INDX") == 0) { + mIndxTagOffset = (u32)(pos - data); + indxPayload = pos + 8; + indxPayloadSize = secSize - 8; + } + + pos += secSize; + } + + if (!dataPayload || !typePayload || !indxPayload) return false; + + // Store DATA section raw bytes + mDataSection.assign(dataPayload, dataPayload + dataPayloadSize); + mOrigDataPayloadSize = (u32)dataPayloadSize; + + // Store TYPE section verbatim (including its own size+magic header) + mTypeSection.assign(typePayload, typePayload + typePayloadSize); + + // Parse TYPE section for type definitions + parseTypeSection(); + + // Parse INDX section (ITEM + PTCH) + const u8* indxPos = indxPayload; + const u8* indxEnd = indxPayload + indxPayloadSize; + + while (indxPos < indxEnd) { + u32 subSecSize = readBE32(indxPos) & 0x3FFFFFFF; + if (subSecSize == 0) break; + char subMagic[5] = {}; + memcpy(subMagic, indxPos + 4, 4); + + if (strcmp(subMagic, "ITEM") == 0) { + const u8* itemData = indxPos + 8; + size_t itemDataSize = subSecSize - 8; + size_t itemCount = itemDataSize / 12; + mItems.resize(itemCount); + + for (size_t i = 0; i < itemCount; i++) { + const u8* entry = itemData + i * 12; + mItems[i].typeIndex = *(u16*)(entry); + // entry[2] is padding + mItems[i].flags = entry[3]; + mItems[i].offset = *(u32*)(entry + 4); + mItems[i].count = *(u32*)(entry + 8); + } + } else if (strcmp(subMagic, "PTCH") == 0) { + const u8* ptchData = indxPos + 8; + size_t ptchDataSize = subSecSize - 8; + const u8* p = ptchData; + const u8* ptchEnd = ptchData + ptchDataSize; + + while (p < ptchEnd) { + PatchEntry patch; + patch.typeIndex = *(u32*)p; + u32 cnt = *(u32*)(p + 4); + p += 8; + patch.offsets.resize(cnt); + for (u32 j = 0; j < cnt; j++) { + patch.offsets[j] = *(u32*)p; + p += 4; + } + mPatches.push_back(std::move(patch)); + } + } + + indxPos += subSecSize; + } + + return true; +} + +// ============================================================================ +// TYPE Section Parsing +// ============================================================================ + +void hkTagfile::parseTypeSection() { + if (mTypeSection.size() < 8) return; + + const u8* typeStart = mTypeSection.data(); + u32 typeSize = readBE32(typeStart) & 0x3FFFFFFF; + const u8* pos = typeStart + 8; // skip TYPE size+magic + const u8* typeEnd = typeStart + typeSize; + + const u8* tst1Data = nullptr; size_t tst1Size = 0; + const u8* tna1Data = nullptr; size_t tna1Size = 0; + const u8* fst1Data = nullptr; size_t fst1Size = 0; + const u8* tbdyData = nullptr; size_t tbdySize = 0; + + while (pos < typeEnd) { + u32 subSize = readBE32(pos) & 0x3FFFFFFF; + if (subSize == 0) break; // Prevent infinite loop + char subMagic[5] = {}; + memcpy(subMagic, pos + 4, 4); + + if (strcmp(subMagic, "TST1") == 0) { tst1Data = pos + 8; tst1Size = subSize - 8; } + else if (strcmp(subMagic, "TNA1") == 0) { tna1Data = pos + 8; tna1Size = subSize - 8; } + else if (strcmp(subMagic, "FST1") == 0) { fst1Data = pos + 8; fst1Size = subSize - 8; } + else if (strcmp(subMagic, "TBDY") == 0) { tbdyData = pos + 8; tbdySize = subSize - 8; } + + pos += subSize; + } + + // Parse string tables + if (tst1Data) { + mTypeStrings.clear(); + const char* s = (const char*)tst1Data; + const char* end = s + tst1Size; + while (s < end) { + size_t len = strnlen(s, end - s); + mTypeStrings.push_back(std::string(s, len)); + s += len + 1; + } + } + + if (fst1Data) { + mFieldStrings.clear(); + const char* s = (const char*)fst1Data; + const char* end = s + fst1Size; + while (s < end) { + size_t len = strnlen(s, end - s); + mFieldStrings.push_back(std::string(s, len)); + s += len + 1; + } + } + + // Parse TNA1 (type name association: sequential entries, NOT VLE) + // Format: count(u8), then (count-1) entries of: index(u8) + templateCount(u8) + templates + if (tna1Data && tna1Size > 0) { + mTypeNames.clear(); + const u8* p = tna1Data; + const u8* end = tna1Data + tna1Size; + + u8 count = *p++; + u32 numEntries = count - 1; + mTypeNames.resize(numEntries + 1); // typeIds start at 1 + + for (u32 i = 0; i < numEntries && p < end; i++) { + u8 strIdx = *p++; + u8 templateCount = *p++; + + u32 typeId = i + 1; // TNA1 entries correspond to TBDY typeIds 1, 2, 3, ... + if (strIdx < mTypeStrings.size()) + mTypeNames[typeId] = mTypeStrings[strIdx]; + + // Skip template entries (2 bytes each: index + value) + for (u8 t = 0; t < templateCount && p + 1 < end; t++) { + p += 2; + } + } + } + + // Parse TBDY + if (tbdyData) { + parseTbdy(tbdyData, tbdySize); + resolveTypeNames(); + } +} + +void hkTagfile::parseTbdy(const u8* tbdyData, size_t tbdySize) { + VleReader reader(tbdyData, tbdySize); + mTypes.clear(); + + while (reader.mPos < tbdySize) { + TypeDef type; + type.typeId = (u32)reader.readVle(); + type.parentTypeId = (u32)reader.readVle(); + type.optbits = (u32)reader.readVle(); + + if (type.optbits & OPT_FORMAT) { + type.format = (u32)reader.readVle(); + type.kind = (FormatKind)(type.format & KIND_MASK); + } + if (type.optbits & OPT_SUBTYPE) + type.subtypeId = (u32)reader.readVle(); + if (type.optbits & OPT_VERSION) + type.version = (u32)reader.readVle(); + if (type.optbits & OPT_SIZE_ALIGN) { + type.size = (u32)reader.readVle(); + type.align = (u32)reader.readVle(); + } + if (type.optbits & OPT_FLAGS) + type.flags = (u32)reader.readVle(); + if (type.optbits & OPT_DECLS) { + u64 encoded = reader.readVle(); + u32 numFields = (u32)(encoded & 0xFFFF); + u32 numProperties = (u32)(encoded >> 16); + (void)numProperties; + type.fields.resize(numFields); + for (u32 i = 0; i < numFields; i++) { + type.fields[i].nameStringId = (u32)reader.readVle(); + type.fields[i].flags = (u32)reader.readVle(); + type.fields[i].offset = (u32)reader.readVle(); + type.fields[i].typeId = (u32)reader.readVle(); + } + } + if (type.optbits & OPT_INTERFACES) { + u32 numIfaces = (u32)reader.readVle(); + type.interfaces.resize(numIfaces); + for (u32 i = 0; i < numIfaces; i++) { + type.interfaces[i].typeId = (u32)reader.readVle(); + type.interfaces[i].offset = (u32)reader.readVle(); + } + } + if (type.optbits & OPT_ATTRIBUTE_STRING) { + reader.readVle(); // skip attribute string id + } + if (type.optbits & OPT_MUTABLE) { + reader.readVle(); // skip mutable info + } + + // Ensure types vector is large enough + if (type.typeId >= mTypes.size()) + mTypes.resize(type.typeId + 1); + mTypes[type.typeId] = std::move(type); + } +} + +void hkTagfile::resolveTypeNames() { + for (auto& type : mTypes) { + if (type.typeId == 0) continue; + // Set name from TNA1 mapping + if (type.typeId < mTypeNames.size() && !mTypeNames[type.typeId].empty()) + type.name = mTypeNames[type.typeId]; + // Set field names from FST1 + for (auto& field : type.fields) { + if (field.nameStringId < mFieldStrings.size()) + field.name = mFieldStrings[field.nameStringId]; + } + } +} + +// ============================================================================ +// Type Helpers +// ============================================================================ + +const TypeDef* hkTagfile::getType(u32 typeId) const { + if (typeId < mTypes.size() && mTypes[typeId].typeId == typeId) + return &mTypes[typeId]; + return nullptr; +} + +u32 hkTagfile::getTypeByteSize(u32 typeId) const { + const TypeDef* type = getType(typeId); + if (!type) return 0; + if (type->size > 0) return type->size; + // Inherit size from parent chain + if (type->parentTypeId != 0) + return getTypeByteSize(type->parentTypeId); + return 0; +} + +u32 hkTagfile::getTypeAlignment(u32 typeId) const { + const TypeDef* type = getType(typeId); + if (!type) return 1; + if (type->align > 1 || (type->optbits & OPT_SIZE_ALIGN)) return type->align > 0 ? type->align : 1; + // Inherit alignment from parent chain + if (type->parentTypeId != 0) + return getTypeAlignment(type->parentTypeId); + return 1; +} + +bool hkTagfile::isPointerLikeType(u32 typeId) const { + const TypeDef* type = getType(typeId); + if (!type) return false; + if (type->kind == KIND_POINTER || type->kind == KIND_STRING || type->kind == KIND_ARRAY) + return true; + // Check parent chain + if (type->kind == KIND_VOID && type->parentTypeId != 0) + return isPointerLikeType(type->parentTypeId); + return false; +} + +FormatKind hkTagfile::resolveEffectiveKind(u32 typeId) const { + const TypeDef* type = getType(typeId); + if (!type) return KIND_VOID; + if (type->kind != KIND_VOID) return type->kind; + if (type->parentTypeId != 0) return resolveEffectiveKind(type->parentTypeId); + return KIND_VOID; +} + +const TypeDef* hkTagfile::resolveEffectiveType(u32 typeId) const { + const TypeDef* type = getType(typeId); + if (!type) return nullptr; + if (type->kind != KIND_VOID || type->parentTypeId == 0) return type; + return resolveEffectiveType(type->parentTypeId); +} + +// ============================================================================ +// JSON Serialization +// ============================================================================ + +json hkTagfile::decodeValue(const u8* data, const TypeDef& type) const { + switch (type.kind) { + case KIND_INT: { + u32 numBits = type.format >> INT_NUM_BITS_SHIFT; + bool isSigned = (type.format & INT_SIGNED_BIT) != 0; + if (numBits <= 8) { + return isSigned ? json((s8)data[0]) : json(data[0]); + } else if (numBits <= 16) { + u16 v = *(const u16*)data; + return isSigned ? json((s16)v) : json(v); + } else if (numBits <= 32) { + u32 v = *(const u32*)data; + return isSigned ? json((s32)v) : json(v); + } else { + u64 v = *(const u64*)data; + return isSigned ? json((s64)v) : json(v); + } + } + case KIND_FLOAT: { + if (type.size == 4) { + float v; + memcpy(&v, data, 4); + return json(v); + } else if (type.size == 8) { + double v; + memcpy(&v, data, 8); + return json(v); + } else if (type.size == 2) { + // Half float - store as raw u16 for now + u16 v = *(const u16*)data; + return json(v); + } + return json(nullptr); + } + case KIND_BOOL: { + if (type.size == 1) return json(data[0] != 0); + if (type.size == 4) return json(*(const u32*)data != 0); + return json(data[0] != 0); + } + case KIND_POINTER: + case KIND_STRING: { + // Pointer fields store item index as u64 + u64 itemRef = *(const u64*)data; + if (itemRef == 0) return json(nullptr); + json ref; + ref["$ref"] = itemRef; + return ref; + } + default: + break; + } + + // Opaque/void or unknown: return hex + if (type.size > 0) { + std::string hex; + hex.reserve(type.size * 2); + for (u32 i = 0; i < type.size; i++) { + char buf[4]; + snprintf(buf, sizeof(buf), "%02x", data[i]); + hex += buf; + } + return json(hex); + } + return json(nullptr); +} + +json hkTagfile::decodeRecord(const u8* data, const TypeDef& type) const { + json obj = json::object(); + + for (const auto& field : type.fields) { + const TypeDef* fieldType = getType(field.typeId); + if (!fieldType) continue; + + const u8* fieldData = data + field.offset; + + // Resolve effective kind through parent chain + FormatKind fKind = resolveEffectiveKind(field.typeId); + const TypeDef* fEffective = resolveEffectiveType(field.typeId); + if (!fEffective) fEffective = fieldType; + + if (fKind == KIND_RECORD) { + // Find type with actual fields + const TypeDef* rt = fEffective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (!rt) rt = fEffective; + obj[field.name] = decodeRecord(fieldData, *rt); + } else if (fKind == KIND_ARRAY) { + json arrObj = json::object(); + u64 itemRef = *(const u64*)fieldData; + arrObj["$ref"] = itemRef; + arrObj["m_size"] = *(const s32*)(fieldData + 8); + arrObj["m_capacityAndFlags"] = *(const s32*)(fieldData + 12); + obj[field.name] = arrObj; + } else { + obj[field.name] = decodeValue(fieldData, *fEffective); + } + } + + // Store any bytes not covered by fields as gaps + u32 typeSize = getTypeByteSize(type.typeId); + if (typeSize > 0) { + std::vector covered(typeSize, false); + for (const auto& field : type.fields) { + u32 fieldSize = getTypeByteSize(field.typeId); + for (u32 i = field.offset; i < field.offset + fieldSize && i < typeSize; i++) + covered[i] = true; + } + + bool hasGap = false; + for (u32 i = 0; i < typeSize; i++) { + if (!covered[i] && data[i] != 0) { hasGap = true; break; } + } + + if (hasGap) { + std::string gapHex; + gapHex.reserve(typeSize * 2); + for (u32 i = 0; i < typeSize; i++) { + char buf[4]; + snprintf(buf, sizeof(buf), "%02x", data[i]); + gapHex += buf; + } + obj["_raw"] = gapHex; + } + } + + return obj; +} + +json hkTagfile::itemToJson(size_t itemIndex) const { + if (itemIndex == 0 || itemIndex >= mItems.size()) return json(nullptr); + + const auto& item = mItems[itemIndex]; + const TypeDef* type = getType(item.typeIndex); + + json result = json::object(); + if (type) { + result["_type"] = type->name.empty() ? + ("type_" + std::to_string(item.typeIndex)) : type->name; + } else { + result["_type"] = "unknown_type_" + std::to_string(item.typeIndex); + } + result["_typeIndex"] = item.typeIndex; + result["_flags"] = item.flags; + result["_offset"] = item.offset; + result["_count"] = item.count; + + u32 elementSize = type ? getTypeByteSize(item.typeIndex) : 0; + + // Store raw hex for every item to guarantee perfect round-tripping + if (item.count > 0 && elementSize > 0) { + size_t totalSize = (size_t)item.count * elementSize; + if (item.offset + totalSize <= mDataSection.size()) { + const u8* rawData = mDataSection.data() + item.offset; + std::string hex; + hex.reserve(totalSize * 2); + for (size_t i = 0; i < totalSize; i++) { + char buf[4]; + snprintf(buf, sizeof(buf), "%02x", rawData[i]); + hex += buf; + } + result["_hex"] = hex; + } + } + + // Resolve the effective type kind through parent chain (only if type is known) + if (!type) return result; + + FormatKind effectiveKind = resolveEffectiveKind(item.typeIndex); + const TypeDef* effectiveType = resolveEffectiveType(item.typeIndex); + if (!effectiveType) effectiveType = type; + + if (effectiveKind == KIND_RECORD && item.count > 0 && elementSize > 0) { + // Find the type with actual fields (may need to walk parent chain) + const TypeDef* recordType = effectiveType; + while (recordType && recordType->fields.empty() && recordType->parentTypeId != 0) { + const TypeDef* parent = getType(recordType->parentTypeId); + if (parent) recordType = parent; + else break; + } + + if (item.count == 1 && item.flags == 0x10) { + const u8* objData = mDataSection.data() + item.offset; + result["_fields"] = decodeRecord(objData, *recordType); + } else { + json elements = json::array(); + for (u32 i = 0; i < item.count; i++) { + const u8* objData = mDataSection.data() + item.offset + i * elementSize; + elements.push_back(decodeRecord(objData, *recordType)); + } + result["_elements"] = elements; + } + } else if (effectiveKind == KIND_INT && item.count > 0) { + u32 numBits = effectiveType->format >> INT_NUM_BITS_SHIFT; + if (numBits <= 8) { + const u8* strData = mDataSection.data() + item.offset; + bool isString = true; + for (u32 i = 0; i < item.count; i++) { + u8 c = strData[i]; + if (c != 0 && (c < 0x20 || c > 0x7E)) { isString = false; break; } + } + if (isString && item.count > 0) { + size_t len = strnlen((const char*)strData, item.count); + result["_string"] = std::string((const char*)strData, len); + } + } else { + json arr = json::array(); + const u8* arrData = mDataSection.data() + item.offset; + for (u32 i = 0; i < item.count; i++) { + const u8* elem = arrData + i * elementSize; + arr.push_back(decodeValue(elem, *effectiveType)); + } + result["_elements"] = arr; + } + } else if (effectiveKind == KIND_FLOAT && item.count > 0) { + json arr = json::array(); + const u8* arrData = mDataSection.data() + item.offset; + for (u32 i = 0; i < item.count; i++) { + const u8* elem = arrData + i * elementSize; + arr.push_back(decodeValue(elem, *effectiveType)); + } + result["_elements"] = arr; + } else if (effectiveKind == KIND_BOOL && item.count > 0) { + json arr = json::array(); + const u8* arrData = mDataSection.data() + item.offset; + for (u32 i = 0; i < item.count; i++) { + const u8* elem = arrData + i * elementSize; + arr.push_back(decodeValue(elem, *effectiveType)); + } + result["_elements"] = arr; + } + + return result; +} + +json hkTagfile::toJson() const { + json root = json::object(); + root["sdk_version"] = mSdkVersion; + root["data_size"] = (u64)mDataSection.size(); + root["sdkv_size_raw"] = mSdkvSizeRaw; + root["data_size_raw"] = mDataSizeRaw; + root["indx_tag_offset"] = mIndxTagOffset; + + // Store TYPE section as hex for verbatim round-trip + { + std::string typeHex; + typeHex.reserve(mTypeSection.size() * 2); + for (size_t i = 0; i < mTypeSection.size(); i++) { + char buf[4]; + snprintf(buf, sizeof(buf), "%02x", mTypeSection[i]); + typeHex += buf; + } + root["type_section"] = typeHex; + } + + // Store ENTIRE DATA section as hex for guaranteed perfect round-trip + { + std::string dataHex; + dataHex.reserve(mDataSection.size() * 2); + for (size_t i = 0; i < mDataSection.size(); i++) { + char buf[4]; + snprintf(buf, sizeof(buf), "%02x", mDataSection[i]); + dataHex += buf; + } + root["data_section"] = dataHex; + } + + // Items (decoded for human readability) + json items = json::array(); + items.push_back(nullptr); // item 0 is always null + for (size_t i = 1; i < mItems.size(); i++) { + items.push_back(itemToJson(i)); + } + root["items"] = items; + + // Patches (for reference) + json patches = json::array(); + for (const auto& patch : mPatches) { + json p = json::object(); + const TypeDef* patchType = getType(patch.typeIndex); + if (patchType && !patchType->name.empty()) + p["type"] = patchType->name; + p["typeIndex"] = patch.typeIndex; + p["offsets"] = patch.offsets; + patches.push_back(p); + } + root["patches"] = patches; + + return root; +} + +// ============================================================================ +// JSON Deserialization +// ============================================================================ + +void hkTagfile::encodeValue(u8* data, const TypeDef& type, const json& j) const { + switch (type.kind) { + case KIND_INT: { + u32 numBits = type.format >> INT_NUM_BITS_SHIFT; + bool isSigned = (type.format & INT_SIGNED_BIT) != 0; + if (numBits <= 8) { + if (isSigned) data[0] = (u8)(s8)j.get(); + else data[0] = (u8)j.get(); + } else if (numBits <= 16) { + u16 v; + if (isSigned) v = (u16)(s16)j.get(); + else v = (u16)j.get(); + memcpy(data, &v, 2); + } else if (numBits <= 32) { + if (isSigned) { + s32 v = j.get(); + memcpy(data, &v, 4); + } else { + u32 v = j.get(); + memcpy(data, &v, 4); + } + } else { + if (isSigned) { + s64 v = j.get(); + memcpy(data, &v, 8); + } else { + u64 v = j.get(); + memcpy(data, &v, 8); + } + } + break; + } + case KIND_FLOAT: { + if (type.size == 4) { + float v = j.get(); + memcpy(data, &v, 4); + } else if (type.size == 8) { + double v = j.get(); + memcpy(data, &v, 8); + } else if (type.size == 2) { + u16 v = j.get(); + memcpy(data, &v, 2); + } + break; + } + case KIND_BOOL: { + bool v = j.get(); + if (type.size == 1) data[0] = v ? 1 : 0; + else if (type.size == 4) { u32 bv = v ? 1 : 0; memcpy(data, &bv, 4); } + break; + } + case KIND_POINTER: + case KIND_STRING: { + u64 itemRef = 0; + if (!j.is_null() && j.is_object() && j.contains("$ref")) + itemRef = j["$ref"].get(); + memcpy(data, &itemRef, 8); + break; + } + default: break; + } +} + +void hkTagfile::encodeRecord(u8* data, const TypeDef& type, const json& j) const { + // If there's a _raw field, start from that + if (j.contains("_raw")) { + std::string hex = j["_raw"].get(); + u32 typeSize = getTypeByteSize(type.typeId); + for (size_t i = 0; i + 1 < hex.size() && i / 2 < typeSize; i += 2) { + char byte[3] = {hex[i], hex[i+1], 0}; + data[i/2] = (u8)strtoul(byte, nullptr, 16); + } + } + + for (const auto& field : type.fields) { + if (!j.contains(field.name)) continue; + const json& fieldJson = j[field.name]; + const TypeDef* fieldType = getType(field.typeId); + if (!fieldType) continue; + + u8* fieldData = data + field.offset; + + FormatKind fKind = resolveEffectiveKind(field.typeId); + const TypeDef* fEffective = resolveEffectiveType(field.typeId); + if (!fEffective) fEffective = fieldType; + + if (fKind == KIND_RECORD) { + const TypeDef* rt = fEffective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (!rt) rt = fEffective; + encodeRecord(fieldData, *rt, fieldJson); + } else if (fKind == KIND_ARRAY) { + u64 itemRef = 0; + s32 mSize = 0, mCapFlags = 0; + if (fieldJson.is_object()) { + if (fieldJson.contains("$ref")) + itemRef = fieldJson["$ref"].get(); + if (fieldJson.contains("m_size")) + mSize = fieldJson["m_size"].get(); + if (fieldJson.contains("m_capacityAndFlags")) + mCapFlags = fieldJson["m_capacityAndFlags"].get(); + } + memcpy(fieldData, &itemRef, 8); + memcpy(fieldData + 8, &mSize, 4); + memcpy(fieldData + 12, &mCapFlags, 4); + } else { + encodeValue(fieldData, *fEffective, fieldJson); + } + } +} + +bool hkTagfile::fromJson(const json& j) { + mSdkVersion = j["sdk_version"].get(); + + // Restore structural metadata + mSdkvSizeRaw = j.value("sdkv_size_raw", (u32)0x40000010); + mDataSizeRaw = j.value("data_size_raw", (u32)0); + mIndxTagOffset = j.value("indx_tag_offset", (u32)0); + + // Restore TYPE section from hex + std::string typeHex = j["type_section"].get(); + mTypeSection.resize(typeHex.size() / 2); + for (size_t i = 0; i + 1 < typeHex.size(); i += 2) { + char byte[3] = {typeHex[i], typeHex[i+1], 0}; + mTypeSection[i/2] = (u8)strtoul(byte, nullptr, 16); + } + + // Re-parse TYPE section + parseTypeSection(); + + // Restore DATA section from hex (primary path for perfect round-trip) + if (j.contains("data_section")) { + std::string dataHex = j["data_section"].get(); + mDataSection.resize(dataHex.size() / 2); + for (size_t i = 0; i + 1 < dataHex.size(); i += 2) { + char byte[3] = {dataHex[i], dataHex[i+1], 0}; + mDataSection[i/2] = (u8)strtoul(byte, nullptr, 16); + } + } else { + // Fallback: reconstruct DATA from per-item _hex fields + u64 dataSize = 0; + if (j.contains("data_size")) + dataSize = j["data_size"].get(); + mDataSection.resize(dataSize, 0); + } + + // Parse items from JSON (metadata for ITEM table) + const json& jsonItems = j["items"]; + mItems.resize(jsonItems.size()); + mItems[0] = {}; // null item + + for (size_t i = 1; i < jsonItems.size(); i++) { + const json& itemJson = jsonItems[i]; + if (itemJson.is_null()) continue; + + mItems[i].typeIndex = itemJson["_typeIndex"].get(); + mItems[i].flags = itemJson["_flags"].get(); + + if (itemJson.contains("_offset")) + mItems[i].offset = itemJson["_offset"].get(); + else + mItems[i].offset = 0; + + if (itemJson.contains("_count")) { + mItems[i].count = itemJson["_count"].get(); + } else if (itemJson.contains("_elements")) { + mItems[i].count = (u32)itemJson["_elements"].size(); + } else if (itemJson.contains("_string")) { + std::string str = itemJson["_string"].get(); + mItems[i].count = (u32)(str.size() + 1); + } else if (itemJson.contains("_hex")) { + u32 elementSize = getTypeByteSize(mItems[i].typeIndex); + std::string hex = itemJson["_hex"].get(); + u32 totalBytes = (u32)(hex.size() / 2); + mItems[i].count = elementSize > 0 ? totalBytes / elementSize : totalBytes; + } else if (itemJson.contains("_fields")) { + mItems[i].count = 1; + } else { + mItems[i].count = 1; + } + } + + // If no data_section was available, try reconstructing from per-item _hex + if (!j.contains("data_section")) { + for (size_t i = 1; i < jsonItems.size(); i++) { + const json& itemJson = jsonItems[i]; + if (itemJson.is_null()) continue; + if (!itemJson.contains("_hex")) continue; + if (mItems[i].offset >= mDataSection.size()) continue; + + std::string hex = itemJson["_hex"].get(); + u8* itemData = mDataSection.data() + mItems[i].offset; + for (size_t h = 0; h + 1 < hex.size(); h += 2) { + char byte[3] = {hex[h], hex[h+1], 0}; + if (mItems[i].offset + h/2 < mDataSection.size()) + itemData[h/2] = (u8)strtoul(byte, nullptr, 16); + } + } + } + + // Reconstruct patches from JSON + mPatches.clear(); + if (j.contains("patches")) { + for (const auto& pj : j["patches"]) { + PatchEntry patch; + patch.typeIndex = pj["typeIndex"].get(); + for (const auto& off : pj["offsets"]) + patch.offsets.push_back(off.get()); + mPatches.push_back(std::move(patch)); + } + } + + return true; +} + +// ============================================================================ +// YAML - Value Decoding +// ============================================================================ + +YAML::Node hkTagfile::decodeValueYaml(const u8* data, const TypeDef& type) const { + YAML::Node node; + switch (type.kind) { + case KIND_INT: { + u32 numBits = type.format >> INT_NUM_BITS_SHIFT; + bool isSigned = (type.format & INT_SIGNED_BIT) != 0; + if (numBits <= 8) { + node = isSigned ? (int)(s8)data[0] : (int)data[0]; + } else if (numBits <= 16) { + u16 raw; memcpy(&raw, data, 2); + node = isSigned ? (int)(s16)raw : (int)raw; + } else if (numBits <= 32) { + if (isSigned) { s32 v; memcpy(&v, data, 4); node = v; } + else { u32 v; memcpy(&v, data, 4); node = (int64_t)v; } + } else { + if (isSigned) { s64 v; memcpy(&v, data, 8); node = v; } + else { u64 v; memcpy(&v, data, 8); node = v; } + } + break; + } + case KIND_FLOAT: { + if (type.size == 4) { + float v; memcpy(&v, data, 4); + node = floatToYaml(v); + } else if (type.size == 8) { + double v; memcpy(&v, data, 8); + node = v; + } else if (type.size == 2) { + u16 raw; memcpy(&raw, data, 2); + // half-float → store as raw int for lossless round-trip + node = (int)raw; + node.SetTag("!half"); + } + break; + } + case KIND_BOOL: { + if (type.size == 1) node = (data[0] != 0); + else if (type.size == 4) { u32 v; memcpy(&v, data, 4); node = (v != 0); } + else node = (data[0] != 0); + break; + } + case KIND_POINTER: + case KIND_STRING: { + // Pointers resolve to item references + u64 ptrVal; memcpy(&ptrVal, data, 8); + if (ptrVal == 0) { + node = YAML::Node(YAML::NodeType::Null); + } else { + // Find the item at this offset (only valid if fits in u32) + if (ptrVal <= UINT32_MAX) { + for (size_t i = 1; i < mItems.size(); i++) { + if (mItems[i].offset == (u32)ptrVal) { + // If it's a string item, inline the string value + if (type.kind == KIND_STRING) { + const u8* strData = mDataSection.data() + mItems[i].offset; + node = std::string((const char*)strData); + } else { + node = (int64_t)i; + node.SetTag("!ref"); + } + break; + } + } + } + if (!node.IsDefined() || (node.IsNull() && ptrVal != 0)) { + node = (int64_t)ptrVal; + node.SetTag("!ptr"); + } + } + break; + } + default: + node = YAML::Node(YAML::NodeType::Null); + break; + } + return node; +} + +// ============================================================================ +// YAML - Record Decoding +// ============================================================================ + +YAML::Node hkTagfile::decodeRecordYaml(const u8* data, const TypeDef& type) const { + // Special handling: hkVector4/hkVector4f types should display as 4 floats + { + bool isVec4 = false; + const TypeDef* check = &type; + while (check) { + if (check->name == "hkVector4" || check->name == "hkVector4f") { + isVec4 = true; + break; + } + if (check->parentTypeId != 0) + check = getType(check->parentTypeId); + else + check = nullptr; + } + if (isVec4) { + YAML::Node vec(YAML::NodeType::Sequence); + for (int i = 0; i < 4; i++) { + float f; + memcpy(&f, data + i * 4, 4); + vec.push_back(floatToYaml(f)); + } + return vec; + } + } + + YAML::Node node(YAML::NodeType::Map); + + // Follow parent chain to get all fields + const TypeDef* cur = &type; + while (cur) { + for (const auto& field : cur->fields) { + const u8* fieldData = data + field.offset; + const TypeDef* fieldType = getType(field.typeId); + if (!fieldType) continue; + + FormatKind fKind = resolveEffectiveKind(field.typeId); + const TypeDef* fEffective = resolveEffectiveType(field.typeId); + if (!fEffective) fEffective = fieldType; + + if (fKind == KIND_RECORD) { + // Inline record - walk parent chain to find fields + const TypeDef* rt = fEffective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (!rt) rt = fEffective; + node[field.name] = decodeRecordYaml(fieldData, *rt); + } else if (fKind == KIND_ARRAY) { + // Distinguish hkArray (with m_data/m_size/m_capacityAndFlags fields) + // from fixed-size inline arrays like hkVector4f (no fields, just subtype) + const TypeDef* arrType = fieldType; + bool isHkArray = false; + const TypeDef* checkArr = arrType; + while (checkArr) { + if (!checkArr->fields.empty()) { + for (const auto& af : checkArr->fields) { + if (af.name == "m_data") { isHkArray = true; break; } + } + break; + } + if (checkArr->parentTypeId != 0) + checkArr = getType(checkArr->parentTypeId); + else + checkArr = nullptr; + } + + if (!isHkArray && fEffective && fEffective->subtypeId != 0 && fEffective->size > 0) { + // Fixed-size inline array (e.g., hkVector4f = 4 floats) + const TypeDef* elemType = getType(fEffective->subtypeId); + u32 elemSize = elemType ? elemType->size : 0; + if (elemType && elemSize > 0) { + u32 elemCount = fEffective->size / elemSize; + YAML::Node vec(YAML::NodeType::Sequence); + for (u32 vi = 0; vi < elemCount; vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f; + memcpy(&f, fieldData + vi * 4, 4); + vec.push_back(floatToYaml(f)); + } else if (elemType->kind == KIND_INT && elemSize == 4) { + s32 val; + memcpy(&val, fieldData + vi * 4, 4); + vec.push_back(val); + } else { + vec.push_back((int)fieldData[vi * elemSize]); + } + } + node[field.name] = vec; + } else { + // Fallback: store as raw values + node[field.name] = decodeValueYaml(fieldData, *fEffective); + } + } else { + // Standard hkArray: ptr(8) + size(4) + capAndFlags(4) = 16 bytes + u64 ptr; memcpy(&ptr, fieldData, 8); + s32 mSize; memcpy(&mSize, fieldData + 8, 4); + s32 mCapFlags; memcpy(&mCapFlags, fieldData + 12, 4); + + YAML::Node arrNode(YAML::NodeType::Map); + if (ptr == 0) { + arrNode["$ref"] = YAML::Node(YAML::NodeType::Null); + } else { + // Find item at this offset + for (size_t i = 1; i < mItems.size(); i++) { + if (mItems[i].offset == (u32)ptr) { + arrNode["$ref"] = (int64_t)i; + break; + } + } + } + arrNode["m_size"] = mSize; + arrNode["m_capacityAndFlags"] = mCapFlags; + node[field.name] = arrNode; + } + } else { + node[field.name] = decodeValueYaml(fieldData, *fEffective); + } + } + if (cur->parentTypeId != 0) cur = getType(cur->parentTypeId); + else cur = nullptr; + } + + return node; +} + +// ============================================================================ +// YAML - Item Decoding +// ============================================================================ + +YAML::Node hkTagfile::itemToYaml(size_t itemIndex) const { + if (itemIndex >= mItems.size()) return YAML::Node(YAML::NodeType::Null); + const auto& item = mItems[itemIndex]; + + if (item.offset >= mDataSection.size() && item.offset != 0) + return YAML::Node(YAML::NodeType::Null); + + const TypeDef* type = getType(item.typeIndex); + + YAML::Node node(YAML::NodeType::Map); + node["_typeIndex"] = item.typeIndex; + node["_flags"] = (int)item.flags; + + if (type && !type->name.empty()) + node["_typeName"] = type->name; + + FormatKind kind = resolveEffectiveKind(item.typeIndex); + const TypeDef* effective = resolveEffectiveType(item.typeIndex); + if (!effective) effective = type; + + const u8* itemData = mDataSection.data() + item.offset; + u32 elementSize = type ? getTypeByteSize(item.typeIndex) : 0; + + // tT is always char (1 byte) regardless of type table inheritance + bool isCharType = (type && (type->name == "tT" || type->name == "char")); + if (isCharType) elementSize = 1; + + u32 totalBytes = elementSize > 0 ? elementSize * item.count : 0; + + // String items (including tT char arrays) + if (kind == KIND_STRING || isCharType) { + u32 strLen = item.count > 0 ? item.count - 1 : 0; + if (item.offset + strLen <= mDataSection.size()) + node["_string"] = std::string((const char*)itemData, strLen); + return node; + } + + // Items with known type - decode fields + // Records and opaque structs (KIND_VOID with size > 0 treated as potential records) + if (type && effective && (kind == KIND_RECORD || (kind == KIND_VOID && elementSize > 0))) { + const TypeDef* rt = effective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (!rt || rt->fields.empty()) { + // No field info - store as byte array + if (totalBytes > 0 && item.offset + totalBytes <= mDataSection.size()) { + YAML::Node bytes(YAML::NodeType::Sequence); + for (u32 b = 0; b < totalBytes; b++) + bytes.push_back((int)itemData[b]); + node["_bytes"] = bytes; + } + return node; + } + + if (item.flags == 0x10) { + // Single record + node["_fields"] = decodeRecordYaml(itemData, *rt); + } else if (item.flags == 0x20 && item.count > 0) { + // Array of records + YAML::Node elements(YAML::NodeType::Sequence); + for (u32 e = 0; e < item.count; e++) { + const u8* eData = itemData + e * elementSize; + if (item.offset + (e + 1) * elementSize <= mDataSection.size()) + elements.push_back(decodeRecordYaml(eData, *rt)); + } + node["_elements"] = elements; + } + return node; + } + + // Items with known non-record types (int, float, bool, pointer arrays) + if (type && effective && (kind == KIND_INT || kind == KIND_FLOAT || kind == KIND_BOOL || kind == KIND_POINTER)) { + // Check for char/byte arrays that are actually strings + if (kind == KIND_INT && elementSize == 1 && item.flags == 0x20 && item.count > 0) { + // Check if this looks like a null-terminated string + u32 strLen = 0; + bool isAscii = true; + for (u32 i = 0; i < item.count && item.offset + i < mDataSection.size(); i++) { + u8 c = itemData[i]; + if (c == 0) { strLen = i; break; } + if (c < 0x20 || c > 0x7E) { isAscii = false; break; } + strLen = i + 1; + } + if (isAscii && strLen > 0) { + node["_string"] = std::string((const char*)itemData, strLen); + return node; + } + } + + if (item.count == 1 || item.flags == 0x10) { + node["_value"] = decodeValueYaml(itemData, *effective); + } else { + YAML::Node elements(YAML::NodeType::Sequence); + for (u32 e = 0; e < item.count; e++) { + const u8* eData = itemData + e * elementSize; + if (item.offset + (e + 1) * elementSize <= mDataSection.size()) + elements.push_back(decodeValueYaml(eData, *effective)); + } + node["_elements"] = elements; + } + return node; + } + + // Fixed-size inline array types (e.g., hkVector4f = 4 floats inline) + if (type && effective && kind == KIND_ARRAY && effective->subtypeId != 0 && effective->size > 0) { + const TypeDef* elemType = getType(effective->subtypeId); + u32 elemSize = elemType ? elemType->size : 0; + if (elemType && elemSize > 0) { + // Use the item's own type size (elementSize) when it's smaller than the + // ancestor array's size - the type overrides the array's inline extent + u32 vecSize = (elementSize > 0 && elementSize < effective->size) ? elementSize : effective->size; + u32 elemsPerVec = vecSize / elemSize; + if (item.count == 1 || item.flags == 0x10) { + // Single vector + YAML::Node vec(YAML::NodeType::Sequence); + for (u32 vi = 0; vi < elemsPerVec; vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f; memcpy(&f, itemData + vi * 4, 4); + vec.push_back(floatToYaml(f)); + } else { + s32 val; memcpy(&val, itemData + vi * elemSize, std::min(elemSize, 4u)); + vec.push_back(val); + } + } + node["_elements"] = vec; + } else if (item.count > 0) { + // Array of vectors + YAML::Node elements(YAML::NodeType::Sequence); + for (u32 e = 0; e < item.count; e++) { + const u8* eData = itemData + e * vecSize; + if (item.offset + (e + 1) * vecSize > mDataSection.size()) break; + YAML::Node vec(YAML::NodeType::Sequence); + for (u32 vi = 0; vi < elemsPerVec; vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f; memcpy(&f, eData + vi * 4, 4); + vec.push_back(floatToYaml(f)); + } else { + s32 val; memcpy(&val, eData + vi * elemSize, std::min(elemSize, 4u)); + vec.push_back(val); + } + } + elements.push_back(vec); + } + node["_elements"] = elements; + } + return node; + } + } + + // Unknown type - store as byte array (decimal, not hex) + // Only use item.count as byte count if elementSize > 0 (avoids reading random bytes + // for zero-size reference anchor items) + if (totalBytes == 0 && elementSize > 0) totalBytes = item.count; + if (totalBytes > 0 && item.offset + totalBytes <= mDataSection.size()) { + YAML::Node bytes(YAML::NodeType::Sequence); + for (u32 b = 0; b < totalBytes; b++) + bytes.push_back((int)itemData[b]); + node["_bytes"] = bytes; + } + + return node; +} + +// ============================================================================ +// YAML Serialization +// ============================================================================ + +YAML::Node hkTagfile::toYaml() const { + YAML::Node root(YAML::NodeType::Map); + root["sdk_version"] = mSdkVersion; + + // Items (decoded for human readability) + YAML::Node items(YAML::NodeType::Sequence); + items.push_back(YAML::Node(YAML::NodeType::Null)); // item 0 is always null + for (size_t i = 1; i < mItems.size(); i++) { + items.push_back(itemToYaml(i)); + } + root["items"] = items; + + // Patches (for reference, human-readable) + YAML::Node patches(YAML::NodeType::Sequence); + for (const auto& patch : mPatches) { + YAML::Node p(YAML::NodeType::Map); + const TypeDef* patchType = getType(patch.typeIndex); + if (patchType && !patchType->name.empty()) + p["type"] = patchType->name; + p["typeIndex"] = patch.typeIndex; + YAML::Node offsets(YAML::NodeType::Sequence); + for (u32 off : patch.offsets) offsets.push_back(off); + p["offsets"] = offsets; + patches.push_back(p); + } + root["patches"] = patches; + + // ---- Metadata section (for byte-perfect round-trip) ---- + YAML::Node metadata(YAML::NodeType::Map); + metadata["sdkv_size_raw"] = mSdkvSizeRaw; + metadata["data_size_raw"] = mDataSizeRaw; + metadata["indx_tag_offset"] = mIndxTagOffset; + metadata["data_size"] = (u64)mDataSection.size(); + metadata["orig_data_payload_size"] = mOrigDataPayloadSize; + + // Type section as base64 + metadata["type_section"] = base64Encode(mTypeSection); + + // Data section as base64 + metadata["data_section"] = base64Encode(mDataSection); + + // Item layout (offsets/counts for reconstruction) + YAML::Node itemLayout(YAML::NodeType::Sequence); + for (size_t i = 0; i < mItems.size(); i++) { + YAML::Node il(YAML::NodeType::Map); + il["typeIndex"] = mItems[i].typeIndex; + il["flags"] = (int)mItems[i].flags; + il["offset"] = mItems[i].offset; + il["count"] = mItems[i].count; + itemLayout.push_back(il); + } + metadata["item_layout"] = itemLayout; + + root["_metadata"] = metadata; + + return root; +} + +// ============================================================================ +// YAML - Value Encoding (inverse of decodeValueYaml) +// ============================================================================ + +void hkTagfile::encodeValueYaml(const YAML::Node& node, u8* dst, const TypeDef& type) const { + switch (type.kind) { + case KIND_INT: { + u32 numBits = type.format >> INT_NUM_BITS_SHIFT; + bool isSigned = (type.format & INT_SIGNED_BIT) != 0; + if (numBits <= 8) { + if (isSigned) dst[0] = (u8)(s8)node.as(); + else dst[0] = (u8)node.as(); + } else if (numBits <= 16) { + u16 v; + if (isSigned) v = (u16)(s16)node.as(); + else v = (u16)node.as(); + memcpy(dst, &v, 2); + } else if (numBits <= 32) { + if (isSigned) { s32 v = node.as(); memcpy(dst, &v, 4); } + else { u32 v = (u32)node.as(); memcpy(dst, &v, 4); } + } else { + if (isSigned) { s64 v = node.as(); memcpy(dst, &v, 8); } + else { u64 v = node.as(); memcpy(dst, &v, 8); } + } + break; + } + case KIND_FLOAT: { + if (type.size == 4) { + float v = yamlToFloat(node); + memcpy(dst, &v, 4); + } else if (type.size == 8) { + double v = node.as(); + memcpy(dst, &v, 8); + } else if (type.size == 2) { + // Half-float stored as raw u16 with !half tag + u16 v = (u16)node.as(); + memcpy(dst, &v, 2); + } + break; + } + case KIND_BOOL: { + bool v = node.as(); + if (type.size == 1) dst[0] = v ? 1 : 0; + else if (type.size == 4) { u32 bv = v ? 1 : 0; memcpy(dst, &bv, 4); } + break; + } + case KIND_POINTER: + case KIND_STRING: { + // Encode pointer values from YAML tagged nodes + if (node.Tag() == "!ref") { + size_t refIdx = node.as(); + u64 ptrVal = 0; + if (refIdx < mItems.size()) + ptrVal = mItems[refIdx].offset; + memcpy(dst, &ptrVal, 8); + } else if (node.Tag() == "!ptr") { + u64 ptrVal = (u64)node.as(); + memcpy(dst, &ptrVal, 8); + } + // Null and untagged: keep original bytes from data copy + break; + } + default: break; + } +} + +void hkTagfile::encodeRecordYaml(const YAML::Node& node, u8* dst, const TypeDef& type) const { + // hkVector4/hkVector4f: sequence of 4 floats + { + bool isVec4 = false; + const TypeDef* check = &type; + while (check) { + if (check->name == "hkVector4" || check->name == "hkVector4f") { + isVec4 = true; + break; + } + if (check->parentTypeId != 0) + check = getType(check->parentTypeId); + else + check = nullptr; + } + if (isVec4 && node.IsSequence() && node.size() == 4) { + for (int i = 0; i < 4; i++) { + float f = yamlToFloat(node[i]); + memcpy(dst + i * 4, &f, 4); + } + return; + } + } + + if (!node.IsMap()) return; + + // Follow parent chain to encode all fields + const TypeDef* cur = &type; + while (cur) { + for (const auto& field : cur->fields) { + if (!node[field.name]) continue; + const YAML::Node& fieldNode = node[field.name]; + const TypeDef* fieldType = getType(field.typeId); + if (!fieldType) continue; + + u8* fieldData = dst + field.offset; + + FormatKind fKind = resolveEffectiveKind(field.typeId); + const TypeDef* fEffective = resolveEffectiveType(field.typeId); + if (!fEffective) fEffective = fieldType; + + if (fKind == KIND_RECORD) { + const TypeDef* rt = fEffective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (!rt) rt = fEffective; + encodeRecordYaml(fieldNode, fieldData, *rt); + } else if (fKind == KIND_ARRAY) { + // Distinguish hkArray from fixed-size inline arrays + const TypeDef* arrType = fieldType; + bool isHkArray = false; + const TypeDef* checkArr = arrType; + while (checkArr) { + if (!checkArr->fields.empty()) { + for (const auto& af : checkArr->fields) { + if (af.name == "m_data") { isHkArray = true; break; } + } + break; + } + if (checkArr->parentTypeId != 0) + checkArr = getType(checkArr->parentTypeId); + else + checkArr = nullptr; + } + + if (!isHkArray && fEffective && fEffective->subtypeId != 0 && fEffective->size > 0) { + // Fixed-size inline array (e.g., 4 floats for hkVector4f) + const TypeDef* elemType = getType(fEffective->subtypeId); + u32 elemSize = elemType ? elemType->size : 0; + if (elemType && elemSize > 0 && fieldNode.IsSequence()) { + u32 elemCount = fEffective->size / elemSize; + for (u32 vi = 0; vi < elemCount && vi < fieldNode.size(); vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f = yamlToFloat(fieldNode[vi]); + memcpy(fieldData + vi * 4, &f, 4); + } else if (elemType->kind == KIND_INT && elemSize == 4) { + s32 val = fieldNode[vi].as(); + memcpy(fieldData + vi * 4, &val, 4); + } else { + fieldData[vi * elemSize] = (u8)fieldNode[vi].as(); + } + } + } + } else if (fieldNode.IsMap()) { + // Standard hkArray: keep pointer, write m_size and m_capacityAndFlags + // Pointer ($ref) is NOT modified - it stays from metadata + if (fieldNode["m_size"]) { + s32 mSize = fieldNode["m_size"].as(); + memcpy(fieldData + 8, &mSize, 4); + } + if (fieldNode["m_capacityAndFlags"]) { + s32 mCapFlags = fieldNode["m_capacityAndFlags"].as(); + memcpy(fieldData + 12, &mCapFlags, 4); + } + // Handle $ref (pointer to element data item) + if (fieldNode["$ref"] && !fieldNode["$ref"].IsNull()) { + size_t refIdx = fieldNode["$ref"].as(); + if (refIdx < mItems.size()) { + u64 ptrVal = mItems[refIdx].offset; + memcpy(fieldData, &ptrVal, 8); + } + } + } + } else { + encodeValueYaml(fieldNode, fieldData, *fEffective); + } + } + if (cur->parentTypeId != 0) cur = getType(cur->parentTypeId); + else cur = nullptr; + } +} + +void hkTagfile::applyYamlItemToData(const YAML::Node& itemNode, size_t itemIndex) { + if (itemNode.IsNull() || itemIndex >= mItems.size()) return; + + const auto& item = mItems[itemIndex]; + if (item.offset >= mDataSection.size()) return; + + const TypeDef* type = getType(item.typeIndex); + FormatKind kind = resolveEffectiveKind(item.typeIndex); + const TypeDef* effective = resolveEffectiveType(item.typeIndex); + if (!effective) effective = type; + + // Ensure data section is large enough for this item's data. + // Some items at the boundary may need a few extra bytes for type-resolved + // element sizes that slightly exceed the stored data section. + // During rebuild, cap to allocated size to prevent unintended growth. + u32 elementSize = type ? getTypeByteSize(item.typeIndex) : 0; + u32 requiredEnd = item.offset + elementSize * std::max(item.count, 1u); + if (requiredEnd > (u32)mDataSection.size()) { + // Only grow if the item is genuinely past the end (new item), + // not just because type size overestimates the stored data + if (item.offset + 1 > (u32)mDataSection.size()) { + mDataSection.resize(requiredEnd, 0); + } else { + // Clamp to available space + requiredEnd = (u32)mDataSection.size(); + } + } + + u8* itemData = mDataSection.data() + item.offset; + + bool isCharType = (type && (type->name == "tT" || type->name == "char")); + if (isCharType) elementSize = 1; + + // _bytes: direct byte overwrite + if (itemNode["_bytes"] && itemNode["_bytes"].IsSequence()) { + const YAML::Node& bytes = itemNode["_bytes"]; + for (size_t b = 0; b < bytes.size() && item.offset + b < mDataSection.size(); b++) + itemData[b] = (u8)bytes[b].as(); + return; + } + + // _string: write string bytes + null terminator + if (itemNode["_string"]) { + std::string str = itemNode["_string"].as(); + u32 maxLen = item.count > 0 ? item.count : (u32)str.size() + 1; + memset(itemData, 0, maxLen); + memcpy(itemData, str.c_str(), std::min((u32)str.size(), maxLen)); + return; + } + + // _fields: encode record fields + if (itemNode["_fields"] && type && effective && kind == KIND_RECORD) { + const TypeDef* rt = effective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (rt && !rt->fields.empty()) { + encodeRecordYaml(itemNode["_fields"], itemData, *rt); + } + return; + } + + // _elements: encode array of records/values/vectors + if (itemNode["_elements"]) { + const YAML::Node& elements = itemNode["_elements"]; + + if (type && effective && kind == KIND_RECORD) { + // Array of records + const TypeDef* rt = effective; + while (rt && rt->fields.empty() && rt->parentTypeId != 0) + rt = getType(rt->parentTypeId); + if (rt && elements.IsSequence()) { + for (u32 e = 0; e < elements.size() && e < item.count; e++) { + u8* eData = itemData + e * elementSize; + if (item.offset + (e + 1) * elementSize <= mDataSection.size()) + encodeRecordYaml(elements[e], eData, *rt); + } + } + } else if (type && effective && kind == KIND_ARRAY && + effective->subtypeId != 0 && effective->size > 0) { + // Fixed-size inline arrays (vectors) + const TypeDef* elemType = getType(effective->subtypeId); + u32 elemSize = elemType ? elemType->size : 0; + if (elemType && elemSize > 0) { + u32 vecSize = (elementSize > 0 && elementSize < effective->size) ? elementSize : effective->size; + u32 elemsPerVec = vecSize / elemSize; + if (item.count == 1 || item.flags == 0x10) { + // Single vector + if (elements.IsSequence()) { + for (u32 vi = 0; vi < elemsPerVec && vi < elements.size(); vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f = yamlToFloat(elements[vi]); + memcpy(itemData + vi * 4, &f, 4); + } else if (elemType->kind == KIND_INT && elemSize == 4) { + s32 val = elements[vi].as(); + memcpy(itemData + vi * 4, &val, 4); + } else { + itemData[vi * elemSize] = (u8)elements[vi].as(); + } + } + } + } else if (item.count > 0 && elements.IsSequence()) { + // Array of vectors + for (u32 e = 0; e < elements.size() && e < item.count; e++) { + u8* eData = itemData + e * vecSize; + if (item.offset + (e + 1) * vecSize > mDataSection.size()) break; + if (elements[e].IsSequence()) { + for (u32 vi = 0; vi < elemsPerVec && vi < elements[e].size(); vi++) { + if (elemType->kind == KIND_FLOAT && elemSize == 4) { + float f = yamlToFloat(elements[e][vi]); + memcpy(eData + vi * 4, &f, 4); + } else if (elemType->kind == KIND_INT && elemSize == 4) { + s32 val = elements[e][vi].as(); + memcpy(eData + vi * 4, &val, 4); + } else { + eData[vi * elemSize] = (u8)elements[e][vi].as(); + } + } + } + } + } + } + } else if (type && effective && + (kind == KIND_INT || kind == KIND_FLOAT || kind == KIND_BOOL)) { + // Array of scalar values + if (elements.IsSequence()) { + for (u32 e = 0; e < elements.size() && e < item.count; e++) { + u8* eData = itemData + e * elementSize; + if (item.offset + (e + 1) * elementSize <= mDataSection.size()) + encodeValueYaml(elements[e], eData, *effective); + } + } + } + return; + } + + // _value: encode single scalar + if (itemNode["_value"] && type && effective) { + encodeValueYaml(itemNode["_value"], itemData, *effective); + return; + } +} + +void hkTagfile::rebuildDataFromYaml(const YAML::Node& root) { + const YAML::Node& items = root["items"]; + if (!items || !items.IsSequence()) return; + + // ---- Full data section rebuild ---- + size_t newItemCount = items.size(); + + // Save old item offsets before modification (for pointer fixup intervals) + std::vector oldOffsets(mItems.size(), 0); + for (size_t i = 1; i < mItems.size(); i++) + oldOffsets[i] = mItems[i].offset; + + // Compute old item gap sizes (offset intervals) for layout preservation + std::vector sortedByOff(mItems.size()); + for (size_t i = 0; i < mItems.size(); i++) sortedByOff[i] = i; + std::sort(sortedByOff.begin(), sortedByOff.end(), + [&](size_t a, size_t b) { return oldOffsets[a] < oldOffsets[b]; }); + std::vector oldItemSizes(mItems.size(), 0); + for (size_t si = 0; si < sortedByOff.size(); si++) { + size_t idx = sortedByOff[si]; + if (idx == 0) continue; + // Find next non-zero offset after this item + u32 nextOff = (u32)mDataSection.size(); + for (size_t sj = si + 1; sj < sortedByOff.size(); sj++) { + size_t nidx = sortedByOff[sj]; + if (nidx > 0 && oldOffsets[nidx] > oldOffsets[idx]) { + nextOff = oldOffsets[nidx]; + break; + } + } + oldItemSizes[idx] = nextOff - oldOffsets[idx]; + } + + mItems.resize(newItemCount); + + std::vector itemSizes(newItemCount, 0); + for (size_t i = 1; i < newItemCount; i++) { + if (items[i].IsNull()) continue; + + if (items[i]["_typeIndex"]) + mItems[i].typeIndex = items[i]["_typeIndex"].as(); + if (items[i]["_flags"]) + mItems[i].flags = items[i]["_flags"].as(); + + u32 elementSize = getTypeByteSize(mItems[i].typeIndex); + const TypeDef* type = getType(mItems[i].typeIndex); + bool isCharType = (type && (type->name == "tT" || type->name == "char")); + if (isCharType) elementSize = 1; + + // Determine YAML-derived count for this item + u32 yamlCount = mItems[i].count; // default to original + bool countChanged = false; + + if (items[i]["_string"]) { + std::string str = items[i]["_string"].as(); + u32 newCount = (u32)(str.size() + 1); + countChanged = (newCount != mItems[i].count); + mItems[i].count = newCount; + itemSizes[i] = mItems[i].count; + } else if (items[i]["_bytes"] && items[i]["_bytes"].IsSequence()) { + u32 byteCount = (u32)items[i]["_bytes"].size(); + u32 newCount = elementSize > 0 ? byteCount / elementSize : byteCount; + countChanged = (newCount != mItems[i].count); + itemSizes[i] = byteCount; + mItems[i].count = newCount; + } else if (items[i]["_elements"] && items[i]["_elements"].IsSequence()) { + yamlCount = (u32)items[i]["_elements"].size(); + // Empty _elements with non-zero original count means the encoder + // couldn't decode the data — preserve original count, not 0 + if (yamlCount == 0 && mItems[i].count > 0) { + // Keep original count and size + } else { + FormatKind kind = resolveEffectiveKind(mItems[i].typeIndex); + const TypeDef* effective = resolveEffectiveType(mItems[i].typeIndex); + if (!effective) effective = type; + if (kind == KIND_ARRAY && effective && effective->subtypeId != 0 && effective->size > 0) { + u32 vecSize = (elementSize > 0 && elementSize < effective->size) ? elementSize : effective->size; + bool isSingleVec = (yamlCount > 0 && !items[i]["_elements"][0].IsSequence()); + if (isSingleVec) { + countChanged = (1 != mItems[i].count); + mItems[i].count = 1; + itemSizes[i] = vecSize; + } else { + countChanged = (yamlCount != mItems[i].count); + mItems[i].count = yamlCount; + itemSizes[i] = vecSize * yamlCount; + } + } else { + countChanged = (yamlCount != mItems[i].count); + mItems[i].count = yamlCount; + itemSizes[i] = elementSize * yamlCount; + } + } + } else if (items[i]["_fields"]) { + mItems[i].count = 1; + itemSizes[i] = elementSize; + } else if (items[i]["_value"]) { + mItems[i].count = 1; + itemSizes[i] = elementSize; + } else { + itemSizes[i] = elementSize * mItems[i].count; + } + + // If count hasn't changed and we have a known gap size from the original + // layout, use that instead — type-derived sizes can be inaccurate for + // complex types where the serialized size differs from the type definition. + if (!countChanged && i < oldItemSizes.size() && oldItemSizes[i] > 0) { + itemSizes[i] = oldItemSizes[i]; + } + } + + // Compute new offsets using incremental shift to preserve original layout. + // Items in offset-order get shifted by cumulative delta of size changes. + std::vector> offsetSorted; // (old_offset, index) + for (size_t i = 1; i < newItemCount; i++) { + if (items[i].IsNull()) continue; + u32 oldOff = (i < oldOffsets.size()) ? oldOffsets[i] : 0; + offsetSorted.push_back({oldOff, i}); + } + std::sort(offsetSorted.begin(), offsetSorted.end()); + + // Compute new offsets: keep items at their original positions. + // Only items whose content grew beyond their gap get relocated + // to overflow space at the end of the data section. + std::vector newOffsets(newItemCount, 0); + u32 overflowStart = (u32)mDataSection.size(); // append after original data + u32 overflowCount = 0; + for (auto& [oldOff, idx] : offsetSorted) { + u32 oldGapSz = (idx < oldItemSizes.size()) ? oldItemSizes[idx] : 0; + u32 newContentSz = itemSizes[idx]; + + if (newContentSz <= oldGapSz) { + // Fits in original gap - keep at original position + newOffsets[idx] = oldOff; + } else { + // Overflow - relocate to end of data section + overflowCount++; + u32 alignment = getTypeAlignment(mItems[idx].typeIndex); + if (alignment == 0) alignment = 1; + overflowStart = (overflowStart + alignment - 1) & ~(alignment - 1); + newOffsets[idx] = overflowStart; + overflowStart += newContentSz; + } + } + if (overflowCount > 0) { + // Items that grew beyond their original gap were relocated to the end + } + + u32 totalSize = overflowStart; + // Pad DATA section to at least original size to keep TYPE/INDX at same positions + if (mOrigDataPayloadSize > 0 && totalSize < mOrigDataPayloadSize) + totalSize = mOrigDataPayloadSize; + + // Build per-item old→new offset mapping for pointer fixup + struct ItemRemap { u32 oldOff; u32 newOff; u32 oldSize; }; + std::vector remaps; + std::map offsetMap; // old item start → new item start + for (size_t i = 1; i < newItemCount; i++) { + u32 oldOff = (i < oldOffsets.size()) ? oldOffsets[i] : 0; + u32 oldSz = (i < oldItemSizes.size()) ? oldItemSizes[i] : itemSizes[i]; + offsetMap[oldOff] = newOffsets[i]; + remaps.push_back({oldOff, newOffsets[i], oldSz}); + } + // Sort remaps by old offset for binary search + std::sort(remaps.begin(), remaps.end(), + [](const ItemRemap& a, const ItemRemap& b) { return a.oldOff < b.oldOff; }); + + std::vector newData(totalSize, 0); + + // Copy original data where possible (use gap sizes to preserve padding) + for (size_t i = 1; i < newItemCount; i++) { + if (items[i].IsNull()) continue; + u32 oldOff = (i < oldOffsets.size()) ? oldOffsets[i] : 0; + u32 newOff = newOffsets[i]; + // Copy the full old gap (content + padding) to preserve inter-item bytes + u32 oldGap = (i < oldItemSizes.size()) ? oldItemSizes[i] : 0; + u32 copySize = oldGap; + if (oldOff + copySize > mDataSection.size()) + copySize = (oldOff < mDataSection.size()) ? (u32)(mDataSection.size() - oldOff) : 0; + if (newOff + copySize > newData.size()) + copySize = (newOff < newData.size()) ? (u32)(newData.size() - newOff) : 0; + if (copySize > 0) { + memcpy(newData.data() + newOff, mDataSection.data() + oldOff, copySize); + } + } + + for (size_t i = 1; i < newItemCount; i++) + mItems[i].offset = newOffsets[i]; + + mDataSection = std::move(newData); + + // Apply YAML values on top + for (size_t i = 1; i < newItemCount; i++) { + if (items[i].IsNull()) continue; + applyYamlItemToData(items[i], i); + } + + // Fix up patch offsets and pointer values using sorted interval lookup + auto remapOffset = [&](u32 off) -> u32 { + // Binary search: find the last remap whose oldOff <= off + auto it = std::upper_bound(remaps.begin(), remaps.end(), off, + [](u32 val, const ItemRemap& r) { return val < r.oldOff; }); + if (it != remaps.begin()) { + --it; + if (off >= it->oldOff && off < it->oldOff + it->oldSize) + return it->newOff + (off - it->oldOff); + } + return off; // unmapped offset stays as-is + }; + + for (auto& patch : mPatches) { + for (auto& off : patch.offsets) { + u32 newPatchOff = remapOffset(off); + + // Fix pointer value at this location + if (newPatchOff + 8 <= mDataSection.size()) { + u64 ptrVal; + memcpy(&ptrVal, mDataSection.data() + newPatchOff, 8); + if (ptrVal != 0) { + auto it = offsetMap.find((u32)ptrVal); + if (it != offsetMap.end()) { + u64 newPtr = it->second; + memcpy(mDataSection.data() + newPatchOff, &newPtr, 8); + } + } + } + off = newPatchOff; + } + } + + // Pad DATA to 4-byte alignment (parser skips padding in 4-byte increments) + while (mDataSection.size() % 4 != 0) mDataSection.push_back(0); + u32 dataFlags = mDataSizeRaw & 0xC0000000; + mDataSizeRaw = ((u32)mDataSection.size() + 8) | dataFlags; + +} + +// ============================================================================ +// YAML Deserialization +// ============================================================================ + +bool hkTagfile::fromYaml(const YAML::Node& root) { + mSdkVersion = root["sdk_version"].as(); + + // Restore from metadata section + const YAML::Node& meta = root["_metadata"]; + mSdkvSizeRaw = meta["sdkv_size_raw"].as(); + mDataSizeRaw = meta["data_size_raw"].as(); + mIndxTagOffset = meta["indx_tag_offset"].as(); + mOrigDataPayloadSize = meta["orig_data_payload_size"] + ? meta["orig_data_payload_size"].as() : 0; + + // Restore TYPE section from base64 + mTypeSection = base64Decode(meta["type_section"].as()); + parseTypeSection(); + + // Restore DATA section from base64 + mDataSection = base64Decode(meta["data_section"].as()); + + // Restore items from layout metadata + const YAML::Node& layout = meta["item_layout"]; + mItems.resize(layout.size()); + for (size_t i = 0; i < layout.size(); i++) { + mItems[i].typeIndex = layout[i]["typeIndex"].as(); + mItems[i].flags = layout[i]["flags"].as(); + mItems[i].offset = layout[i]["offset"].as(); + mItems[i].count = layout[i]["count"].as(); + } + + // Restore patches + mPatches.clear(); + if (root["patches"]) { + for (const auto& pn : root["patches"]) { + PatchEntry patch; + patch.typeIndex = pn["typeIndex"].as(); + for (const auto& off : pn["offsets"]) + patch.offsets.push_back(off.as()); + mPatches.push_back(std::move(patch)); + } + } + + // Apply YAML item values back to data section (supports editing and expansion) + rebuildDataFromYaml(root); + + return true; +} + +// ============================================================================ +// Binary Writing +// ============================================================================ + +void hkTagfile::writeToBinary(std::vector& out) const { + out.clear(); + out.reserve(256 * 1024); + + auto writeU32BE = [&](u32 pos, u32 val) { + out[pos] = (u8)(val >> 24); + out[pos+1] = (u8)(val >> 16); + out[pos+2] = (u8)(val >> 8); + out[pos+3] = (u8)val; + }; + + auto appendU32BE = [&](u32 val) { + out.push_back((u8)(val >> 24)); + out.push_back((u8)(val >> 16)); + out.push_back((u8)(val >> 8)); + out.push_back((u8)val); + }; + + auto appendStr = [&](const char* str, size_t len) { + for (size_t i = 0; i < len; i++) out.push_back(str[i]); + }; + + auto appendAlign = [&](size_t alignment) { + while (out.size() % alignment != 0) out.push_back(0); + }; + + // TAG0 header placeholder + size_t tag0Start = out.size(); + appendU32BE(0); // placeholder for TAG0 size + appendStr("TAG0", 4); + + // SDKV section (preserve original size+flags) + appendU32BE(mSdkvSizeRaw); + appendStr("SDKV", 4); + for (char c : mSdkVersion) out.push_back(c); + // Pad SDKV content to match original section size + u32 sdkvContentSize = (mSdkvSizeRaw & 0x3FFFFFFF) - 8; + while (out.size() < tag0Start + 8 + (mSdkvSizeRaw & 0x3FFFFFFF)) + out.push_back(0); + + // DATA section (preserve original size+flags format) + // Pad DATA content to 4-byte alignment so subsequent sections stay aligned + // (the parser skips zero-padding in 4-byte increments between sections) + size_t dataContentSize = mDataSection.size(); + size_t dataContentAligned = (dataContentSize + 3) & ~(size_t)3; + u32 dataSizeWithHeader = (u32)(dataContentAligned + 8); + u32 dataFlags = mDataSizeRaw & 0xC0000000; + if (mDataSizeRaw == 0) dataFlags = 0x40000000; + appendU32BE(dataSizeWithHeader | dataFlags); + appendStr("DATA", 4); + out.insert(out.end(), mDataSection.begin(), mDataSection.end()); + // Pad DATA content to alignment + while (out.size() < tag0Start + 8 + (mSdkvSizeRaw & 0x3FFFFFFF) + dataSizeWithHeader) + out.push_back(0); + + // TYPE section (verbatim - includes internal TPAD for alignment) + out.insert(out.end(), mTypeSection.begin(), mTypeSection.end()); + + // Pad to reach original INDX offset if known + if (mIndxTagOffset > 0) { + size_t targetPos = tag0Start + mIndxTagOffset; + while (out.size() < targetPos) out.push_back(0); + } + + // INDX section (immediately follows TYPE + padding) + size_t indxStart = out.size(); + appendU32BE(0); // placeholder for INDX size + appendStr("INDX", 4); + + // ITEM subsection + size_t itemStart = out.size(); + appendU32BE(0); // placeholder for ITEM size + appendStr("ITEM", 4); + + for (const auto& item : mItems) { + // typeIndex (u16) + pad (u8) + flags (u8) + out.push_back((u8)(item.typeIndex & 0xFF)); + out.push_back((u8)(item.typeIndex >> 8)); + out.push_back(0); // pad + out.push_back(item.flags); + // dataOffset (u32 LE) + u32 off = item.offset; + out.push_back((u8)(off)); out.push_back((u8)(off >> 8)); + out.push_back((u8)(off >> 16)); out.push_back((u8)(off >> 24)); + // count (u32 LE) + u32 cnt = item.count; + out.push_back((u8)(cnt)); out.push_back((u8)(cnt >> 8)); + out.push_back((u8)(cnt >> 16)); out.push_back((u8)(cnt >> 24)); + } + + // Write ITEM size (includes size+magic, with flag bit) + size_t itemSize = out.size() - itemStart; + writeU32BE((u32)itemStart, (u32)itemSize | 0x40000000); + + // PTCH subsection + size_t ptchStart = out.size(); + appendU32BE(0); // placeholder for PTCH size + appendStr("PTCH", 4); + + for (const auto& patch : mPatches) { + u32 ti = patch.typeIndex; + out.push_back((u8)(ti)); out.push_back((u8)(ti >> 8)); + out.push_back((u8)(ti >> 16)); out.push_back((u8)(ti >> 24)); + u32 cnt = (u32)patch.offsets.size(); + out.push_back((u8)(cnt)); out.push_back((u8)(cnt >> 8)); + out.push_back((u8)(cnt >> 16)); out.push_back((u8)(cnt >> 24)); + for (u32 off : patch.offsets) { + out.push_back((u8)(off)); out.push_back((u8)(off >> 8)); + out.push_back((u8)(off >> 16)); out.push_back((u8)(off >> 24)); + } + } + + size_t ptchSize = out.size() - ptchStart; + writeU32BE((u32)ptchStart, (u32)ptchSize | 0x40000000); + + // Write INDX size + size_t indxSize = out.size() - indxStart; + writeU32BE((u32)indxStart, (u32)indxSize); + + // Write TAG0 size (no trailing alignment - INDX ends the TAG file) + size_t tag0Size = out.size() - tag0Start; + writeU32BE((u32)tag0Start, (u32)tag0Size); +} + +} // namespace Havok diff --git a/source/havok/hkTagfile.hpp b/source/havok/hkTagfile.hpp new file mode 100644 index 0000000..ce73840 --- /dev/null +++ b/source/havok/hkTagfile.hpp @@ -0,0 +1,253 @@ +#pragma once + +#include "types.h" +#include "Json.hpp" +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +namespace Havok { + +// Havok VLE (Variable Length Encoding) - Dlugosz format +struct VleReader { + const u8* mData; + size_t mPos; + size_t mSize; + + VleReader(const u8* data, size_t size) : mData(data), mPos(0), mSize(size) {} + + bool hasMore() const { return mPos < mSize; } + + u8 readByte() { + if (mPos >= mSize) return 0; + return mData[mPos++]; + } + + u64 readVle() { + u8 b0 = readByte(); + if ((b0 & 0x80) == 0) return b0; + if ((b0 & 0xC0) == 0x80) { + u8 b1 = readByte(); + return ((u64)(b0 & 0x3F) << 8) | b1; + } + if ((b0 & 0xE0) == 0xC0) { + u8 b1 = readByte(), b2 = readByte(); + return ((u64)(b0 & 0x1F) << 16) | ((u64)b1 << 8) | b2; + } + if ((b0 & 0xF0) == 0xE0) { + if ((b0 & 0xF8) == 0xE0) { + u8 b1 = readByte(), b2 = readByte(), b3 = readByte(); + return ((u64)(b0 & 0x0F) << 24) | ((u64)b1 << 16) | ((u64)b2 << 8) | b3; + } + if ((b0 & 0xF8) == 0xE8) { + u8 b1 = readByte(), b2 = readByte(), b3 = readByte(), b4 = readByte(); + return ((u64)(b0 & 0x07) << 32) | ((u64)b1 << 24) | ((u64)b2 << 16) | ((u64)b3 << 8) | b4; + } + if ((b0 & 0xFE) == 0xF0) { + u64 val = 0; + for (int i = 0; i < 7; i++) val = (val << 8) | readByte(); + return ((u64)(b0 & 0x01) << 56) | val; + } + } + if (b0 == 0xF8) { + u64 val = 0; + for (int i = 0; i < 5; i++) val = (val << 8) | readByte(); + return val; + } + if (b0 == 0xF9) { + u64 val = 0; + for (int i = 0; i < 8; i++) val = (val << 8) | readByte(); + return val; + } + return 0; + } +}; + +struct VleWriter { + std::vector& mData; + + VleWriter(std::vector& data) : mData(data) {} + + void writeVle(u64 val) { + if (val < 0x80) { + mData.push_back((u8)val); + } else if (val < 0x4000) { + mData.push_back((u8)(0x80 | (val >> 8))); + mData.push_back((u8)(val & 0xFF)); + } else if (val < 0x200000) { + mData.push_back((u8)(0xC0 | (val >> 16))); + mData.push_back((u8)((val >> 8) & 0xFF)); + mData.push_back((u8)(val & 0xFF)); + } else if (val < 0x10000000) { + mData.push_back((u8)(0xE0 | (val >> 24))); + mData.push_back((u8)((val >> 16) & 0xFF)); + mData.push_back((u8)((val >> 8) & 0xFF)); + mData.push_back((u8)(val & 0xFF)); + } else if (val < 0x800000000ULL) { + mData.push_back((u8)(0xE8 | (val >> 32))); + mData.push_back((u8)((val >> 24) & 0xFF)); + mData.push_back((u8)((val >> 16) & 0xFF)); + mData.push_back((u8)((val >> 8) & 0xFF)); + mData.push_back((u8)(val & 0xFF)); + } else if (val < 0x2000000000000ULL) { + mData.push_back((u8)(0xF0 | (val >> 56))); + for (int i = 6; i >= 0; i--) + mData.push_back((u8)((val >> (i * 8)) & 0xFF)); + } else if (val < 0x10000000000ULL) { + mData.push_back(0xF8); + for (int i = 4; i >= 0; i--) + mData.push_back((u8)((val >> (i * 8)) & 0xFF)); + } else { + mData.push_back(0xF9); + for (int i = 7; i >= 0; i--) + mData.push_back((u8)((val >> (i * 8)) & 0xFF)); + } + } +}; + +// Format kind constants +enum FormatKind : u32 { + KIND_VOID = 0, + KIND_OPAQUE = 1, + KIND_BOOL = 2, + KIND_STRING = 3, + KIND_INT = 4, + KIND_FLOAT = 5, + KIND_POINTER = 6, + KIND_RECORD = 7, + KIND_ARRAY = 8, + KIND_MASK = 0x1F, +}; + +// TBDY opt bits +enum OptBits : u32 { + OPT_FORMAT = 1 << 0, + OPT_SUBTYPE = 1 << 1, + OPT_VERSION = 1 << 2, + OPT_SIZE_ALIGN = 1 << 3, + OPT_FLAGS = 1 << 4, + OPT_DECLS = 1 << 5, + OPT_INTERFACES = 1 << 6, + OPT_ATTRIBUTE_STRING = 1 << 7, + OPT_MUTABLE = 1 << 8, +}; + +// INT format encoding +static const u32 INT_BIG_ENDIAN_BIT = 1 << 8; +static const u32 INT_SIGNED_BIT = 1 << 9; +static const u32 INT_NUM_BITS_SHIFT = 10; + +struct TypeField { + std::string name; + u32 nameStringId = 0; + u32 flags = 0; + u32 offset = 0; + u32 typeId = 0; +}; + +struct TypeInterface { + u32 typeId = 0; + u32 offset = 0; +}; + +struct TypeDef { + u32 typeId = 0; + u32 parentTypeId = 0; + std::string name; + u32 format = 0; + FormatKind kind = KIND_VOID; + u32 subtypeId = 0; + u32 version = 0; + u32 size = 0; + u32 align = 1; + u32 flags = 0; + std::vector fields; + std::vector interfaces; + u32 optbits = 0; +}; + +struct TagfileItem { + u16 typeIndex = 0; + u8 flags = 0; + u32 offset = 0; + u32 count = 0; +}; + +struct PatchEntry { + u32 typeIndex = 0; + std::vector offsets; +}; + +class hkTagfile { +public: + std::string mSdkVersion; + std::vector mDataSection; + std::vector mTypeSection; + std::vector mItems; + std::vector mPatches; + std::vector mTypes; + + // String tables from TYPE section + std::vector mTypeStrings; + std::vector mFieldStrings; + std::vector mTypeNames; + + // Structural metadata for round-tripping + u32 mSdkvSizeRaw = 0x40000010; // Original SDKV section size+flags + u32 mDataSizeRaw = 0; // Original DATA section size+flags + u32 mIndxTagOffset = 0; // INDX start offset within TAG file (for padding) + u32 mOrigDataPayloadSize = 0; // Original DATA payload size (for rebuild padding) + + bool loadFromBinary(const u8* data, size_t size); + void writeToBinary(std::vector& out) const; + + json toJson() const; + bool fromJson(const json& j); + + YAML::Node toYaml() const; + bool fromYaml(const YAML::Node& root); + +private: + void parseTypeSection(); + void parseTbdy(const u8* tbdyData, size_t tbdySize); + void resolveTypeNames(); + + json itemToJson(size_t itemIndex) const; + json decodeRecord(const u8* data, const TypeDef& type) const; + json decodeValue(const u8* data, const TypeDef& type) const; + json decodeArray(size_t itemIndex) const; + + YAML::Node itemToYaml(size_t itemIndex) const; + YAML::Node decodeRecordYaml(const u8* data, const TypeDef& type) const; + YAML::Node decodeValueYaml(const u8* data, const TypeDef& type) const; + + void encodeValueYaml(const YAML::Node& node, u8* dst, const TypeDef& type) const; + void encodeRecordYaml(const YAML::Node& node, u8* dst, const TypeDef& type) const; + void applyYamlItemToData(const YAML::Node& itemNode, size_t itemIndex); + void rebuildDataFromYaml(const YAML::Node& root); + + void itemFromJson(const json& j, size_t itemIndex, std::vector& dataOut) const; + void encodeRecord(u8* data, const TypeDef& type, const json& j) const; + void encodeValue(u8* data, const TypeDef& type, const json& j) const; + + // Helpers + const TypeDef* getType(u32 typeId) const; + u32 getTypeByteSize(u32 typeId) const; + u32 getTypeAlignment(u32 typeId) const; + bool isPointerLikeType(u32 typeId) const; + FormatKind resolveEffectiveKind(u32 typeId) const; + const TypeDef* resolveEffectiveType(u32 typeId) const; + + static u32 readBE32(const u8* p) { + return ((u32)p[0] << 24) | ((u32)p[1] << 16) | ((u32)p[2] << 8) | p[3]; + } + static void writeBE32(u8* p, u32 v) { + p[0] = (u8)(v >> 24); p[1] = (u8)(v >> 16); p[2] = (u8)(v >> 8); p[3] = (u8)v; + } +}; + +} // namespace Havok diff --git a/source/main.cpp b/source/main.cpp index 0cfe0ac..3c3513f 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -4,6 +4,10 @@ #include #include #include "fivex/CollisionInfo.hpp" +#include "fivex/ClothInfo.hpp" +#include "fivex/NavMeshInfo.hpp" +#include "fivex/NvtInfo.hpp" +#include "fivex/AampFile.hpp" using json = nlohmann::json; @@ -65,6 +69,190 @@ bool convertPhiveToObj(const std::string& phivePath, const std::string& outDir) return true; } +bool convertBphclToYaml(const std::string& bphclPath, const std::string& outDir) { + std::ifstream inFile(bphclPath, std::ios::binary | std::ios::ate); + if (!inFile) { std::cerr << "Failed to open bphcl file: " << bphclPath << "\n"; return false; } + std::streamsize size = inFile.tellg(); + inFile.seekg(0, std::ios::beg); + std::vector data((size_t)size); + if (!inFile.read(reinterpret_cast(data.data()), size)) { + std::cerr << "Failed to read bphcl file: " << bphclPath << "\n"; return false; + } + + FiveX::ClothInfo info; + info.loadFromBphcl(data); + + std::vector tagYaml, aampYaml; + info.serializeToYaml(tagYaml, aampYaml); + + std::filesystem::path outPath(outDir); + std::string outName = std::filesystem::path(bphclPath).stem().string(); + std::string tagPath = (outPath / (outName + ".bphcl.yaml")).string(); + std::string aampPath = (outPath / (outName + ".bphcl.aamp.yaml")).string(); + + std::ofstream tagFile(tagPath, std::ios::binary); + if (!tagFile) { std::cerr << "Failed to create yaml file: " << tagPath << "\n"; return false; } + tagFile.write(reinterpret_cast(tagYaml.data()), tagYaml.size()); + + if (!aampYaml.empty()) { + std::ofstream aampFile(aampPath, std::ios::binary); + if (!aampFile) { std::cerr << "Failed to create aamp yaml: " << aampPath << "\n"; return false; } + aampFile.write(reinterpret_cast(aampYaml.data()), aampYaml.size()); + std::cout << "Saved bphcl yaml to " << std::filesystem::absolute(tagPath) + << " + " << std::filesystem::absolute(aampPath) << "\n"; + } else { + std::cout << "Saved bphcl yaml to " << std::filesystem::absolute(tagPath) << "\n"; + } + return true; +} + +bool convertYamlToBphcl(const std::string& yamlPath, const std::string& outPath) { + std::ifstream tagFile(yamlPath); + if (!tagFile) { std::cerr << "Failed to open yaml file: " << yamlPath << "\n"; return false; } + std::string tagStr((std::istreambuf_iterator(tagFile)), std::istreambuf_iterator()); + std::vector tagYaml(tagStr.begin(), tagStr.end()); + + // Try to load companion AAMP yaml + std::string aampPath = yamlPath.substr(0, yamlPath.size() - 5) + ".aamp.yaml"; + std::vector aampYaml; + std::ifstream aampFile(aampPath); + if (aampFile) { + std::string aampStr((std::istreambuf_iterator(aampFile)), std::istreambuf_iterator()); + aampYaml.assign(aampStr.begin(), aampStr.end()); + } + + FiveX::ClothInfo info; + info.loadFromYaml(tagYaml, aampYaml); + + std::vector outData; + info.serializeToBphcl(outData); + + std::ofstream outFile(outPath, std::ios::binary); + if (!outFile) { std::cerr << "Failed to create output file: " << outPath << "\n"; return false; } + outFile.write(reinterpret_cast(outData.data()), outData.size()); + + std::cout << "Saved bphcl to " << std::filesystem::absolute(outPath) << "\n"; + return true; +} + +bool convertBphnmToYaml(const std::string& bphnmPath, const std::string& outDir) { + std::ifstream inFile(bphnmPath, std::ios::binary | std::ios::ate); + if (!inFile) { std::cerr << "Failed to open bphnm file: " << bphnmPath << "\n"; return false; } + std::streamsize size = inFile.tellg(); + inFile.seekg(0, std::ios::beg); + std::vector data((size_t)size); + if (!inFile.read(reinterpret_cast(data.data()), size)) { + std::cerr << "Failed to read bphnm file: " << bphnmPath << "\n"; return false; + } + + FiveX::NavMeshInfo info; + info.loadFromBphnm(data); + + std::vector yamlOut; + info.serializeToYaml(yamlOut); + + std::filesystem::path outPath(outDir); + std::string outName = std::filesystem::path(bphnmPath).stem().string(); + std::string yamlPath = (outPath / (outName + ".bphnm.yaml")).string(); + + std::ofstream yamlFile(yamlPath, std::ios::binary); + if (!yamlFile) { std::cerr << "Failed to create yaml file: " << yamlPath << "\n"; return false; } + yamlFile.write(reinterpret_cast(yamlOut.data()), yamlOut.size()); + + std::cout << "Saved bphnm yaml to " << std::filesystem::absolute(yamlPath) << "\n"; + + // Also export navmesh as OBJ for visualization + std::vector objOut; + if (info.serializeToObj(objOut)) { + std::string objPath = (outPath / (outName + ".bphnm.obj")).string(); + std::ofstream objFile(objPath, std::ios::binary); + if (objFile) { + objFile.write(reinterpret_cast(objOut.data()), objOut.size()); + std::cout << "Saved navmesh obj to " << std::filesystem::absolute(objPath) << "\n"; + } + } + + return true; +} + +bool convertYamlToBphnm(const std::string& yamlPath, const std::string& outPath) { + std::ifstream yamlFile(yamlPath); + if (!yamlFile) { std::cerr << "Failed to open yaml file: " << yamlPath << "\n"; return false; } + std::string yamlStr((std::istreambuf_iterator(yamlFile)), std::istreambuf_iterator()); + std::vector yamlData(yamlStr.begin(), yamlStr.end()); + + FiveX::NavMeshInfo info; + info.loadFromYaml(yamlData); + + std::vector outData; + info.serializeToBphnm(outData); + + std::ofstream outFile(outPath, std::ios::binary); + if (!outFile) { std::cerr << "Failed to create output file: " << outPath << "\n"; return false; } + outFile.write(reinterpret_cast(outData.data()), outData.size()); + + std::cout << "Saved bphnm to " << std::filesystem::absolute(outPath) << "\n"; + return true; +} + +bool convertNvtToYaml(const std::string& nvtPath, const std::string& outDir) { + std::ifstream inFile(nvtPath, std::ios::binary | std::ios::ate); + if (!inFile) { std::cerr << "Failed to open nvt file: " << nvtPath << "\n"; return false; } + std::streamsize size = inFile.tellg(); + inFile.seekg(0, std::ios::beg); + std::vector data((size_t)size); + if (!inFile.read(reinterpret_cast(data.data()), size)) { + std::cerr << "Failed to read nvt file: " << nvtPath << "\n"; return false; + } + + FiveX::NvtInfo info; + info.loadFromNvt(data); + + std::vector yamlOut; + info.serializeToYaml(yamlOut); + + std::filesystem::path outPath(outDir); + std::string outName = std::filesystem::path(nvtPath).stem().string(); + std::string yamlPath = (outPath / (outName + ".nvt.yaml")).string(); + + std::ofstream yamlFile(yamlPath, std::ios::binary); + if (!yamlFile) { std::cerr << "Failed to create yaml file: " << yamlPath << "\n"; return false; } + yamlFile.write(reinterpret_cast(yamlOut.data()), yamlOut.size()); + std::cout << "Saved nvt yaml to " << std::filesystem::absolute(yamlPath) << "\n"; + + // Also export OBJ with path waypoints + std::vector objOut; + info.serializeToObj(objOut); + std::string objPath = (outPath / (outName + ".nvt.obj")).string(); + std::ofstream objFile(objPath, std::ios::binary); + if (objFile) { + objFile.write(reinterpret_cast(objOut.data()), objOut.size()); + std::cout << "Saved nvt obj to " << std::filesystem::absolute(objPath) << "\n"; + } + + return true; +} + +bool convertYamlToNvt(const std::string& yamlPath, const std::string& outPath) { + std::ifstream yamlFile(yamlPath); + if (!yamlFile) { std::cerr << "Failed to open yaml file: " << yamlPath << "\n"; return false; } + std::string yamlStr((std::istreambuf_iterator(yamlFile)), std::istreambuf_iterator()); + std::vector yamlData(yamlStr.begin(), yamlStr.end()); + + FiveX::NvtInfo info; + info.loadFromYaml(yamlData); + + std::vector outData; + info.serializeToNvt(outData); + + std::ofstream outFile(outPath, std::ios::binary); + if (!outFile) { std::cerr << "Failed to create output file: " << outPath << "\n"; return false; } + outFile.write(reinterpret_cast(outData.data()), outData.size()); + + std::cout << "Saved nvt to " << std::filesystem::absolute(outPath) << "\n"; + return true; +} + bool convertObjToPhive(const std::string& objPath, const std::string& matInfoPath, const std::string& outPath) { std::ifstream objFile(objPath); if (!objFile) { std::cerr << "Failed to open obj file: " << objPath << "\n"; return false; } @@ -94,16 +282,143 @@ bool convertObjToPhive(const std::string& objPath, const std::string& matInfoPat } int main(int argc, const char** argv) { + // Auto-load hash dictionary from next to executable + { + std::filesystem::path exePath = std::filesystem::canonical("/proc/self/exe"); + std::filesystem::path dictPath = exePath.parent_path() / "aamp_hashes.txt"; + if (std::filesystem::exists(dictPath)) { + FiveX::AampFile::loadHashDictionary(dictPath.string()); + } + } + if (argc < 2 || std::string(argv[1]) == "-h" || std::string(argv[1]) == "help") { std::cout << "Usage:\n" - << " PhiveConverter.exe FILE.bphsh - convert bphsh to obj+json\n" - << " PhiveConverter.exe FILE.obj FILE.json - convert obj+json to bphsh\n" - << " PhiveConverter.exe FILE.obj - convert obj to bphsh (no mat info)\n" - << " PhiveConverter.exe -p PHIVE -o OUTDIR - phive to obj\n" - << " PhiveConverter.exe -obj OBJ -mat MAT -o OUT - obj to phive\n"; + << " PhiveConverter FILE.bphsh - convert bphsh to obj+json\n" + << " PhiveConverter FILE.obj FILE.json - convert obj+json to bphsh\n" + << " PhiveConverter FILE.obj - convert obj to bphsh (no mat info)\n" + << " PhiveConverter FILE.bphcl - convert bphcl to yaml\n" + << " PhiveConverter FILE.bphcl.yaml - convert yaml to bphcl\n" + << " PhiveConverter FILE.bphnm - convert bphnm to yaml+obj\n" + << " PhiveConverter FILE.bphnm.yaml - convert yaml to bphnm\n" + << " PhiveConverter FILE.bphnm.obj - convert obj+yaml to bphnm (vertex edit)\n" + << " PhiveConverter FILE.nvt - convert nvt to yaml+obj\n" + << " PhiveConverter FILE.nvt.yaml - convert yaml to nvt\n" + << " PhiveConverter FILE.nvt.obj - convert obj+yaml to nvt (waypoint edit)\n" + << " PhiveConverter -p PHIVE -o OUTDIR - phive to obj\n" + << " PhiveConverter -obj OBJ -mat MAT -o OUT - obj to phive\n"; return 0; } + // Simple file detection for new formats + std::string firstArg = argv[1]; + if (firstArg[0] != '-') { + // YAML format (primary) + if (endsWith(firstArg, ".bphcl")) { + std::string outDir = std::filesystem::path(firstArg).parent_path().string(); + if (outDir.empty()) outDir = "."; + return convertBphclToYaml(firstArg, outDir) ? 0 : 1; + } + if (endsWith(firstArg, ".bphcl.yaml")) { + std::string outPath = firstArg.substr(0, firstArg.size() - 5); // remove ".yaml" + return convertYamlToBphcl(firstArg, outPath) ? 0 : 1; + } + if (endsWith(firstArg, ".bphnm")) { + std::string outDir = std::filesystem::path(firstArg).parent_path().string(); + if (outDir.empty()) outDir = "."; + return convertBphnmToYaml(firstArg, outDir) ? 0 : 1; + } + if (endsWith(firstArg, ".bphnm.yaml")) { + std::string outPath = firstArg.substr(0, firstArg.size() - 5); // remove ".yaml" + return convertYamlToBphnm(firstArg, outPath) ? 0 : 1; + } + if (endsWith(firstArg, ".bphnm.obj")) { + // OBJ import: look for companion YAML, load it, then patch vertices from OBJ + std::string yamlPath = firstArg.substr(0, firstArg.size() - 4) + ".yaml"; // .obj -> .yaml + std::string outPath = firstArg.substr(0, firstArg.size() - 4); // remove ".obj" + if (!std::filesystem::exists(yamlPath)) { + std::cerr << "Companion YAML file required: " << yamlPath << "\n"; + return 1; + } + // Load YAML first to restore all binary data + std::ifstream yamlFile(yamlPath); + if (!yamlFile) { std::cerr << "Failed to open yaml file: " << yamlPath << "\n"; return 1; } + std::string yamlStr((std::istreambuf_iterator(yamlFile)), std::istreambuf_iterator()); + std::vector yamlData(yamlStr.begin(), yamlStr.end()); + + FiveX::NavMeshInfo info; + info.loadFromYaml(yamlData); + + // Load OBJ and patch vertex positions + std::ifstream objFile(firstArg, std::ios::binary | std::ios::ate); + if (!objFile) { std::cerr << "Failed to open OBJ file: " << firstArg << "\n"; return 1; } + std::streamsize objSize = objFile.tellg(); + objFile.seekg(0, std::ios::beg); + std::vector objData((size_t)objSize); + objFile.read(reinterpret_cast(objData.data()), objSize); + + if (!info.loadFromObj(objData)) { + std::cerr << "Failed to apply OBJ vertices to navmesh\n"; + return 1; + } + + // Serialize to bphnm + std::vector outData; + info.serializeToBphnm(outData); + + std::ofstream outFile(outPath, std::ios::binary); + if (!outFile) { std::cerr << "Failed to create output: " << outPath << "\n"; return 1; } + outFile.write(reinterpret_cast(outData.data()), outData.size()); + std::cout << "Saved bphnm to " << std::filesystem::absolute(outPath) << "\n"; + return 0; + } + if (endsWith(firstArg, ".nvt")) { + std::string outDir = std::filesystem::path(firstArg).parent_path().string(); + if (outDir.empty()) outDir = "."; + return convertNvtToYaml(firstArg, outDir) ? 0 : 1; + } + if (endsWith(firstArg, ".nvt.yaml")) { + std::string outPath = firstArg.substr(0, firstArg.size() - 5); // remove ".yaml" + return convertYamlToNvt(firstArg, outPath) ? 0 : 1; + } + if (endsWith(firstArg, ".nvt.obj")) { + // OBJ import: look for companion YAML, load it, then patch waypoints from OBJ + std::string yamlPath = firstArg.substr(0, firstArg.size() - 4) + ".yaml"; // .obj -> .yaml + std::string outPath = firstArg.substr(0, firstArg.size() - 4); // remove ".obj" + if (!std::filesystem::exists(yamlPath)) { + std::cerr << "Companion YAML file required: " << yamlPath << "\n"; + return 1; + } + std::ifstream yamlFile(yamlPath); + if (!yamlFile) { std::cerr << "Failed to open yaml file: " << yamlPath << "\n"; return 1; } + std::string yamlStr((std::istreambuf_iterator(yamlFile)), std::istreambuf_iterator()); + std::vector yamlData(yamlStr.begin(), yamlStr.end()); + + FiveX::NvtInfo info; + info.loadFromYaml(yamlData); + + std::ifstream objFile(firstArg, std::ios::binary | std::ios::ate); + if (!objFile) { std::cerr << "Failed to open OBJ file: " << firstArg << "\n"; return 1; } + std::streamsize objSize = objFile.tellg(); + objFile.seekg(0, std::ios::beg); + std::vector objData((size_t)objSize); + objFile.read(reinterpret_cast(objData.data()), objSize); + + if (!info.loadFromObj(objData)) { + std::cerr << "Failed to apply OBJ waypoints to navtable\n"; + return 1; + } + + std::vector outData; + info.serializeToNvt(outData); + + std::ofstream outFile(outPath, std::ios::binary); + if (!outFile) { std::cerr << "Failed to create output: " << outPath << "\n"; return 1; } + outFile.write(reinterpret_cast(outData.data()), outData.size()); + std::cout << "Saved nvt to " << std::filesystem::absolute(outPath) << "\n"; + return 0; + } + } + if (!loadPhiveConfig()) { std::cerr << "Failed to load phive config!\n"; return 1;