diff --git a/src/OpenAnimationReplacer.cpp b/src/OpenAnimationReplacer.cpp index a477e6d..ffaf941 100644 --- a/src/OpenAnimationReplacer.cpp +++ b/src/OpenAnimationReplacer.cpp @@ -525,7 +525,7 @@ void OpenAnimationReplacer::CreateReplacerMods() auto endOfParsingTime = std::chrono::high_resolution_clock::now(); - if (parseResults.modParseResults.empty() && parseResults.legacyParseResults.empty()) { + if (parseResults.modParseResultFutures.empty() && parseResults.legacyParseResultFutures.empty()) { logger::info("No replacer mods found."); Parsing::LogTimingStats(); return; @@ -533,7 +533,8 @@ void OpenAnimationReplacer::CreateReplacerMods() // add all parsed mods logger::info("Adding parsed replacer mods..."); - for (auto& modParseResult : parseResults.modParseResults) { + for (auto& future : parseResults.modParseResultFutures) { + auto modParseResult = future.get(); AddModParseResult(modParseResult); } logger::info("Added parsed replacer mods."); @@ -542,8 +543,8 @@ void OpenAnimationReplacer::CreateReplacerMods() // add all parsed legacy mods logger::info("Adding parsed legacy replacer mods..."); - for (auto& subModParseResult : parseResults.legacyParseResults) { - if (subModParseResult.bSuccess) { + for (auto& future : parseResults.legacyParseResultFutures) { + if (auto subModParseResult = future.get(); subModParseResult.bSuccess) { auto replacerMod = GetOrCreateLegacyReplacerMod(); AddSubModParseResult(replacerMod, subModParseResult); } diff --git a/src/PCH.h b/src/PCH.h index aadc76d..e3c7075 100644 --- a/src/PCH.h +++ b/src/PCH.h @@ -4,7 +4,7 @@ #define DIRECTINPUT_VERSION 0x0800 #define IMGUI_DEFINE_MATH_OPERATORS -#define USE_BS_LOCKS +//#define USE_BS_LOCKS CommonlibSSE constantly spams relocations for each lock. Expensive. #pragma warning(push) #include diff --git a/src/Parsing.cpp b/src/Parsing.cpp index 94fe2e2..df0d350 100644 --- a/src/Parsing.cpp +++ b/src/Parsing.cpp @@ -3,16 +3,42 @@ #include #include #include -#include #include #include +#include #include #include +#include "AnimationFileHashCache.h" #include "OpenAnimationReplacer.h" #include "Settings.h" +#ifdef _WIN32 +constexpr ULONG ThreadIoPriority = 19; +constexpr ULONG LowThreadIoPriority = 0; +constexpr ULONG HighThreadIoPriority = 3; +using NtSetInformationThreadFn = NTSTATUS(NTAPI*)(HANDLE, ULONG, PVOID, ULONG); + +static std::atomic g_directoryCacheThreadId{ 0 }; + +static void SetIOPriority(HANDLE a_thread, ULONG a_priority) +{ + if (const HMODULE ntdll = GetModuleHandleW(L"ntdll.dll")) { + if (const auto ntSetInformationThread = reinterpret_cast(GetProcAddress(ntdll, "NtSetInformationThread"))) { + ntSetInformationThread(a_thread, ThreadIoPriority, &a_priority, sizeof(a_priority)); + } + } +} + +static void SetLowIOPriority() +{ + SetIOPriority(GetCurrentThread(), LowThreadIoPriority); +} +#endif + +static std::atomic g_precachedHashCount{ 0 }; + namespace Parsing { namespace @@ -20,23 +46,9 @@ namespace Parsing constexpr auto timingBucketCount = static_cast(TimingBucket::kTotal); constexpr auto timingCounterCount = static_cast(TimingCounter::kTotal); - enum class WallTimingBucket : uint8_t - { - kDiscovery, - kExecuteParseJobs, - kOARModJobs, - kLegacyCustomConditionJobs, - kLegacyPluginJobs, - kTotal - }; - - constexpr auto wallTimingBucketCount = static_cast(WallTimingBucket::kTotal); - std::array, timingBucketCount> timingNanoseconds; std::array, timingBucketCount> timingCounts; std::array, timingCounterCount> timingCounters; - std::array wallTimingNanoseconds; - std::array wallTimingCounts; constexpr std::string_view GetTimingBucketName(TimingBucket a_bucket) { @@ -64,59 +76,6 @@ namespace Parsing { return std::chrono::duration(std::chrono::nanoseconds(a_nanoseconds)).count(); } - - constexpr std::string_view GetWallTimingBucketName(WallTimingBucket a_bucket) - { - switch (a_bucket) { - case WallTimingBucket::kDiscovery: - return "Discovery"; - case WallTimingBucket::kExecuteParseJobs: - return "Execute parse jobs"; - case WallTimingBucket::kOARModJobs: - return "Waiting for OAR mod jobs"; - case WallTimingBucket::kLegacyCustomConditionJobs: - return "Waiting for DAR _CustomConditions jobs"; - case WallTimingBucket::kLegacyPluginJobs: - return "Waiting for DAR plugin jobs"; - default: - return "Unknown"; - } - } - - void AddWallTiming(WallTimingBucket a_bucket, std::chrono::nanoseconds a_duration) - { - if constexpr (bEnableParseTiming) { - const auto index = static_cast(a_bucket); - wallTimingNanoseconds[index] += a_duration.count(); - ++wallTimingCounts[index]; - } - } - - class ScopedWallTimer - { - public: - explicit ScopedWallTimer(WallTimingBucket a_bucket) : - _bucket(a_bucket) - { - if constexpr (bEnableParseTiming) { - _startTime = std::chrono::steady_clock::now(); - } - } - - ScopedWallTimer(const ScopedWallTimer&) = delete; - ScopedWallTimer& operator=(const ScopedWallTimer&) = delete; - - ~ScopedWallTimer() - { - if constexpr (bEnableParseTiming) { - AddWallTiming(_bucket, std::chrono::duration_cast(std::chrono::steady_clock::now() - _startTime)); - } - } - - private: - WallTimingBucket _bucket; - std::chrono::steady_clock::time_point _startTime; - }; } void ResetTimingStats() @@ -131,8 +90,6 @@ namespace Parsing for (auto& counter : timingCounters) { counter.store(0, std::memory_order_relaxed); } - wallTimingNanoseconds.fill(0); - wallTimingCounts.fill(0); } } @@ -164,15 +121,6 @@ namespace Parsing if (directoryEntries || directoryDirectories || directoryFiles || directoryHkxFiles || directoryInvalidPaths || directoryHiddenRecursionSkips) { logger::info(" Directory scan details: {} entries, {} directories, {} files, {} hkx files, {} invalid paths, {} hidden recursion skips", directoryEntries, directoryDirectories, directoryFiles, directoryHkxFiles, directoryInvalidPaths, directoryHiddenRecursionSkips); } - - logger::info("Parsing wall timings (main-thread waits; nested phases may overlap with worker-time above):"); - for (size_t i = 0; i < wallTimingBucketCount; ++i) { - const auto duration = wallTimingNanoseconds[i]; - const auto count = wallTimingCounts[i]; - if (count > 0) { - logger::info(" {}: {:.3f}ms ({} calls)", GetWallTimingBucketName(static_cast(i)), ToMilliseconds(duration), count); - } - } } } @@ -787,236 +735,119 @@ namespace Parsing return static_cast(-1); } - struct ParseJobs + template + static std::future MakeFuture(T& a_t) { - std::vector modDirectories; - std::vector legacyCustomConditionDirectories; - std::vector legacyPluginDirectories; - }; - - class WorkerPool - { - public: - explicit WorkerPool(std::size_t a_workerCount) - { - for (std::size_t i = 0; i < a_workerCount; ++i) { - _threads.emplace_back([this] { - Run(); - }); - } - } + std::promise p; + p.set_value(std::forward(a_t)); + return p.get_future(); + } - ~WorkerPool() - { - { - std::lock_guard lock(_mutex); - _stopping = true; - } + static DirectoryCache g_directoryCache; - _cv.notify_all(); + static bool IsDirectoryCacheReady(); + static void WaitForDirectoryCache(); - for (auto& thread : _threads) { - if (thread.joinable()) { - thread.join(); - } + static void ParseCachedOARDirectory(const CachedOARDirectory& a_cachedOAR, ParseResults& a_outParseResults) + { + for (const auto& cachedMod : a_cachedOAR.modDirectories) { + if (!IsPathValid(cachedMod.path)) { + continue; } - } - - template - auto Enqueue(F&& a_func) - { - using Result = std::invoke_result_t>; - auto task = std::make_shared>(std::forward(a_func)); - auto future = task->get_future(); + if (!cachedMod.subModDirectories.empty()) { + a_outParseResults.modParseResultFutures.emplace_back(std::async(std::launch::async, [cachedMod]() { + return ParseModDirectory(cachedMod); + })); + continue; + } - { - std::lock_guard lock(_mutex); - _jobs.emplace([task] { - (*task)(); - }); + std::filesystem::directory_entry modEntry(cachedMod.path); + if (!Utils::IsDirectory(modEntry)) { + continue; } - _cv.notify_one(); - return future; + a_outParseResults.modParseResultFutures.emplace_back(std::async(std::launch::async, [modEntry]() { + return ParseModDirectory(modEntry); + })); } + } - private: - void Run() - { - while (true) { - std::function job; - - { - std::unique_lock lock(_mutex); - _cv.wait(lock, [this] { - return _stopping || !_jobs.empty(); - }); + static void ParseCachedLegacyDirectory(const CachedLegacyDirectory& a_cachedLegacy, ParseResults& a_outParseResults) + { + if (!a_cachedLegacy.detailedEntries.empty()) { + for (const auto& detailedEntry : a_cachedLegacy.detailedEntries) { + if (!IsPathValid(detailedEntry.path)) { + continue; + } - if (_stopping && _jobs.empty()) { - return; + if (detailedEntry.isCustomConditions) { + for (const auto& cachedSubMod : detailedEntry.subMods) { + a_outParseResults.legacyParseResultFutures.emplace_back(std::async(std::launch::async, [cachedSubMod]() { + return ParseLegacyCustomConditionsDirectory(cachedSubMod); + })); + } + } else { + for (auto subModParseResults = ParseLegacyPluginDirectory(detailedEntry); auto& subModParseResult : subModParseResults) { + if (subModParseResult.bSuccess) { + a_outParseResults.legacyParseResultFutures.emplace_back(MakeFuture(subModParseResult)); + } } - - job = std::move(_jobs.front()); - _jobs.pop(); } - - job(); - } - } - - std::vector _threads; - std::queue> _jobs; - std::mutex _mutex; - std::condition_variable _cv; - bool _stopping = false; - }; - - void DiscoverOARModJobs(const std::filesystem::directory_entry& a_oarDirectory, ParseJobs& a_outParseJobs) - { - for (const auto& subEntry : std::filesystem::directory_iterator(a_oarDirectory)) { - if (Utils::IsDirectory(subEntry)) { - // We're in a mod folder. We have the subfolders here and a json. - a_outParseJobs.modDirectories.emplace_back(subEntry.path()); } + return; } - } - void DiscoverLegacyCustomConditionJobs(const std::filesystem::directory_entry& a_customConditionsDirectory, ParseJobs& a_outParseJobs) - { - for (const auto& subEntry : std::filesystem::directory_iterator(a_customConditionsDirectory)) { - if (Utils::IsDirectory(subEntry)) { - a_outParseJobs.legacyCustomConditionDirectories.emplace_back(subEntry.path()); + for (const auto& cachedEntry : a_cachedLegacy.entries) { + if (!cachedEntry.isDirectory) { + continue; } - } - } - void DiscoverLegacyJobs(const std::filesystem::directory_entry& a_legacyDirectory, ParseJobs& a_outParseJobs) - { - for (const auto& subEntry : std::filesystem::directory_iterator(a_legacyDirectory)) { + std::filesystem::directory_entry subEntry(cachedEntry.path); if (!Utils::IsDirectory(subEntry) || !IsPathValid(subEntry.path())) { continue; } if (Utils::CompareStringsIgnoreCase(subEntry.path().stem().string(), "_CustomConditions"sv)) { - // We're in the DAR _CustomConditions directory. - DiscoverLegacyCustomConditionJobs(subEntry, a_outParseJobs); + for (const auto& subSubEntry : std::filesystem::directory_iterator(subEntry)) { + if (Utils::IsDirectory(subSubEntry)) { + a_outParseResults.legacyParseResultFutures.emplace_back(std::async(std::launch::async, [subSubEntry]() { + return ParseLegacyCustomConditionsDirectory(subSubEntry); + })); + } + } } else { - a_outParseJobs.legacyPluginDirectories.emplace_back(subEntry); - } - } - } - - ParseJobs DiscoverParseJobs(const std::filesystem::directory_entry& a_directory) - { - ParseJobs parseJobs; - - static constexpr auto oarFolderName = "openanimationreplacer"sv; - static constexpr auto legacyFolderName = "dynamicanimationreplacer"sv; - - for (std::filesystem::recursive_directory_iterator i(a_directory), end; i != end; ++i) { - auto entry = *i; - if (!Utils::IsDirectory(entry)) { - continue; - } - - if (!IsPathValid(entry.path())) { - i.disable_recursion_pending(); - continue; - } - - std::string stemString = entry.path().stem().string(); - - if (Utils::CompareStringsIgnoreCase(stemString, oarFolderName)) { - DiscoverOARModJobs(entry, parseJobs); - i.disable_recursion_pending(); - } else if (Utils::CompareStringsIgnoreCase(stemString, legacyFolderName)) { - DiscoverLegacyJobs(entry, parseJobs); - i.disable_recursion_pending(); - } - } - - return parseJobs; - } - - void AppendLegacyParseResults(std::vector& a_parseResults, ParseResults& a_outParseResults) - { - for (auto& subModParseResult : a_parseResults) { - if (subModParseResult.bSuccess) { - a_outParseResults.legacyParseResults.emplace_back(std::move(subModParseResult)); + for (auto subModParseResults = ParseLegacyPluginDirectory(subEntry); auto& subModParseResult : subModParseResults) { + if (subModParseResult.bSuccess) { + a_outParseResults.legacyParseResultFutures.emplace_back(MakeFuture(subModParseResult)); + } + } } } } - void ExecuteParseJobs(const ParseJobs& a_parseJobs, ParseResults& a_outParseResults) + void ParseDirectory(const std::filesystem::directory_entry& a_directory, ParseResults& a_outParseResults) { - ScopedWallTimer executeTimer(WallTimingBucket::kExecuteParseJobs); - const auto workerCount = std::clamp(Settings::uParsingWorkerCount, 1u, 32u); - logger::info("Using {} parsing worker(s).", workerCount); - - WorkerPool pool(workerCount); - - std::vector> modFutures; - std::vector> legacyFutures; - std::vector>> legacyPluginFutures; - - for (const auto& path : a_parseJobs.modDirectories) { - modFutures.emplace_back(pool.Enqueue([path] { - return ParseModDirectory(std::filesystem::directory_entry(path)); - })); - } - - for (const auto& path : a_parseJobs.legacyCustomConditionDirectories) { - legacyFutures.emplace_back(pool.Enqueue([path] { - return ParseLegacyCustomConditionsDirectory(std::filesystem::directory_entry(path)); - })); - } - - for (const auto& directory : a_parseJobs.legacyPluginDirectories) { - legacyPluginFutures.emplace_back(pool.Enqueue([directory] { - return ParseLegacyPluginDirectory(directory); - })); + if (!Utils::Exists(a_directory)) { + return; } - { - ScopedWallTimer timer(WallTimingBucket::kOARModJobs); - for (auto& future : modFutures) { - auto modParseResult = future.get(); - a_outParseResults.modParseResults.emplace_back(std::move(modParseResult)); - } + if (!IsDirectoryCacheReady()) { + logger::info("Waiting for directory cache to complete..."); + WaitForDirectoryCache(); } { - ScopedWallTimer timer(WallTimingBucket::kLegacyCustomConditionJobs); - for (auto& future : legacyFutures) { - auto subModParseResult = future.get(); - a_outParseResults.legacyParseResults.emplace_back(std::move(subModParseResult)); + std::shared_lock lock(g_directoryCache.cacheLock); + for (const auto& cachedOAR : g_directoryCache.oarDirectories) { + ParseCachedOARDirectory(cachedOAR, a_outParseResults); } - } - - { - ScopedWallTimer timer(WallTimingBucket::kLegacyPluginJobs); - for (auto& future : legacyPluginFutures) { - auto subModParseResults = future.get(); - AppendLegacyParseResults(subModParseResults, a_outParseResults); + for (const auto& cachedLegacy : g_directoryCache.legacyDirectories) { + ParseCachedLegacyDirectory(cachedLegacy, a_outParseResults); } } + return; } - - void ParseDirectory(const std::filesystem::directory_entry& a_directory, ParseResults& a_outParseResults) - { - if (!Utils::Exists(a_directory)) { - return; - } - - ParseJobs parseJobs; - { - ScopedWallTimer timer(WallTimingBucket::kDiscovery); - parseJobs = DiscoverParseJobs(a_directory); - } - ExecuteParseJobs(parseJobs, a_outParseResults); - } - ModParseResult ParseModDirectory(const std::filesystem::directory_entry& a_directory) { ModParseResult result; @@ -1061,6 +892,51 @@ namespace Parsing } } } + return result; + } + ModParseResult ParseModDirectory(const CachedModDirectory& a_cachedMod) + { + ModParseResult result; + + if (IsPathValid(a_cachedMod.path)) { + bool bDeserializeSuccess = false; + + const auto configJsonPath = a_cachedMod.path / "config.json"sv; + if (Utils::IsRegularFile(configJsonPath)) { + result.configSource = ConfigSource::kAuthor; + + const auto userJsonPath = a_cachedMod.path / "user.json"sv; + if (Utils::IsRegularFile(userJsonPath)) { + result.configSource = ConfigSource::kUser; + + if (!DeserializeMod(configJsonPath, DeserializeMode::kInfoOnly, result)) { + return result; + } + } + + if (result.configSource == ConfigSource::kUser) { + bDeserializeSuccess = DeserializeMod(userJsonPath, DeserializeMode::kDataOnly, result); + } else { + bDeserializeSuccess = DeserializeMod(configJsonPath, DeserializeMode::kFull, result); + } + } + + if (bDeserializeSuccess) { + std::vector> futures; + for (const auto& cachedSubMod : a_cachedMod.subModDirectories) { + futures.emplace_back(std::async(std::launch::async, [cachedSubMod]() { + return ParseModSubdirectory(cachedSubMod, false); + })); + } + + for (auto& future : futures) { + auto subModParseResult = future.get(); + if (subModParseResult.bSuccess) { + result.subModParseResults.emplace_back(std::move(subModParseResult)); + } + } + } + } return result; } @@ -1128,6 +1004,63 @@ namespace Parsing return result; } + SubModParseResult ParseModSubdirectory(const CachedSubModDirectory& a_cachedSubMod, bool a_bIsLegacy) + { + SubModParseResult result; + + if (IsPathValid(a_cachedSubMod.path)) { + bool bDeserializeSuccess = false; + + if (a_bIsLegacy) { + const auto userJsonPath = a_cachedSubMod.path / "user.json"sv; + if (Utils::IsRegularFile(userJsonPath)) { + result.configSource = ConfigSource::kUser; + bDeserializeSuccess = DeserializeSubMod(userJsonPath, DeserializeMode::kDataOnly, result); + } else { + return result; + } + } else { + const auto configJsonPath = a_cachedSubMod.path / "config.json"sv; + if (Utils::IsRegularFile(configJsonPath)) { + result.configSource = ConfigSource::kAuthor; + + const auto userJsonPath = a_cachedSubMod.path / "user.json"sv; + if (Utils::IsRegularFile(userJsonPath)) { + result.configSource = ConfigSource::kUser; + + if (!DeserializeSubMod(configJsonPath, DeserializeMode::kInfoOnly, result)) { + return result; + } + } + + if (result.configSource == ConfigSource::kUser) { + bDeserializeSuccess = DeserializeSubMod(userJsonPath, DeserializeMode::kDataOnly, result); + } else { + bDeserializeSuccess = DeserializeSubMod(configJsonPath, DeserializeMode::kFull, result); + } + } else { + return result; + } + } + + if (bDeserializeSuccess) { + if (result.overrideAnimationsFolder.empty()) { + result.animationFiles = ParseAnimationsFromCache(a_cachedSubMod, a_bIsLegacy); + } else { + const auto overridePath = a_cachedSubMod.path.parent_path() / result.overrideAnimationsFolder; + const auto overrideDirectory = std::filesystem::directory_entry(overridePath); + if (Utils::IsDirectory(overrideDirectory)) { + result.animationFiles = ParseAnimationsInDirectory(overrideDirectory, a_bIsLegacy); + } else { + result.bSuccess = false; + } + } + } + } + + return result; + } + SubModParseResult ParseLegacyCustomConditionsDirectory(const std::filesystem::directory_entry& a_directory) { SubModParseResult result; @@ -1180,6 +1113,145 @@ namespace Parsing return result; } + static std::vector ParseAnimationsFromLegacyCache(const CachedLegacySubMod& a_cachedSubMod) + { + std::vector result; + + for (const auto& cachedAnim : a_cachedSubMod.animationFiles) { + if (auto anim = ParseReplacementAnimationEntry(cachedAnim.path.string())) { + result.emplace_back(*anim); + } + } + + return result; + } + + SubModParseResult ParseLegacyCustomConditionsDirectory(const CachedLegacySubMod& a_cachedSubMod) + { + SubModParseResult result; + + if (IsPathValid(a_cachedSubMod.path)) { + const std::filesystem::path txtPath = a_cachedSubMod.path / "_conditions.txt"sv; + if (Utils::Exists(txtPath)) { + const auto jsonPath = a_cachedSubMod.path / "user.json"sv; + if (Utils::IsRegularFile(jsonPath)) { + CachedSubModDirectory cachedOARSubMod; + cachedOARSubMod.path = a_cachedSubMod.path; + cachedOARSubMod.animationFiles = a_cachedSubMod.animationFiles; + result = ParseModSubdirectory(cachedOARSubMod, true); + result.name = std::to_string(result.priority); + return result; + } + } + + int32_t priority = 0; + std::string directoryName = a_cachedSubMod.path.filename().string(); + + if (directoryName.find_first_not_of("-0123456789"sv) == std::string::npos) { + auto [ptr, ec]{ std::from_chars(directoryName.data(), directoryName.data() + directoryName.size(), priority) }; + if (ec == std::errc()) { + if (Utils::Exists(txtPath)) { + result.configSource = ConfigSource::kLegacy; + result.name = std::to_string(priority); + result.priority = priority; + result.conditionSet = ParseConditionsTxt(txtPath); + result.animationFiles = ParseAnimationsFromLegacyCache(a_cachedSubMod); + result.bSuccess = true; + } else { + const auto subEntryPath = a_cachedSubMod.path.u8string(); + std::string_view subEntryPathSv(reinterpret_cast(subEntryPath.data()), subEntryPath.size()); + logger::warn("directory at {} is missing the _conditions.txt file, skipping", subEntryPathSv); + } + } else { + const auto subEntryPath = a_cachedSubMod.path.u8string(); + std::string_view subEntryPathSv(reinterpret_cast(subEntryPath.data()), subEntryPath.size()); + logger::warn("invalid directory name at {}, skipping", subEntryPathSv); + } + } else { + const auto subEntryPath = a_cachedSubMod.path.u8string(); + std::string_view subEntryPathSv(reinterpret_cast(subEntryPath.data()), subEntryPath.size()); + logger::warn("invalid directory name at {}, skipping", subEntryPathSv); + } + + result.path = a_cachedSubMod.path.string(); + } + + return result; + } + + std::vector ParseLegacyPluginDirectory(const CachedLegacyEntry& a_cachedEntry) + { + std::vector results; + + for (const auto& cachedSubMod : a_cachedEntry.subMods) { + if (!IsPathValid(cachedSubMod.path)) { + continue; + } + + auto jsonPath = cachedSubMod.path / "user.json"sv; + if (Utils::IsRegularFile(jsonPath)) { + CachedSubModDirectory cachedOARSubMod; + cachedOARSubMod.path = cachedSubMod.path; + cachedOARSubMod.animationFiles = cachedSubMod.animationFiles; + auto result = ParseModSubdirectory(cachedOARSubMod, true); + + const std::string fileString = a_cachedEntry.path.stem().string(); + const std::string extensionString = a_cachedEntry.path.extension().string(); + + const std::string modName = fileString + extensionString; + const std::string formIDString = cachedSubMod.path.filename().string(); + result.name = modName; + result.name += '|'; + result.name += formIDString; + + results.emplace_back(std::move(result)); + continue; + } + + if (cachedSubMod.animationFiles.empty()) { + continue; + } + + RE::FormID formID; + std::string directoryName = cachedSubMod.path.filename().string(); + + auto [ptr, ec]{ std::from_chars(directoryName.data(), directoryName.data() + directoryName.size(), formID, 16) }; + if (ec == std::errc()) { + std::string fileString = a_cachedEntry.path.stem().string(); + std::string extensionString = a_cachedEntry.path.extension().string(); + + std::string modName = fileString + extensionString; + if (auto form = Utils::LookupForm(formID, modName)) { + auto conditionSet = std::make_unique(); + std::string argument = modName; + argument += '|'; + argument += directoryName; + auto condition = OpenAnimationReplacer::GetSingleton().CreateCondition("IsActorBase"); + static_cast(condition.get())->formComponent->SetTESFormValue(form); + conditionSet->Add(condition); + + SubModParseResult result; + + result.configSource = ConfigSource::kLegacyActorBase; + result.name = argument; + result.priority = 0; + result.conditionSet = std::move(conditionSet); + result.animationFiles = ParseAnimationsFromLegacyCache(cachedSubMod); + result.bSuccess = true; + result.path = cachedSubMod.path.string(); + + results.emplace_back(std::move(result)); + } + } else { + auto subEntryPath = cachedSubMod.path.u8string(); + std::string_view subEntryPathSv(reinterpret_cast(subEntryPath.data()), subEntryPath.size()); + logger::warn("invalid directory name at {}, skipping", subEntryPathSv); + } + } + + return results; + } + std::vector ParseLegacyPluginDirectory(const std::filesystem::directory_entry& a_directory) { std::vector results; @@ -1367,6 +1439,22 @@ namespace Parsing return ReplacementAnimationFile(a_fullVariantsPath, variants); } + std::optional ParseReplacementAnimationVariants(const CachedAnimationFile& a_cachedVariants) + { + if (!a_cachedVariants.isVariantsDirectory || a_cachedVariants.variantPaths.empty()) { + return std::nullopt; + } + + std::vector variants; + variants.reserve(a_cachedVariants.variantPaths.size()); + + for (const auto& variantPath : a_cachedVariants.variantPaths) { + variants.emplace_back(variantPath.string()); + } + + return ReplacementAnimationFile(a_cachedVariants.path.string(), variants); + } + std::vector ParseNonLegacyAnimationsInDirectory(const std::filesystem::directory_entry& a_directory) { std::vector result; @@ -1440,6 +1528,30 @@ namespace Parsing result.emplace_back(*anim); } } + return result; + } + + std::vector ParseAnimationsFromCache(const CachedSubModDirectory& a_cachedSubMod, bool a_bIsLegacy /* = false*/) + { + std::vector result; + + for (const auto& cachedAnim : a_cachedSubMod.animationFiles) { + if (cachedAnim.isVariantsDirectory) { + if (auto anim = ParseReplacementAnimationVariants(cachedAnim)) { + result.emplace_back(*anim); + } + } else if (auto anim = ParseReplacementAnimationEntry(cachedAnim.path.string())) { + result.emplace_back(*anim); + } + } + + if (!a_bIsLegacy) { + for (const auto& subdirectory : a_cachedSubMod.subdirectories) { + auto res = ParseAnimationsFromCache(subdirectory, a_bIsLegacy); + result.reserve(result.size() + res.size()); + result.insert(result.end(), std::make_move_iterator(res.begin()), std::make_move_iterator(res.end())); + } + } return result; } @@ -1474,4 +1586,320 @@ namespace Parsing return true; } + + static void PrecacheAnimationHashes(const std::filesystem::path& a_directory) + { + if (!Settings::bFilterOutDuplicateAnimations) { + return; + } + + try { + for (const auto& entry : std::filesystem::recursive_directory_iterator(a_directory)) { + if (!entry.is_regular_file()) { + continue; + } + + const auto& path = entry.path(); + if (!path.has_extension() || !Utils::CompareStringsIgnoreCase(path.extension().string(), ".hkx"sv) || !IsPathValid(path)) { + continue; + } + + AnimationFileHashCache::CalculateHash(path.string()); + g_precachedHashCount.fetch_add(1, std::memory_order_relaxed); + } + } catch (const std::filesystem::filesystem_error& e) { + logger::warn("Error pre-caching animation hashes in {}: {}", a_directory.string(), e.what()); + } + } + + static bool IsDirectoryCacheReady() + { + return g_directoryCache.bIsComplete.load(std::memory_order_acquire); + } + + static void WaitForDirectoryCache() + { +#ifdef _WIN32 + bool bBoostedCacheThread = false; +#endif + while (!g_directoryCache.bIsComplete.load(std::memory_order_acquire)) { +#ifdef _WIN32 + if (!bBoostedCacheThread) { + const auto cacheThreadId = g_directoryCacheThreadId.load(std::memory_order_acquire); + if (cacheThreadId != 0) { + if (const auto cacheThread = OpenThread(THREAD_SET_INFORMATION, false, cacheThreadId)) { + SetThreadPriority(cacheThread, THREAD_PRIORITY_HIGHEST); + SetIOPriority(cacheThread, HighThreadIoPriority); + CloseHandle(cacheThread); + bBoostedCacheThread = true; + } + } + } +#endif + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + + static void CacheSubModDirectoryContents(const std::filesystem::path& a_path, CachedSubModDirectory& a_outCached, bool a_bIsLegacy); + + static void CacheAnimationFilesInDirectory(const std::filesystem::path& a_directory, CachedSubModDirectory& a_outCached, bool a_bIsLegacy) + { + if (!a_bIsLegacy) { + std::vector variantDirectoryNames; + + for (const auto& entry : std::filesystem::directory_iterator(a_directory)) { + if (!IsPathValid(entry.path())) { + continue; + } + + if (!Utils::IsDirectory(entry)) { + continue; + } + + std::string directoryName = entry.path().filename().string(); + if (directoryName.starts_with("_variants_"sv)) { + variantDirectoryNames.emplace_back(ConvertVariantsPath(directoryName)); + + CachedAnimationFile cachedAnim; + cachedAnim.path = entry.path(); + cachedAnim.isVariantsDirectory = true; + + for (const auto& variantEntry : std::filesystem::directory_iterator(entry)) { + if (IsPathValid(variantEntry.path()) && + Utils::IsRegularFile(variantEntry) && + Utils::CompareStringsIgnoreCase(variantEntry.path().extension().string(), ".hkx"sv)) { + cachedAnim.variantPaths.push_back(variantEntry.path()); + } + } + + if (!cachedAnim.variantPaths.empty()) { + a_outCached.animationFiles.push_back(std::move(cachedAnim)); + } + continue; + } + + CachedSubModDirectory subdirectory; + CacheSubModDirectoryContents(entry.path(), subdirectory, a_bIsLegacy); + if (!subdirectory.animationFiles.empty() || !subdirectory.subdirectories.empty()) { + a_outCached.subdirectories.push_back(std::move(subdirectory)); + } + } + + for (const auto& entry : std::filesystem::directory_iterator(a_directory)) { + if (!IsPathValid(entry.path()) || !Utils::IsRegularFile(entry) || !Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { + continue; + } + + std::string filename = entry.path().filename().string(); + const bool bSkip = std::ranges::any_of(variantDirectoryNames, [&](const auto& name) { + return filename == name; + }); + + if (!bSkip) { + CachedAnimationFile cachedAnim; + cachedAnim.path = entry.path(); + a_outCached.animationFiles.push_back(std::move(cachedAnim)); + } + } + } else { + for (const auto& entry : std::filesystem::recursive_directory_iterator(a_directory)) { + if (IsPathValid(entry.path()) && + Utils::IsRegularFile(entry) && + Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { + CachedAnimationFile cachedAnim; + cachedAnim.path = entry.path(); + a_outCached.animationFiles.push_back(std::move(cachedAnim)); + } + } + } + } + + static void CacheSubModDirectoryContents(const std::filesystem::path& a_path, CachedSubModDirectory& a_outCached, bool a_bIsLegacy) + { + a_outCached.path = a_path; + CacheAnimationFilesInDirectory(a_path, a_outCached, a_bIsLegacy); + } + + static void CacheOARDirectoryContents(const std::filesystem::directory_entry& a_oarEntry, CachedOARDirectory& a_outCached) + { + a_outCached.path = a_oarEntry.path(); + + for (const auto& modEntry : std::filesystem::directory_iterator(a_oarEntry)) { + if (!Utils::IsDirectory(modEntry) || !IsPathValid(modEntry.path())) { + continue; + } + + CachedModDirectory cachedMod; + cachedMod.path = modEntry.path(); + + for (const auto& subEntry : std::filesystem::directory_iterator(modEntry)) { + if (!IsPathValid(subEntry.path())) { + continue; + } + + cachedMod.entries.push_back({ subEntry.path(), Utils::IsDirectory(subEntry) }); + + if (Utils::IsDirectory(subEntry)) { + CachedSubModDirectory cachedSubMod; + CacheSubModDirectoryContents(subEntry.path(), cachedSubMod, false); + cachedMod.subModDirectories.push_back(std::move(cachedSubMod)); + + PrecacheAnimationHashes(subEntry.path()); + } + } + + a_outCached.modDirectories.push_back(std::move(cachedMod)); + } + } + + static void CacheLegacyAnimationFiles(const std::filesystem::path& a_directory, CachedLegacySubMod& a_outCached) + { + a_outCached.path = a_directory; + + for (const auto& entry : std::filesystem::recursive_directory_iterator(a_directory)) { + if (IsPathValid(entry.path()) && + Utils::IsRegularFile(entry) && + Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { + CachedAnimationFile cachedAnim; + cachedAnim.path = entry.path(); + a_outCached.animationFiles.push_back(std::move(cachedAnim)); + } + } + } + + static void CacheLegacyDirectoryContents(const std::filesystem::directory_entry& a_legacyEntry, CachedLegacyDirectory& a_outCached) + { + a_outCached.path = a_legacyEntry.path(); + + for (const auto& subEntry : std::filesystem::directory_iterator(a_legacyEntry)) { + if (!Utils::IsDirectory(subEntry) || !IsPathValid(subEntry.path())) { + continue; + } + + a_outCached.entries.push_back({ subEntry.path(), true }); + + CachedLegacyEntry detailedEntry; + detailedEntry.path = subEntry.path(); + + if (Utils::CompareStringsIgnoreCase(subEntry.path().stem().string(), "_CustomConditions"sv)) { + detailedEntry.isCustomConditions = true; + + for (const auto& customConditionEntry : std::filesystem::directory_iterator(subEntry)) { + if (Utils::IsDirectory(customConditionEntry) && IsPathValid(customConditionEntry.path())) { + CachedLegacySubMod cachedSubMod; + CacheLegacyAnimationFiles(customConditionEntry.path(), cachedSubMod); + detailedEntry.subMods.push_back(std::move(cachedSubMod)); + + PrecacheAnimationHashes(customConditionEntry.path()); + } + } + } else { + for (const auto& formIdEntry : std::filesystem::directory_iterator(subEntry)) { + if (Utils::IsDirectory(formIdEntry) && IsPathValid(formIdEntry.path())) { + CachedLegacySubMod cachedSubMod; + CacheLegacyAnimationFiles(formIdEntry.path(), cachedSubMod); + detailedEntry.subMods.push_back(std::move(cachedSubMod)); + + PrecacheAnimationHashes(formIdEntry.path()); + } + } + } + + a_outCached.detailedEntries.push_back(std::move(detailedEntry)); + } + } + + static void CacheDirectoriesInternal() + { + // We do NOT want to overly optimize/parallelize this thread - we want the lowest IO priority possible + // Otherwise all perf gain is lost due to other loading stuff choking (i.e .esp loading, etc) +#ifdef _WIN32 + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_LOWEST); + SetLowIOPriority(); +#endif + + logger::info("Starting directory cache..."); + const auto startTime = std::chrono::high_resolution_clock::now(); + + static constexpr auto oarFolderName = "openanimationreplacer"sv; + static constexpr auto legacyFolderName = "dynamicanimationreplacer"sv; + static constexpr auto meshesPath = "data\\meshes\\"sv; + + const std::filesystem::directory_entry meshesDir(meshesPath); + if (!Utils::Exists(meshesDir)) { + logger::warn("Meshes directory not found, skipping directory cache"); + g_directoryCache.bIsComplete.store(true, std::memory_order_release); + return; + } + + std::vector oarDirs; + std::vector legacyDirs; + + try { + for (std::filesystem::recursive_directory_iterator i(meshesDir), end; i != end; ++i) { + auto entry = *i; + if (!Utils::IsDirectory(entry)) { + continue; + } + + if (!IsPathValid(entry.path())) { + i.disable_recursion_pending(); + continue; + } + + std::string stemString = entry.path().stem().string(); + if (Utils::CompareStringsIgnoreCase(stemString, oarFolderName)) { + CachedOARDirectory cachedOAR; + CacheOARDirectoryContents(entry, cachedOAR); + oarDirs.push_back(std::move(cachedOAR)); + i.disable_recursion_pending(); + } else if (Utils::CompareStringsIgnoreCase(stemString, legacyFolderName)) { + CachedLegacyDirectory cachedLegacy; + CacheLegacyDirectoryContents(entry, cachedLegacy); + legacyDirs.push_back(std::move(cachedLegacy)); + i.disable_recursion_pending(); + } + } + } catch (const std::filesystem::filesystem_error& e) { + logger::warn("Error while caching directories: {}", e.what()); + } + + { + std::unique_lock lock(g_directoryCache.cacheLock); + g_directoryCache.oarDirectories = std::move(oarDirs); + g_directoryCache.legacyDirectories = std::move(legacyDirs); + } + + const auto endTime = std::chrono::high_resolution_clock::now(); + const auto duration = std::chrono::duration_cast(endTime - startTime).count(); + const auto hashCount = g_precachedHashCount.load(std::memory_order_relaxed); + + if (Settings::bFilterOutDuplicateAnimations) { + logger::info("Directory cache complete: {} OAR directories, {} legacy directories, {} animation hashes ({}ms)", + g_directoryCache.oarDirectories.size(), + g_directoryCache.legacyDirectories.size(), + hashCount, + duration); + } else { + logger::info("Directory cache complete: {} OAR directories, {} legacy directories ({}ms)", + g_directoryCache.oarDirectories.size(), + g_directoryCache.legacyDirectories.size(), + duration); + } + + g_directoryCache.bIsComplete.store(true, std::memory_order_release); + } + + void StartDirectoryCaching() + { + std::thread([]() { +#ifdef _WIN32 + g_directoryCacheThreadId.store(GetCurrentThreadId(), std::memory_order_release); +#endif + CacheDirectoriesInternal(); +#ifdef _WIN32 + g_directoryCacheThreadId.store(0, std::memory_order_release); +#endif + }).detach(); + } } diff --git a/src/Parsing.h b/src/Parsing.h index 12f922c..9d402d2 100644 --- a/src/Parsing.h +++ b/src/Parsing.h @@ -5,7 +5,10 @@ #include "ReplacementAnimation.h" #include "Settings.h" +#include #include +#include +#include struct ReplacementAnimData { @@ -208,8 +211,8 @@ namespace Parsing struct ParseResults { - std::vector modParseResults; - std::vector legacyParseResults; + std::vector> modParseResultFutures; + std::vector> legacyParseResultFutures; }; [[nodiscard]] std::unique_ptr ParseConditionsTxt(const std::filesystem::path& a_txtPath); @@ -224,14 +227,82 @@ namespace Parsing [[nodiscard]] uint16_t GetOriginalAnimationBindingIndex(RE::hkbCharacterStringData* a_stringData, std::string_view a_animationName); + struct CachedDirectoryEntry + { + std::filesystem::path path; + bool isDirectory = false; + }; + + struct CachedAnimationFile + { + std::filesystem::path path; + bool isVariantsDirectory = false; + std::vector variantPaths; + }; + + struct CachedSubModDirectory + { + std::filesystem::path path; + std::vector animationFiles; + std::vector subdirectories; + }; + + struct CachedModDirectory + { + std::filesystem::path path; + std::vector entries; + std::vector subModDirectories; + }; + + struct CachedOARDirectory + { + std::filesystem::path path; + std::vector modDirectories; + }; + + struct CachedLegacySubMod + { + std::filesystem::path path; + std::vector animationFiles; + }; + + struct CachedLegacyEntry + { + std::filesystem::path path; + bool isCustomConditions = false; + std::vector subMods; + }; + + struct CachedLegacyDirectory + { + std::filesystem::path path; + std::vector entries; + std::vector detailedEntries; + }; + + struct DirectoryCache + { + std::vector oarDirectories; + std::vector legacyDirectories; + std::atomic bIsComplete{ false }; + mutable std::shared_mutex cacheLock; + }; + void ParseDirectory(const std::filesystem::directory_entry& a_directory, ParseResults& a_outParseResults); [[nodiscard]] ModParseResult ParseModDirectory(const std::filesystem::directory_entry& a_directory); + [[nodiscard]] ModParseResult ParseModDirectory(const CachedModDirectory& a_cachedMod); [[nodiscard]] SubModParseResult ParseModSubdirectory(const std::filesystem::directory_entry& a_subDirectory, bool a_bIsLegacy = false); + [[nodiscard]] SubModParseResult ParseModSubdirectory(const CachedSubModDirectory& a_cachedSubMod, bool a_bIsLegacy = false); [[nodiscard]] SubModParseResult ParseLegacyCustomConditionsDirectory(const std::filesystem::directory_entry& a_directory); + [[nodiscard]] SubModParseResult ParseLegacyCustomConditionsDirectory(const CachedLegacySubMod& a_cachedSubMod); [[nodiscard]] std::vector ParseLegacyPluginDirectory(const std::filesystem::directory_entry& a_directory); + [[nodiscard]] std::vector ParseLegacyPluginDirectory(const CachedLegacyEntry& a_cachedEntry); [[nodiscard]] std::optional ParseReplacementAnimationEntry(std::string_view a_fullPath); [[nodiscard]] std::optional ParseReplacementAnimationVariants(std::string_view a_fullVariantsPath); + [[nodiscard]] std::optional ParseReplacementAnimationVariants(const CachedAnimationFile& a_cachedVariants); [[nodiscard]] std::vector ParseAnimationsInDirectory(const std::filesystem::directory_entry& a_directory, bool a_bIsLegacy = false); + [[nodiscard]] std::vector ParseAnimationsFromCache(const CachedSubModDirectory& a_cachedSubMod, bool a_bIsLegacy = false); [[nodiscard]] bool IsPathValid(const std::filesystem::path& a_path); + void StartDirectoryCaching(); } diff --git a/src/main.cpp b/src/main.cpp index a6f5d69..83eec64 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,6 @@ #include "Hooks.h" #include "OpenAnimationReplacer.h" +#include "Parsing.h" #include "Settings.h" #include "UI/UIManager.h" @@ -14,6 +15,9 @@ void MessageHandler(SKSE::MessagingInterface::Message* a_msg) { switch (a_msg->type) { + case SKSE::MessagingInterface::kInputLoaded: + Parsing::StartDirectoryCaching(); + break; case SKSE::MessagingInterface::kDataLoaded: OpenAnimationReplacer::GetSingleton().OnDataLoaded(); break;