From e65ae4ea4e74f997950857082a817e6ea1498900 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 25 May 2026 20:46:14 +0200 Subject: [PATCH 1/9] Code --- AssetEditor/App.xaml.cs | 2 +- AssetEditor/UiCommands/OpenGamePackCommand.cs | 2 +- AssetEditor/UiCommands/OpenPackFileCommand.cs | 2 +- AssetEditor/ViewModels/MenuBarViewModel.cs | 2 +- Editors/Ipc/IpcEditor/ExternalPackLoader.cs | 2 +- .../KitbasherEditor/DevConfig/Kitbash_Karl.cs | 2 +- .../DevConfig/Kitbash_MeshFitter.cs | 2 +- .../KitbasherEditor/DevConfig/Kitbash_Ox.cs | 2 +- .../KitbasherEditor/DevConfig/Kitbash_Rat.cs | 2 +- .../DevConfig/Kitbash_RomeShield.cs | 2 +- .../AnimationMeta/DevConfig/AnimMetaTool.cs | 2 +- .../AnimationMeta/DevConfig/SuperView_Rat.cs | 2 +- .../Reports/DeepSearch/DeepSearchReport.cs | 2 +- .../DevConfig/SkeletonTool.cs | 2 +- .../TextureEditor/DevConfig/Texture_Karl.cs | 2 +- ...letonAnimationLookUpHelperKarlPackTests.cs | 2 +- .../Containers/CachedPackFileContainer.cs | 1 - .../PackFileContainerCacheHelper.cs | 30 +-- .../PackFiles/Utility/ManifestHelper.cs | 5 +- .../Utility/PackFileContainerLoader.cs | 13 +- .../Utility/PackFileContainerLoader2.cs | 229 ++++++++++++++++++ .../SharedCore/Shared.Core/Shared.Core.csproj | 1 + .../PackFileContainerCacheHelperTests.cs | 31 ++- .../Utility/PackFileContainerLoaderTests.cs | 25 +- .../Shared/Shared/AssetEditorTestRunner.cs | 4 +- .../TestUtility/PackFileSerivceTestHelper.cs | 4 +- 26 files changed, 304 insertions(+), 71 deletions(-) create mode 100644 Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs diff --git a/AssetEditor/App.xaml.cs b/AssetEditor/App.xaml.cs index a243aeade..c9ad852a4 100644 --- a/AssetEditor/App.xaml.cs +++ b/AssetEditor/App.xaml.cs @@ -100,7 +100,7 @@ private void LoadCAPackFiles(ApplicationSettingsService settingsService) { var packfileService = _serviceProvider.GetRequiredService(); var containerLoader = _serviceProvider.GetRequiredService(); - var loadRes = containerLoader.LoadAllCaFiles(settingsService.CurrentSettings.CurrentGame); + var loadRes = containerLoader.CreateFromGameEnum(PackFileContainerType.Cached, settingsService.CurrentSettings.CurrentGame); if (loadRes == null) MessageBox.Show($"Unable to load all CA packfiles in {gamePath}"); diff --git a/AssetEditor/UiCommands/OpenGamePackCommand.cs b/AssetEditor/UiCommands/OpenGamePackCommand.cs index 61381680a..e1c9129a4 100644 --- a/AssetEditor/UiCommands/OpenGamePackCommand.cs +++ b/AssetEditor/UiCommands/OpenGamePackCommand.cs @@ -50,7 +50,7 @@ public void Execute() using (new WaitCursor()) { - var res = _packFileContainerLoader.LoadAllCaFiles(_game); + var res = _packFileContainerLoader.CreateFromGameEnum(PackFileContainerType.Cached, _game); _packFileService.AddContainer(res); } } diff --git a/AssetEditor/UiCommands/OpenPackFileCommand.cs b/AssetEditor/UiCommands/OpenPackFileCommand.cs index 274eac54b..9862b81b8 100644 --- a/AssetEditor/UiCommands/OpenPackFileCommand.cs +++ b/AssetEditor/UiCommands/OpenPackFileCommand.cs @@ -25,7 +25,7 @@ public void Execute() if (dialog.ShowDialog() != DialogResult.OK) return; - var container = _packFileContainerLoader.Load(dialog.FileName); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, dialog.FileName, false); _packFileService.AddContainer(container, true); } } diff --git a/AssetEditor/ViewModels/MenuBarViewModel.cs b/AssetEditor/ViewModels/MenuBarViewModel.cs index 31c959e7c..2ef5629e2 100644 --- a/AssetEditor/ViewModels/MenuBarViewModel.cs +++ b/AssetEditor/ViewModels/MenuBarViewModel.cs @@ -168,7 +168,7 @@ void CreateRecentPackFilesItems() path, () => { - var container = _packFileContainerLoader.Load(path); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, path, false); if (container == null) { System.Windows.MessageBox.Show($"Unable to load packfiles {path}"); diff --git a/Editors/Ipc/IpcEditor/ExternalPackLoader.cs b/Editors/Ipc/IpcEditor/ExternalPackLoader.cs index 22ff291ac..c26a745c8 100644 --- a/Editors/Ipc/IpcEditor/ExternalPackLoader.cs +++ b/Editors/Ipc/IpcEditor/ExternalPackLoader.cs @@ -40,7 +40,7 @@ public Task EnsureLoadedAsync(string packPathOnDisk, Cancellatio try { - var container = _packFileContainerLoader.Load(normalizedDiskPath); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, normalizedDiskPath, true); if (container == null) return Task.FromResult(PackLoadResult.Fail("Pack file could not be loaded")); diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs index e5e3430dd..c04354270 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs @@ -34,7 +34,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs index 842a5c5bb..a2d65923c 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs @@ -46,7 +46,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\AnimationTransfer_bear.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs index 97e8f3523..6fe72502e 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs @@ -33,7 +33,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\cinderbreath.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs index 63bc839f1..bb6395206 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs @@ -45,7 +45,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs index 77629ba19..012d39321 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs @@ -33,7 +33,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.CurrentGame = GameTypeEnum.Rome2; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Rome_Man_And_Shield_Pack"; - var container = _packFileContainerLoader.LoadSystemFolderAsPackFileContainer(packFile); + var container = _packFileContainerLoader.CreateFromSystemFolder(packFile); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs b/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs index 48a0f57d2..263b437cb 100644 --- a/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs +++ b/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs @@ -26,7 +26,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container!.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs b/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs index 5ea470a3f..bdd07184a 100644 --- a/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs +++ b/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs @@ -28,7 +28,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.CurrentGame = GameTypeEnum.Warhammer3; currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container!.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Reports/DeepSearch/DeepSearchReport.cs b/Editors/Reports/DeepSearch/DeepSearchReport.cs index f5515c4c8..3110236f0 100644 --- a/Editors/Reports/DeepSearch/DeepSearchReport.cs +++ b/Editors/Reports/DeepSearch/DeepSearchReport.cs @@ -73,7 +73,7 @@ public List DeepSearch(string searchStr, bool caseSensetive) { using (var reader = new BinaryReader(fileStram, Encoding.ASCII)) { - var pfc = _loader.Load(packFilePath); + var pfc = _loader.CreateFromPackFile(PackFileContainerType.Normal, packFilePath, true); _logger.Here().Information($"Searching through packfile {currentIndex}/{files.Count} - {packFilePath} {pfc.GetFileCount()} files"); diff --git a/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs b/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs index a88fda5c8..2e2c144f1 100644 --- a/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs +++ b/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs @@ -28,7 +28,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); Guard.IsNotNull(container, "Failed to load pack file container for development config."); container.IsCaPackFile = true; _packFileService.AddContainer(container); diff --git a/Editors/TextureEditor/DevConfig/Texture_Karl.cs b/Editors/TextureEditor/DevConfig/Texture_Karl.cs index d50766aac..a520503ea 100644 --- a/Editors/TextureEditor/DevConfig/Texture_Karl.cs +++ b/Editors/TextureEditor/DevConfig/Texture_Karl.cs @@ -31,7 +31,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) { currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs index 73a43df35..1713f30b0 100644 --- a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs @@ -34,7 +34,7 @@ public void LoadAndUnload_KarlPack_UpdatesSkeletonAndAnimationLookup() var settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager()); var karlPackPath = PathHelper.GetDataFile("Karl_and_celestialgeneral.pack"); - var karlContainer = loader.Load(karlPackPath); + var karlContainer = loader.CreateFromPackFile(PackFileContainerType.Normal, karlPackPath, false); Assert.That(karlContainer, Is.Not.Null); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 0c81f2558..98cfc5055 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Windows.Forms; using Microsoft.EntityFrameworkCore; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models.FileSources; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 53059ee8d..8fd662361 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -12,23 +12,28 @@ namespace Shared.Core.PackFiles.Serialization.CacheDatabase internal static class PackFileContainerCacheHelper { private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerCacheHelper)); - public static string GetCacheFilePath(string gameDataFolder, string gameName, string cacheId) + public static string GetCacheFilePath(string gameName, string cacheId) { var safeGameName = string.Join("_", gameName.Split(Path.GetInvalidFileNameChars())); return Path.Combine(DirectoryHelper.CacheDirectory, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); } - public static string ComputeFingerprint(string gameDataFolder, List packFileNames) + public static string ComputeFingerprint(List packFileNames) { using var sha = SHA256.Create(); var sb = new StringBuilder(); + if (packFileNames.Count == 0) + throw new Exception("Trying to compute CachecFingerPrint, but no files provied"); + + var foundFiles = 0; foreach (var packFileName in packFileNames.OrderBy(x => x, StringComparer.OrdinalIgnoreCase)) { - var fullPath = Path.Combine(gameDataFolder, packFileName); - if (File.Exists(fullPath)) + if (File.Exists(packFileName)) { - var info = new FileInfo(fullPath); + foundFiles++; + + var info = new FileInfo(packFileName); sb.Append(packFileName); sb.Append('|'); sb.Append(info.Length); @@ -36,17 +41,14 @@ public static string ComputeFingerprint(string gameDataFolder, List pack sb.Append(info.LastWriteTimeUtc.Ticks); sb.Append(';'); } + else + { + _logger.Here().Warning($"Trying to compute CachecFingerPrint, but file {packFileName} is not found"); + } } - var manifestPath = Path.Combine(gameDataFolder, "manifest.txt"); - if (File.Exists(manifestPath)) - { - var info = new FileInfo(manifestPath); - sb.Append("manifest.txt|"); - sb.Append(info.Length); - sb.Append('|'); - sb.Append(info.LastWriteTimeUtc.Ticks); - } + if (foundFiles == 0) + throw new Exception("Trying to compute CachecFingerPrint, but no files found. This will result in a default ID which can cause problems"); var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString())); return Convert.ToHexString(hash); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/ManifestHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/ManifestHelper.cs index 8096edf1e..c383f55b7 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/ManifestHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/ManifestHelper.cs @@ -13,7 +13,7 @@ public static List GetPackFilesFromManifest(string gameDataFolder, out b { var items = line.Split('\t'); if (items[0].Contains(".pack")) - output.Add(items[0].Trim()); + output.Add(Path.Combine(gameDataFolder, items[0].Trim())); } manifestFileFound = true; return output; @@ -21,8 +21,7 @@ public static List GetPackFilesFromManifest(string gameDataFolder, out b else { var files = Directory.GetFiles(gameDataFolder) - .Where(x => Path.GetExtension(x) == ".pack") - .Select(x => Path.GetFileName(x)) + .Where(x => Path.GetExtension(x).ToLower() == ".pack") .ToList(); manifestFileFound = false; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs index 0c8d58475..1c94c7cb8 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs @@ -11,14 +11,21 @@ namespace Shared.Core.PackFiles.Utility { - public interface IPackFileContainerLoader + /* public interface IPackFileContainerLoader { IPackFileContainer? Load(string packFileSystemPath); IPackFileContainer? LoadAllCaFiles(GameTypeEnum gameEnum); IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath); - } + }*/ + + + + + +/* + public class PackFileContainerLoader : IPackFileContainerLoader { static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); @@ -251,5 +258,5 @@ private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string g return caPackFileContainer; } - } + }*/ } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs new file mode 100644 index 000000000..2c41833f3 --- /dev/null +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs @@ -0,0 +1,229 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Text; +using Microsoft.Xna.Framework; +using Shared.Core.ErrorHandling; +using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models.Containers; +using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Serialization.CacheDatabase; +using Shared.Core.Services; +using Shared.Core.Settings; + +namespace Shared.Core.PackFiles.Utility +{ + + public enum PackFileContainerType + { + Cached, + Normal, + } + + public interface IPackFileContainerLoader + { + IPackFileContainer CreateFromPackFile(PackFileContainerType type, string packFilePath, bool loadAsReadOnly); + // IPackFileContainer CreateFromCollection(PackFileContainerType type, string packFileSystemPath, List fullPackFilePaths, string createdPackFileName, bool loadAsReadOnly, IDuplicateFileResolver duplicateFileResolver); + IPackFileContainer? CreateFromGameEnum(PackFileContainerType type, GameTypeEnum game); + + IPackFileContainer CreateFromSystemFolder(string folderPath); + } + + public class PackFileContainerLoader : IPackFileContainerLoader + { + static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); + private readonly ApplicationSettingsService _settingsService; + private readonly IStandardDialogs _standardDialogs; + private readonly LocalizationManager _localizationManager; + + public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager) + { + // TODO : Handle context for logger + + _settingsService = settingsService; + _standardDialogs = standardDialogs; + _localizationManager = localizationManager; + } + + public IPackFileContainer CreateFromSystemFolder(string packFileSystemPath) + { + if (Directory.Exists(packFileSystemPath) == false) + { + var location = Assembly.GetEntryAssembly()!.Location; + var loactionDir = Path.GetDirectoryName(location); + throw new Exception($"Unable to find folder {packFileSystemPath}. Curret systempath is {loactionDir}"); + } + + var containerName = Path.GetFileName(packFileSystemPath); + var container = new PackFileContainer(containerName) + { + SystemFilePath = packFileSystemPath, + }; + AddFolderContentToPackFile(container, packFileSystemPath, packFileSystemPath.ToLower() + "\\"); + return container; + } + + private static void AddFolderContentToPackFile(PackFileContainer container, string folderPath, string rootPath) + { + var files = Directory.GetFiles(folderPath); + foreach (var filePath in files) + { + var sanatizedFilePath = filePath.ToLower(); + var relativePath = sanatizedFilePath.Replace(rootPath, ""); + var fileName = Path.GetFileName(sanatizedFilePath); + + container.AddOrUpdateFile(relativePath, PackFile.CreateFromFileSystem(fileName, sanatizedFilePath)); + } + + var folders = Directory.GetDirectories(folderPath); + foreach (var folder in folders) + AddFolderContentToPackFile(container, folder, rootPath); + } + + + public IPackFileContainer CreateFromPackFile(PackFileContainerType type, string packFilePath, bool loadAsReadOnly) + { + var packfileName = Path.GetFileNameWithoutExtension(packFilePath); + return CreateFromCollection(type, packFilePath, [packFilePath], packfileName, loadAsReadOnly, new CustomPackDuplicateFileResolver()); + } + + public IPackFileContainer? CreateFromGameEnum(PackFileContainerType type, GameTypeEnum gameEnum) + { + var game = GameInformationDatabase.GetGameById(gameEnum); + var gamePathInfo = _settingsService.CurrentSettings.GameDirectories.FirstOrDefault(x => x.Game == game.Type); + var gameName = game.DisplayName; + + if (gamePathInfo == null || string.IsNullOrWhiteSpace(gamePathInfo.Path)) + { + var errorMessage = $"Unable to load pack files for {gameName} because no game directory is configured."; + _logger.Here().Error(errorMessage); + return null; + } + + var gameDataFolder = gamePathInfo.Path; + var fullPackFilePaths = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); + + // When loading ca pack packs, we want to use the CA resolver as its faster. + // If there is no manifest file, we need to use the duplicate resolver as it loads all file in the folder. + // There might be custom mods in there that does not follow the rules! + IDuplicateFileResolver packfileResolver = new CaPackDuplicateFileResolver(); + if (manifestFileFound == false) + { + _logger.Here().Warning($"Loading pack files for {gameName}, which does not uses manifest.txt. If there are MODs in the game folder, this might cause issues!"); + packfileResolver = new CustomPackDuplicateFileResolver(); + } + + return CreateFromCollection(PackFileContainerType.Cached, gameDataFolder, fullPackFilePaths, $"All Game Packs - {gameName}", true, packfileResolver); + } + + + public IPackFileContainer CreateFromCollection(PackFileContainerType type, string packFileSystemPath, List fullPackFilePaths, string createdPackFileName, bool loadAsReadOnly, IDuplicateFileResolver duplicateFileResolver) + { + if(type == PackFileContainerType.Cached && loadAsReadOnly == false) + throw new InvalidOperationException($"Cannot load as writable if loading from cache. Caching is only supported for read-only containers. PackFile {createdPackFileName}"); + + + var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(fullPackFilePaths); + var cachePrefix = createdPackFileName; + var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(cachePrefix, fingerprint); + + if (type == PackFileContainerType.Cached) + { + var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); + if (cached != null) + return cached; + + //var cacheInvalidReason = GetCacheInvalidReason(cacheFilePath, fingerprint); + //_logger.Here().Information($"Cache invalid reason for {gameName}: {cacheInvalidReason}"); + //var reasonMessage = string.Format(_localizationManager.Get("PackFileCache.InvalidReason." + cacheInvalidReason), gameName); + //var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); + //var cacheDescription = _localizationManager.Get("PackFileCache.Description"); + _standardDialogs.ShowDialogBox("Failed to load from cache - make better error later"); + + } + + using (_standardDialogs.ShowWaitCursor()) + { + + var container = LoadPackFilesFromDisk(createdPackFileName, fullPackFilePaths, duplicateFileResolver); + container.Name = createdPackFileName; + container.IsCaPackFile = loadAsReadOnly; + container.SystemFilePath = packFileSystemPath; + + if (type == PackFileContainerType.Cached) + { + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); + PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); + // Do we want to set it to the cached version? It should be the same data, just in a different format. + } + + return container; + } + } + + + + + + + + + private static PackFileContainer LoadPackFilesFromDisk(string createdPackFileName, List fullPackFilePaths, IDuplicateFileResolver packfileResolver) + { + var packList = new ConcurrentBag(); + var packsCompressionStats = new ConcurrentDictionary(); + + Parallel.ForEach(fullPackFilePaths, packFilePath => + { + var path = packFilePath; + if (File.Exists(path)) + { + using var fileStream = File.OpenRead(path); + using var reader = new BinaryReader(fileStream, Encoding.ASCII); + + var packFileSize = new FileInfo(path).Length; + var pack = PackFileSerializerLoader.Load(path, packFileSize, reader, packfileResolver); + packList.Add(pack); + + PackFileLog.LogPackCompression(pack); + var packCompressionStats = PackFileLog.GetCompressionInformation(pack); + foreach (var kvp in packCompressionStats) + { + packsCompressionStats.AddOrUpdate( + kvp.Key, + _ => new CompressionInformation(kvp.Value.DiskSize, kvp.Value.UncompressedSize), + (_, existingStats) => new CompressionInformation( + existingStats.DiskSize + kvp.Value.DiskSize, + existingStats.UncompressedSize + kvp.Value.UncompressedSize)); + } + } + else + _logger.Here().Warning($"{createdPackFileName} pack file '{path}' not found, loading skipped"); + } + ); + + PackFileLog.LogPacksCompression(packsCompressionStats); + + // If there is only one packfile - we dont need to sort. Just return it. + // Be aware, that when we create a new PackFileContainer in the case of multiple packfiles, we will lose the original header information of the first packfile. + // This is because we need to create a new header for the new container. This should not be an issue, but its something to be aware of. + if (packList.Count == 1) + return packList.First(); + + var mergedPackFile = new PackFileContainer(createdPackFileName); + var packFilesOrderedByGroup = packList.GroupBy(x => x.Header.LoadOrder).OrderBy(x => x.Key); + + foreach (var group in packFilesOrderedByGroup) + { + var packFilesOrderedByName = group.OrderBy(x => x.Name); + foreach (var packfile in packFilesOrderedByName) + { + if (string.IsNullOrWhiteSpace(packfile.SystemFilePath) == false) + mergedPackFile.SourcePackFilePaths.Add(packfile.SystemFilePath); + mergedPackFile.MergePackFileContainer(packfile); + } + } + + return mergedPackFile; + } + } +} diff --git a/Shared/SharedCore/Shared.Core/Shared.Core.csproj b/Shared/SharedCore/Shared.Core/Shared.Core.csproj index 13f699d45..5b41f4569 100644 --- a/Shared/SharedCore/Shared.Core/Shared.Core.csproj +++ b/Shared/SharedCore/Shared.Core/Shared.Core.csproj @@ -5,6 +5,7 @@ enable enable true + preview diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index a8503b95c..e4367905f 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -192,10 +192,10 @@ public void ComputeFingerprint_DeterministicForSameInputs() Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); File.WriteAllText(Path.Combine(packDir, "b.pack"), "data_b"); - var packFiles = new List { "a.pack", "b.pack" }; + var packFiles = new List { Path.Combine(packDir, "a.pack"), Path.Combine(packDir, "b.pack") }; - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); Assert.That(fp1, Is.EqualTo(fp2)); } @@ -206,13 +206,14 @@ public void ComputeFingerprint_ChangesWhenFileChanges() var packDir = Path.Combine(_tempDir, "gamedata2"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); - var packFiles = new List { "a.pack" }; + + var packFiles = new List { Path.Combine(packDir,"a.pack") }; - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a_modified_longer"); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); Assert.That(fp1, Is.Not.EqualTo(fp2)); } @@ -417,19 +418,20 @@ public void SaveCache_SkipsNonPackedFileSources() } [Test] + public void ComputeFingerprint_IgnoresMissingPackFiles() { var packDir = Path.Combine(_tempDir, "partial"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "exists.pack"), "data"); - var packFiles = new List { "exists.pack", "missing.pack" }; - var fp = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var packFiles = new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }; + var fp = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); Assert.That(fp, Is.Not.Null.And.Not.Empty); // Same result regardless of missing file in list - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, new List { "exists.pack", "missing.pack" }); + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); Assert.That(fp, Is.EqualTo(fp2)); } @@ -442,12 +444,9 @@ public void ComputeFingerprint_OrderIndependent() File.WriteAllText(Path.Combine(packDir, "beta.pack"), "bbb"); File.WriteAllText(Path.Combine(packDir, "gamma.pack"), "ccc"); - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "gamma.pack", "alpha.pack", "beta.pack" }); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "alpha.pack", "beta.pack", "gamma.pack" }); - var fp3 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "beta.pack", "gamma.pack", "alpha.pack" }); + var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); + var fp3 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); Assert.That(fp1, Is.EqualTo(fp2)); Assert.That(fp2, Is.EqualTo(fp3)); @@ -456,7 +455,7 @@ public void ComputeFingerprint_OrderIndependent() [Test] public void GetCacheFilePath_SanitizesInvalidChars() { - var path = PackFileContainerCacheHelper.GetCacheFilePath(@"c:\game", "Game:Name/WithChars", "abc123"); + var path = PackFileContainerCacheHelper.GetCacheFilePath("Game:Name/WithChars", "abc123"); var fileName = Path.GetFileName(path); Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs index 31a1b0c3d..e08b85a74 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -46,10 +46,7 @@ public void TearDown() Directory.Delete(_tempGameDir, true); } - private PackFileContainerLoader CreateLoader() - { - return new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager); - } + private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager); [Test] public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() @@ -57,7 +54,7 @@ public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() _settingsService.CurrentSettings.GameDirectories.Clear(); var loader = CreateLoader(); - var result = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var result = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(result, Is.Null); } @@ -67,7 +64,7 @@ public void LoadAllCaFiles_NoCacheExists_ShowsNotFoundDialogAndWaitCursor() { var loader = CreateLoader(); - var result = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var result = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(result, Is.Not.Null); @@ -85,13 +82,13 @@ public void LoadAllCaFiles_CacheExists_NoDialogShown() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); _dialogs.Invocations.Clear(); // Second call should use cache - no dialogs - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Never); @@ -104,11 +101,11 @@ public void LoadAllCaFiles_CacheCorrupted_ShowsCorruptedDialog() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); // Find and corrupt the cache file - var cacheDir = Shared.Core.Misc.DirectoryHelper.CacheDirectory; + var cacheDir = DirectoryHelper.CacheDirectory; var cacheFiles = Directory.GetFiles(cacheDir, "*.db", SearchOption.AllDirectories); Assert.That(cacheFiles.Length, Is.GreaterThan(0), "Cache file should have been created"); @@ -122,7 +119,7 @@ public void LoadAllCaFiles_CacheCorrupted_ShowsCorruptedDialog() _dialogs.Invocations.Clear(); // Load again — should detect corruption - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); // Single combined dialog: reason + building message @@ -136,7 +133,7 @@ public void LoadAllCaFiles_DataChanged_ShowsDialogsAndRebuildsCache() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); // Add a new pack file to the game dir to change the fingerprint @@ -146,7 +143,7 @@ public void LoadAllCaFiles_DataChanged_ShowsDialogsAndRebuildsCache() _dialogs.Invocations.Clear(); // Load again — fingerprint changed, new fingerprint has no cache file - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); // Single combined dialog: reason + building message @@ -180,7 +177,7 @@ public void LoadAllCaFiles_WaitCursorActive_DuringBuild() .Callback(() => waitCursorDisposed = true); var loader = CreateLoader(); - loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(dialogShownBeforeWaitCursor, Is.True, "Dialog should be shown before wait cursor starts"); Assert.That(waitCursorCreated, Is.True, "Wait cursor should have been created"); diff --git a/Testing/Shared/Shared/AssetEditorTestRunner.cs b/Testing/Shared/Shared/AssetEditorTestRunner.cs index a3362318f..126e8d829 100644 --- a/Testing/Shared/Shared/AssetEditorTestRunner.cs +++ b/Testing/Shared/Shared/AssetEditorTestRunner.cs @@ -49,7 +49,7 @@ public AssetEditorTestRunner(GameTypeEnum gameEnum = GameTypeEnum.Warhammer3, bo public IPackFileContainer? LoadPackFile(string path, bool createOutputPackFile = true) { var loader = ServiceProvider.GetRequiredService(); - var container = loader.Load(path); + var container = loader.CreateFromPackFile(PackFileContainerType.Normal, path, false); PackFileService.AddContainer(container); if (createOutputPackFile) @@ -77,7 +77,7 @@ public T GetRequiredServiceInCurrentEditorScope() public IPackFileContainer LoadFolderPackFile(string path) { var loader = ServiceProvider.GetRequiredService(); - var container = loader.LoadSystemFolderAsPackFileContainer(path); + var container = loader.CreateFromSystemFolder(path); PackFileService.AddContainer(container); return container; diff --git a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs index cf77e6172..0f73a8f9f 100644 --- a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs +++ b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs @@ -13,7 +13,7 @@ public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = G { var pfs = new PackFileService(null); var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager()); - var container = loader.LoadSystemFolderAsPackFileContainer(path); + var container = loader.CreateFromSystemFolder(path); container.IsCaPackFile = true; pfs.AddContainer(container); @@ -25,7 +25,7 @@ public static IPackFileService CreateFromFolder(GameTypeEnum selectedGame, strin var pfs = new PackFileService(null); var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager()); - var container = loader.LoadSystemFolderAsPackFileContainer(PathHelper.GetDataFolder(path)); + var container = loader.CreateFromSystemFolder(PathHelper.GetDataFolder(path)); container.IsCaPackFile = true; pfs.AddContainer(container); return pfs; From b53633ebb12e1ced3ec9eb574351cdc7ec9b372d Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 1 Jun 2026 18:32:56 +0200 Subject: [PATCH 2/9] Code --- .../Containers/CachedPackFileContainer.cs | 47 +++++-- .../Models/Containers/PackFileContainer.cs | 5 + .../PackFiles/Models/IPackFileContainer.cs | 4 +- .../Models/IPackFileContainerInternal.cs | 2 + ...kFileContainerTests_GetAllFilesByFolder.cs | 128 ++++++++++++++++++ .../PackFileContainerTests_TestBase.cs | 45 ++---- .../Shared.CoreTest/Shared.CoreTest.csproj | 10 +- .../Utility/PackFileTreeBuilder.cs | 80 ++++++++--- 8 files changed, 256 insertions(+), 65 deletions(-) create mode 100644 Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 98cfc5055..91cca546b 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using System.Collections.Immutable; +using System.Diagnostics; using Microsoft.EntityFrameworkCore; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models.FileSources; @@ -131,23 +132,45 @@ public bool ContainsFile(string path) return entries.Select(e => (e.RelativePath, ToPackFile(e))).ToList(); } + public SortedDictionary> GetAllFilesByFolder() + { + var time = Stopwatch.StartNew(); + lock (_dbLock) + { + + var rows = _db.Files + .Select(f => new + { + f.FolderPath, + f.FileName + }) + .OrderBy(f => f.FolderPath) + .ThenBy(f => f.FileName) + .ToList(); + + var result = rows + .GroupBy(f => f.FolderPath) + .ToList(); + + var sorted = new SortedDictionary>(StringComparer.InvariantCultureIgnoreCase); + foreach (var re in result) + { + sorted[re.Key] = re.Select(f => f.FileName).OrderByDescending(f=>f).ToList(); + + + } + + _logger.Here().Information("Getting all files from cached container took {ElapsedMilliseconds} ms", time.ElapsedMilliseconds); + return sorted; + } + } + public Dictionary GetAllFiles() { var time = Stopwatch.StartNew(); List entries; lock (_dbLock) { - //var x = _db.Files - // .GroupBy(x=>x.FolderPath) - // .Select(g => new - // { - // Folder = g.Key, - // FileList = g.Select(x => x.FileName).ToList() - // }) - // .ToList(); - // - // - entries = _db.Files.ToList(); } _logger.Here().Information("Getting all files from cached container took {ElapsedMilliseconds} ms", time.ElapsedMilliseconds); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index 4978d807e..4e8689250 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -293,5 +293,10 @@ private static string BuildPackPath(string? directoryPath, string fileName) return PathNormalization.NormalizeFileName(fullPath); } + + public SortedDictionary> GetAllFilesByFolder() + { + throw new NotImplementedException(); + } } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs index 04ec72cf6..37c37a8b4 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs @@ -13,8 +13,8 @@ public interface IPackFileContainer string? GetFullPath(PackFile file); Dictionary GetAllFiles(); - - + SortedDictionary> GetAllFilesByFolder(); + List<(string Path, PackFile File)> SearchFiles(string? textFilter, IReadOnlyList? extensions); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs index e1cc13812..4b62a5482 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs @@ -17,5 +17,7 @@ internal interface IPackFileContainerInternal : IPackFileContainer List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention); List GetSubDirectories(string directoryPath); List<(string Path, PackFile File)> GetDirectoryContent(string directoryPath); + + } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs new file mode 100644 index 000000000..9f1c4fcbb --- /dev/null +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs @@ -0,0 +1,128 @@ +using Shared.Core.PackFiles.Models.Containers; + +namespace Shared.CoreTest.PackFiles.Models.Containers +{ + [TestFixture(typeof(CachedPackFileContainer))] + [TestFixture(typeof(PackFileContainer))] + internal class PackFileContainerTests_GetAllFilesByFolder : PackFileContainerTests_TestBase + { + public PackFileContainerTests_GetAllFilesByFolder(Type containerType) : base(containerType) { } + + [Test] + public void GetAllFilesByFolder_ReturnsExpectedFolderCount() + { + var result = _container.GetAllFilesByFolder(); + + // Folders: folder, other, audio, (root), models, models\textures, models\textures\specular, scripts, folder_a, folder_b, compressed + Assert.That(result.Count, Is.GreaterThanOrEqualTo(10)); + } + + [Test] + public void GetAllFilesByFolder_KeysAreSorted() + { + var result = _container.GetAllFilesByFolder(); + var keys = result.Keys.ToList(); + + for (var i = 1; i < keys.Count; i++) + { + Assert.That(string.Compare(keys[i - 1], keys[i], StringComparison.InvariantCultureIgnoreCase), Is.LessThan(0), + $"Keys not sorted: '{keys[i - 1]}' should come before '{keys[i]}'"); + } + } + + [Test] + public void GetAllFilesByFolder_ContainsExpectedFolders() + { + var result = _container.GetAllFilesByFolder(); + + Assert.That(result.ContainsKey("folder"), Is.True); + Assert.That(result.ContainsKey("other"), Is.True); + Assert.That(result.ContainsKey("audio"), Is.True); + Assert.That(result.ContainsKey("models"), Is.True); + Assert.That(result.ContainsKey(@"models\textures"), Is.True); + Assert.That(result.ContainsKey(@"models\textures\specular"), Is.True); + Assert.That(result.ContainsKey("scripts"), Is.True); + Assert.That(result.ContainsKey("folder_a"), Is.True); + Assert.That(result.ContainsKey("folder_b"), Is.True); + Assert.That(result.ContainsKey("compressed"), Is.True); + } + + [Test] + public void GetAllFilesByFolder_FolderContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var audioFiles = result["audio"]; + Assert.That(audioFiles, Does.Contain("sound.wem")); + Assert.That(audioFiles, Does.Contain("music.wem")); + Assert.That(audioFiles, Does.Contain("battle_sound.wem")); + Assert.That(audioFiles.Count, Is.EqualTo(3)); + } + + [Test] + public void GetAllFilesByFolder_ModelsFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var modelsFiles = result["models"]; + Assert.That(modelsFiles, Does.Contain("unit.model")); + Assert.That(modelsFiles, Does.Contain("vehicle.model")); + Assert.That(modelsFiles.Count, Is.EqualTo(2)); + } + + [Test] + public void GetAllFilesByFolder_NestedFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var textureFiles = result[@"models\textures"]; + Assert.That(textureFiles, Does.Contain("diffuse.dds")); + Assert.That(textureFiles, Does.Contain("normal.dds")); + Assert.That(textureFiles.Count, Is.EqualTo(2)); + } + + [Test] + public void GetAllFilesByFolder_DeeplyNestedFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var specularFiles = result[@"models\textures\specular"]; + Assert.That(specularFiles, Does.Contain("gloss.dds")); + Assert.That(specularFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_SingleFileFolder_ContainsCorrectFile() + { + var result = _container.GetAllFilesByFolder(); + + var folderFiles = result["folder"]; + Assert.That(folderFiles, Does.Contain("file.txt")); + Assert.That(folderFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_FilesAreNotDuplicatedAcrossFolders() + { + var result = _container.GetAllFilesByFolder(); + + // shared.txt exists in both folder_a and folder_b + var folderAFiles = result["folder_a"]; + var folderBFiles = result["folder_b"]; + Assert.That(folderAFiles, Does.Contain("shared.txt")); + Assert.That(folderBFiles, Does.Contain("shared.txt")); + Assert.That(folderAFiles.Count, Is.EqualTo(1)); + Assert.That(folderBFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_TotalFileCountMatchesGetAllFiles() + { + var result = _container.GetAllFilesByFolder(); + var totalFiles = result.Values.Sum(files => files.Count); + var allFiles = _container.GetAllFiles(); + + Assert.That(totalFiles, Is.EqualTo(allFiles.Count)); + } + } +} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index c85d3769d..688746468 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -31,36 +31,21 @@ public void Setup() sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; - sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", - new PackedFileSource(parent, 100, 200, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", - new PackedFileSource(parent, 300, 400, false, true, CompressionFormat.Lz4, 800))); - sourceContainer.AddOrUpdateFile("audio\\sound.wem", new PackFile("sound.wem", - new PackedFileSource(parent, 700, 500, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt", - new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\unit.model", new PackFile("unit.model", - new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\vehicle.model", new PackFile("vehicle.model", - new PackedFileSource(parent, 30, 40, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\diffuse.dds", new PackFile("diffuse.dds", - new PackedFileSource(parent, 70, 50, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\normal.dds", new PackFile("normal.dds", - new PackedFileSource(parent, 120, 60, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\specular\\gloss.dds", new PackFile("gloss.dds", - new PackedFileSource(parent, 180, 30, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\music.wem", new PackFile("music.wem", - new PackedFileSource(parent, 210, 100, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\battle_sound.wem", new PackFile("battle_sound.wem", - new PackedFileSource(parent, 400, 300, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("scripts\\campaign_script.lua", new PackFile("campaign_script.lua", - new PackedFileSource(parent, 850, 80, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_a\\shared.txt", new PackFile("shared.txt", - new PackedFileSource(parent, 900, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_b\\shared.txt", new PackFile("shared.txt", - new PackedFileSource(parent, 910, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("compressed\\data.bin", new PackFile("data.bin", - new PackedFileSource(parent, 1000, 500, true, true, CompressionFormat.Lz4, 2000))); + sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", new PackedFileSource(parent, 100, 200, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 300, 400, false, true, CompressionFormat.Lz4, 800))); + sourceContainer.AddOrUpdateFile("audio\\sound.wem", new PackFile("sound.wem", new PackedFileSource(parent, 700, 500, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt",new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\unit.model", new PackFile("unit.model", new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\vehicle.model", new PackFile("vehicle.model", new PackedFileSource(parent, 30, 40, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\textures\\diffuse.dds", new PackFile("diffuse.dds",new PackedFileSource(parent, 70, 50, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\textures\\normal.dds", new PackFile("normal.dds", new PackedFileSource(parent, 120, 60, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\textures\\specular\\gloss.dds", new PackFile("gloss.dds",new PackedFileSource(parent, 180, 30, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("audio\\music.wem", new PackFile("music.wem", new PackedFileSource(parent, 210, 100, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("audio\\battle_sound.wem", new PackFile("battle_sound.wem", new PackedFileSource(parent, 400, 300, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("scripts\\campaign_script.lua", new PackFile("campaign_script.lua",new PackedFileSource(parent, 850, 80, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("folder_a\\shared.txt", new PackFile("shared.txt",new PackedFileSource(parent, 900, 10, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("folder_b\\shared.txt", new PackFile("shared.txt", new PackedFileSource(parent, 910, 20, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("compressed\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 1000, 500, true, true, CompressionFormat.Lz4, 2000))); if (_useCachedContainer) { diff --git a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj index 27966a8f4..f63e7efe9 100644 --- a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj +++ b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj @@ -8,6 +8,12 @@ false + + + + + + all @@ -32,8 +38,4 @@ - - - - diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs index 5363e8731..c387c7414 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using Shared.Core.PackFiles.Models; +using SharpDX.MediaFoundation; namespace Shared.Ui.BaseDialogs.PackFileTree.Utility { @@ -45,34 +47,78 @@ public int GetHashCode(PathPrefixKey obj) public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer container, bool skipWemFiles) { - var allFiles = container.GetAllFiles(); - var filesByFolder = GroupFilesByFolder(allFiles, skipWemFiles); - var directoryMap = new Dictionary(filesByFolder.Count + 1, PathPrefixKeyComparer.Ordinal) - { - [PathPrefixKey.Empty] = root - }; - var childrenByParent = new Dictionary>(filesByFolder.Count + 1); - var pendingDirectories = new List<(string FolderName, PathPrefixKey FullFolderPath)>(8); + // Get all files sorted by folders. The result is always sorted. + var fileByFolders = container.GetAllFilesByFolder(); + bool addFiles = true; - foreach (var folderPath in filesByFolder.Keys) + var nodeLookUp = new Dictionary(); + + foreach (var folder in fileByFolders) { + var folderPath = folder.Key; + if (folderPath.Length == 0) continue; - EnsureDirectoryPath(root, folderPath, directoryMap, pendingDirectories, childrenByParent); + // If this exact path already exists, skip folder creation + if (!nodeLookUp.ContainsKey(folderPath)) + { + // Search from the right to find the deepest existing ancestor. + // Since keys are sorted, most parent paths will already be in the map. + var foldersToCreate = new List { folderPath }; + var parentNode = root; + var lastSep = folderPath.LastIndexOf(Path.DirectorySeparatorChar); + + while (lastSep != -1) + { + var parentPath = folderPath.Substring(0, lastSep); + if (nodeLookUp.TryGetValue(parentPath, out var foundNode)) + { + parentNode = foundNode; + break; + } + + foldersToCreate.Add(parentPath); + lastSep = parentPath.LastIndexOf(Path.DirectorySeparatorChar); + } + + // Create missing folders from shallowest to deepest + for (var i = foldersToCreate.Count - 1; i >= 0; i--) + { + var path = foldersToCreate[i]; + var sep = path.LastIndexOf(Path.DirectorySeparatorChar); + var folderName = sep == -1 ? path : path.Substring(sep + 1); + + var newNode = new TreeNode(folderName, NodeType.Directory, parentNode); + parentNode.AddChild(newNode); + nodeLookUp[path] = newNode; + parentNode = newNode; + } + } } - foreach (var folderEntry in filesByFolder) + // Add files after all folders have been created + if (addFiles) { - var parentNode = directoryMap[folderEntry.Key]; - foreach (var file in folderEntry.Value) + foreach (var folder in fileByFolders) { - var fileNode = new TreeNode(file.Name, NodeType.File, parentNode); - AddChildForBuild(parentNode, fileNode, childrenByParent); + if (folder.Key.Length == 0) + { + foreach (var fileName in folder.Value) + { + var fileNode = new TreeNode(fileName, NodeType.File, root); + root.AddChild(fileNode); + } + } + + var folderNode = nodeLookUp[folder.Key]; + foreach (var fileName in folder.Value) + { + var fileNode = new TreeNode(fileName, NodeType.File, folderNode); + folderNode.AddChild(fileNode); + } } } - - FinalizeTree(root, childrenByParent); } private static Dictionary> GroupFilesByFolder(Dictionary allFiles, bool skipWemFiles) From 41faf69bb1e4c45e41df56128640263d501107a6 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 1 Jun 2026 18:50:55 +0200 Subject: [PATCH 3/9] Working --- .../Models/Containers/PackFileContainer.cs | 19 ++- .../PackFileTree/PackFileBrowserViewModel.cs | 2 - .../Utility/PackFileTreeBuilder.cs | 129 +----------------- 3 files changed, 21 insertions(+), 129 deletions(-) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index 4e8689250..e9ed76807 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -296,7 +296,24 @@ private static string BuildPackPath(string? directoryPath, string fileName) public SortedDictionary> GetAllFilesByFolder() { - throw new NotImplementedException(); + var result = new SortedDictionary>(StringComparer.InvariantCultureIgnoreCase); + + foreach (var fullPath in FileList.Keys) + { + var lastSep = fullPath.LastIndexOf('\\'); + var folder = lastSep == -1 ? string.Empty : fullPath.Substring(0, lastSep); + var fileName = lastSep == -1 ? fullPath : fullPath.Substring(lastSep + 1); + + if (!result.TryGetValue(folder, out var files)) + { + files = new List(); + result[folder] = files; + } + + files.Add(fileName); + } + + return result; } } } diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs index e62bcf7ad..dc44b79c8 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.ComponentModel; using System.IO; using System.Linq; -using System.Windows.Input; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using Shared.Core.Events; diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs index c387c7414..66f1efb61 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs @@ -1,55 +1,16 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Runtime.InteropServices; using Shared.Core.PackFiles.Models; -using SharpDX.MediaFoundation; namespace Shared.Ui.BaseDialogs.PackFileTree.Utility { public static class PackFileTreeBuilder { - private readonly record struct PathPrefixKey(string Path, int Length) - { - public static readonly PathPrefixKey Empty = new(string.Empty, 0); - - public ReadOnlySpan Span => Path.AsSpan(0, Length); - } - - private sealed class PathPrefixKeyComparer : IEqualityComparer - { - public static readonly PathPrefixKeyComparer Ordinal = new(); - - public bool Equals(PathPrefixKey x, PathPrefixKey y) - { - return x.Length == y.Length && x.Span.SequenceEqual(y.Span); - } - - public int GetHashCode(PathPrefixKey obj) - { - var hash = new HashCode(); - foreach (var ch in obj.Span) - hash.Add(ch); - - return hash.ToHashCode(); - } - } - - private static readonly Comparison TreeNodeComparison = (left, right) => - { - var nodeTypeComparison = left.NodeType.CompareTo(right.NodeType); - if (nodeTypeComparison != 0) - return nodeTypeComparison; - - return StringComparer.CurrentCultureIgnoreCase.Compare(left.Name, right.Name); - }; - public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer container, bool skipWemFiles) { // Get all files sorted by folders. The result is always sorted. var fileByFolders = container.GetAllFilesByFolder(); - bool addFiles = true; + var addFiles = true; var nodeLookUp = new Dictionary(); @@ -109,6 +70,7 @@ public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer containe var fileNode = new TreeNode(fileName, NodeType.File, root); root.AddChild(fileNode); } + continue; } var folderNode = nodeLookUp[folder.Key]; @@ -120,90 +82,5 @@ public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer containe } } } - - private static Dictionary> GroupFilesByFolder(Dictionary allFiles, bool skipWemFiles) - { - var filesByFolder = new Dictionary>(PathPrefixKeyComparer.Ordinal) - { - [PathPrefixKey.Empty] = [] - }; - - foreach (var item in allFiles) - { - var path = item.Key; - if (skipWemFiles && path.EndsWith(".wem", StringComparison.OrdinalIgnoreCase)) - continue; - - var separatorIndex = FindLastDirectorySeparatorIndex(path.AsSpan()); - var folderPath = separatorIndex == -1 - ? PathPrefixKey.Empty - : new PathPrefixKey(path, separatorIndex); - - ref var files = ref CollectionsMarshal.GetValueRefOrAddDefault(filesByFolder, folderPath, out _); - files ??= []; - files.Add(item.Value); - } - - return filesByFolder; - } - - private static TreeNode EnsureDirectoryPath(TreeNode root, PathPrefixKey folderPath, Dictionary directoryMap, List<(string FolderName, PathPrefixKey FullFolderPath)> pendingDirectories, Dictionary> childrenByParent) - { - if (directoryMap.TryGetValue(folderPath, out var existingDirectory)) - return existingDirectory; - - pendingDirectories.Clear(); - var currentFolderPath = folderPath; - while (currentFolderPath.Length > 0 && !directoryMap.TryGetValue(currentFolderPath, out existingDirectory)) - { - var currentPathSpan = currentFolderPath.Span; - var separatorIndex = FindLastDirectorySeparatorIndex(currentPathSpan); - var folderName = separatorIndex == -1 - ? currentFolderPath.Path[..currentFolderPath.Length] - : currentFolderPath.Path.Substring(separatorIndex + 1, currentFolderPath.Length - separatorIndex - 1); - pendingDirectories.Add((folderName, currentFolderPath)); - currentFolderPath = separatorIndex == -1 - ? PathPrefixKey.Empty - : new PathPrefixKey(currentFolderPath.Path, separatorIndex); - } - - var parentNode = currentFolderPath.Length == 0 ? root : directoryMap[currentFolderPath]; - for (var i = pendingDirectories.Count - 1; i >= 0; i--) - { - var currentDirectory = pendingDirectories[i]; - var currentNode = new TreeNode(currentDirectory.FolderName, NodeType.Directory, parentNode); - AddChildForBuild(parentNode, currentNode, childrenByParent); - directoryMap[currentDirectory.FullFolderPath] = currentNode; - parentNode = currentNode; - } - - return parentNode; - } - - private static void AddChildForBuild(TreeNode parent, TreeNode child, Dictionary> childrenByParent) - { - child.Parent = parent; - - ref var children = ref CollectionsMarshal.GetValueRefOrAddDefault(childrenByParent, parent, out _); - children ??= []; - children.Add(child); - } - - private static int FindLastDirectorySeparatorIndex(ReadOnlySpan path) - { - return path.LastIndexOfAny(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - } - - private static void FinalizeTree(TreeNode node, Dictionary> childrenByParent) - { - if (!childrenByParent.TryGetValue(node, out var children) || children.Count == 0) - return; - - children.Sort(TreeNodeComparison); - node.SetChildren(children); - - foreach (var child in children) - FinalizeTree(child, childrenByParent); - } } } From 7e46815c9b96bb1d3ad24b584e3c7b8bea745431 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 1 Jun 2026 19:04:56 +0200 Subject: [PATCH 4/9] Cleanup --- .../Containers/CachedPackFileContainer.cs | 31 ---------------- .../Models/Containers/PackFileContainer.cs | 19 ---------- .../Models/IPackFileContainerInternal.cs | 1 - .../Utility/PackFileServiceUtility.cs | 26 ------------- ...kFileContainerTests_GetDirectoryContent.cs | 8 ++-- ...ackFileContainerTests_GetSubDirectories.cs | 35 ------------------ .../PackFileContainerCacheHelperTests.cs | 9 ++--- .../Utility/PackFileServiceUtilityTests.cs | 37 ++++++++++--------- 8 files changed, 27 insertions(+), 139 deletions(-) delete mode 100644 Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 91cca546b..924fedd5c 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -243,37 +243,6 @@ public Dictionary GetAllFiles() return files; } - public List GetSubDirectories(string directoryPath) - { - var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; - var prefixLength = prefix.Length; - - List folderPaths; - lock (_dbLock) - { - folderPaths = _db.Files - .Where(f => string.IsNullOrEmpty(directoryPath) - ? f.FolderPath != "" - : f.FolderPath.StartsWith(prefix)) - .Select(f => f.FolderPath) - .ToList(); - } - - return folderPaths - .Select(folderPath => string.IsNullOrEmpty(directoryPath) - ? folderPath - : folderPath.Substring(prefixLength)) - .Select(candidate => - { - var separatorIndex = candidate.IndexOf('\\'); - return separatorIndex == -1 ? candidate : candidate.Substring(0, separatorIndex); - }) - .Where(folderName => !string.IsNullOrWhiteSpace(folderName)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - } - public void AddOrUpdateFile(string path, PackFile file) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index e9ed76807..d7cfd8112 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -53,25 +53,6 @@ public void AddOrUpdateFile(string path, PackFile file) return results; } - public List GetSubDirectories(string directoryPath) - { - var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; - var subFolders = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var path in FileList.Keys) - { - if (!string.IsNullOrEmpty(prefix) && !path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) - continue; - - var remainder = string.IsNullOrEmpty(prefix) ? path : path.Substring(prefix.Length); - var separatorIndex = remainder.IndexOf(Path.DirectorySeparatorChar); - if (separatorIndex > 0) - subFolders.Add(remainder.Substring(0, separatorIndex)); - } - - return subFolders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(); - } - public List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention) { extention = extention.ToLower(); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs index 4b62a5482..ddb52d970 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs @@ -15,7 +15,6 @@ internal interface IPackFileContainerInternal : IPackFileContainer void SaveToDisk(string path, bool createBackup, GameInformation gameInformation); List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention); - List GetSubDirectories(string directoryPath); List<(string Path, PackFile File)> GetDirectoryContent(string directoryPath); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs index 4b6c0c5a3..38fddfa7d 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs @@ -2,34 +2,8 @@ namespace Shared.Core.PackFiles.Utility { - public sealed record DirectoryFileEntry(string FileName, PackFile File); - - public sealed record DirectoryEntriesSplitResult( - string DirectoryPath, - IReadOnlyList SubFolders, - IReadOnlyList Files); - public static class PackFileServiceUtility { - public static DirectoryEntriesSplitResult SplitDirectoryEntries(IPackFileContainer container, string directoryPath) - { - var packFileContainerInternal = PackFileService.CastContainer(container); - - var files = packFileContainerInternal.GetDirectoryContent(directoryPath) - .Select(entry => new DirectoryFileEntry(entry.File.Name, entry.File)) - .OrderBy(x => x.FileName, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - - var subFolders = packFileContainerInternal.GetSubDirectories(directoryPath) - .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - - return new DirectoryEntriesSplitResult( - directoryPath, - subFolders, - files); - } - public static List GetAllAnimPacks(IPackFileService pfs) { var animPacks = FindAllWithExtention(pfs, @".animpack"); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs index 8a2ccb56a..b0e09de0c 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs @@ -1,5 +1,4 @@ using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Utility; namespace Shared.CoreTest.PackFiles.Models.Containers { @@ -35,11 +34,10 @@ public void GetDirectoryContent_UnknownFolder_ReturnsEmpty() } [Test] - public void GetDirectoryContent_ComposesWithUtility() + public void GetDirectoryContent_ModelsFolder_ReturnsExpectedFiles() { - var split = PackFileServiceUtility.SplitDirectoryEntries(_container, "models"); - Assert.That(split.Files.Any(x => x.FileName == "unit.model"), Is.True); - Assert.That(split.SubFolders, Does.Contain("textures")); + var files = _container.GetDirectoryContent("models"); + Assert.That(files.Any(x => x.File.Name == "unit.model"), Is.True); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs deleted file mode 100644 index 2aa8aa55c..000000000 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs +++ /dev/null @@ -1,35 +0,0 @@ -using Shared.Core.PackFiles.Models.Containers; - -namespace Shared.CoreTest.PackFiles.Models.Containers -{ - [TestFixture(typeof(CachedPackFileContainer))] - [TestFixture(typeof(PackFileContainer))] - internal class PackFileContainerTests_GetSubDirectories : PackFileContainerTests_TestBase - { - public PackFileContainerTests_GetSubDirectories(Type containerType) : base(containerType) { } - - [Test] - public void GetSubDirectories_Root_ReturnsImmediateSortedFolders() - { - var folders = _container.GetSubDirectories(""); - Assert.That(folders, Does.Contain("audio")); - Assert.That(folders, Does.Contain("models")); - - var sorted = folders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(); - Assert.That(folders, Is.EqualTo(sorted)); - } - - [Test] - public void GetSubDirectories_NestedFolder_ReturnsOnlyChildren() - { - var folders = _container.GetSubDirectories(@"models\textures"); - Assert.That(folders, Is.EqualTo(new[] { "specular" })); - } - - [Test] - public void GetSubDirectories_UnknownFolder_ReturnsEmpty() - { - Assert.That(_container.GetSubDirectories("missing\\folder"), Is.Empty); - } - } -} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index e4367905f..225eb4ece 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -385,12 +385,11 @@ public void SaveCache_StoresFolderPathCorrectly() var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); - var rootContent = PackFileServiceUtility.SplitDirectoryEntries(loaded, ""); - Assert.That(rootContent.Files.Any(f => f.FileName == "root_file.txt"), Is.True); - Assert.That(rootContent.SubFolders, Does.Contain("a")); + var rootFiles = loaded.GetDirectoryContent(""); + Assert.That(rootFiles.Any(f => f.File.Name == "root_file.txt"), Is.True); - var deepContent = PackFileServiceUtility.SplitDirectoryEntries(loaded, "a\\b\\c"); - Assert.That(deepContent.Files.Any(f => f.FileName == "file.txt"), Is.True); + var deepFiles = loaded.GetDirectoryContent("a\\b\\c"); + Assert.That(deepFiles.Any(f => f.File.Name == "file.txt"), Is.True); } [Test] diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs index 2de8df6a0..054f64e3e 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs @@ -1,48 +1,51 @@ -using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Utility; -using Shared.Core.Settings; namespace Shared.CoreTest.PackFiles.Utility { internal class PackFileServiceUtilityTests { [Test] - public void SplitDirectoryEntries_ComposesSortedFoldersAndFiles() + public void GetDirectoryContent_ReturnsSortedFiles() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, "texture"); + var files = container.GetDirectoryContent("texture") + .Select(x => x.File.Name) + .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) + .ToList(); - Assert.That(result.DirectoryPath, Is.EqualTo("texture")); - Assert.That(result.SubFolders, Is.EqualTo(new[] { "mesha", "meshb" })); - Assert.That(result.Files.Select(x => x.FileName), Is.EqualTo(new[] { "alpha.dds", "texture_file.dds" })); + Assert.That(files, Is.EqualTo(new[] { "alpha.dds", "texture_file.dds" })); } + + [Test] - public void SplitDirectoryEntries_RootDirectory_ReturnsTopLevelFoldersAndFiles() + public void GetDirectoryContent_RootDirectory_ReturnsTopLevelFiles() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, ""); + var files = container.GetDirectoryContent("") + .Select(x => x.File.Name) + .ToList(); - Assert.That(result.DirectoryPath, Is.EqualTo(string.Empty)); - Assert.That(result.SubFolders, Is.EqualTo(new[] { "audio", "texture" })); - Assert.That(result.Files.Select(x => x.FileName), Is.EqualTo(new[] { "root.txt" })); + Assert.That(files, Is.EqualTo(new[] { "root.txt" })); } + + [Test] - public void SplitDirectoryEntries_EmptyDirectory_ReturnsEmptyResult() + public void GetDirectoryContent_MissingDirectory_ReturnsEmpty() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, "missing"); - Assert.That(result.DirectoryPath, Is.EqualTo("missing")); - Assert.That(result.SubFolders, Is.Empty); - Assert.That(result.Files, Is.Empty); + Assert.That(container.GetDirectoryContent("missing"), Is.Empty); } + + private static PackFileContainer CreateContainer() { var container = new PackFileContainer("Test"); From f409ea9919bf3eed3aa1ad15a1a9d49fc6a7ec94 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 4 Jun 2026 11:00:00 +0200 Subject: [PATCH 5/9] logging --- .../Models/Containers/PackFileContainer.cs | 26 +++++++++++-- .../Serialization/PackFileSerializerWriter.cs | 39 ++++++++++++++++--- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index d7cfd8112..f65101db8 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -1,4 +1,5 @@ using CommunityToolkit.Diagnostics; +using Shared.Core.ErrorHandling; using Shared.Core.Misc; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization; @@ -9,6 +10,8 @@ namespace Shared.Core.PackFiles.Models.Containers { internal class PackFileContainer : IPackFileContainerInternal { + private static readonly ILogger _logger = Logging.Create(); + public string Name { get; set; } public PFHeader Header { get; set; } public bool IsCaPackFile { get; set; } = false; @@ -25,7 +28,6 @@ public PackFileContainer(string name) Header = new PFHeader(v, PackFileCAType.MOD); } - public int GetFileCount() => FileList.Count; public void AddOrUpdateFile(string path, PackFile file) @@ -99,8 +101,6 @@ public void AddOrUpdateFile(string path, PackFile file) return results; } - - public void MergePackFileContainer(PackFileContainer other) { foreach (var item in other.GetAllFiles()) @@ -232,8 +232,21 @@ public virtual void SaveFileData(PackFile file, byte[] data) public virtual void SaveToDisk(string path, bool createBackup, GameInformation gameInformation) { + _logger.Here().Information("Saving pack file to disk at {Path}, createBackup = {createBackup} Game = {game}", path, createBackup, gameInformation.DisplayName); + if (File.Exists(path) && DirectoryHelper.IsFileLocked(path)) - throw new IOException($"Cannot access {path} because another process has locked it, most likely the game."); + { + var msg = $"Cannot access {path} because another process has locked it, most likely the game."; + _logger.Here().Error(msg); + throw new IOException(msg); + } + + if (File.Exists(path + "_temp") && DirectoryHelper.IsFileLocked(path + "_temp")) + { + var msg = $"Cannot access {path + "_temp"} because another process has locked it, most likely the game."; + _logger.Here().Error(msg); + throw new IOException(msg); + } if (createBackup) SaveUtility.CreateFileBackup(path); @@ -253,11 +266,16 @@ public virtual void SaveToDisk(string path, bool createBackup, GameInformation g PackFileSerializerWriter.SaveToByteArray(path, this, writer, gameInformation); } + _logger.Here().Information("Finished writing pack file to temporary file, now replacing original file with the new one"); File.Delete(path); + + _logger.Here().Information("Original file deleted, now moving temporary file to original file path"); File.Move(path + "_temp", path); SystemFilePath = path; OriginalLoadByteSize = new FileInfo(path).Length; + + _logger.Here().Information("SaveToDisk completed"); } private static string BuildPackPath(string? directoryPath, string fileName) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs index d253b27a6..42f6c78bd 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs @@ -1,4 +1,6 @@ -using System.Text; +using System.Diagnostics; +using System.Text; +using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; @@ -19,8 +21,16 @@ class PackFileWriteInformation(PackFile pf, string fullFileName, long sizePositi static class PackFileSerializerWriter { + private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileSerializerWriter)); + public static void SaveToByteArray(string outputFileName, PackFileContainer container, BinaryWriter writer, GameInformation currentGameInformation) { + var stopWatch = Stopwatch.StartNew(); + + var numFiles = container.GetFileCount(); + var packFileName = container.Name; + _logger.Here().Information("Saving packfile {PackFileName} v={PackFileVersion} with {NumFiles} files to {OutputFileName}. Current game = {Game}", packFileName, container.Header.Version, numFiles, outputFileName, currentGameInformation.DisplayName); + if (container.Header.HasEncryptedData || container.Header.HasEncryptedIndex) throw new InvalidOperationException("Saving encrypted packs is not supported."); @@ -35,11 +45,16 @@ public static void SaveToByteArray(string outputFileName, PackFileContainer cont // Write the core of the file var fileMetaDataTable = BuildMetaDataTable(sortedFiles, container, currentGameInformation); SerializeFileTable(fileMetaDataTable, container, writer); - SerializeFileBlob(outputFileName, fileMetaDataTable, container, writer); + SerializeFileBlob(outputFileName, fileMetaDataTable, container, writer); + + stopWatch.Stop(); + _logger.Here().Information("Saving packfile {PackFileName} completed in {ElapsedMilliseconds} ms", packFileName, stopWatch.ElapsedMilliseconds); } - public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) + static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) { + _logger.Here().Information("Starting write Header"); + var packFileTypeStr = PackFileVersionConverter.ToString(header.Version); // 4 foreach (var c in packFileTypeStr) writer.Write(c); @@ -82,6 +97,8 @@ public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWrit writer.Write(fileNameBytes); writer.Write((byte)0); } + + _logger.Here().Information("Finished writing"); } static long ComputeFileHeaderSpecificByte(PackFileContainer container) @@ -151,6 +168,8 @@ public static FileCompressionInfo DetermineFileCompression(PackFileVersion outpu public static List BuildMetaDataTable(IList> sortedFiles, PackFileContainer container, GameInformation currentGameInformation) { + _logger.Here().Information("Starting to build FileTable"); + var filesToWrite = new List(); foreach (var file in sortedFiles) { @@ -159,11 +178,15 @@ public static List BuildMetaDataTable(IList fileMetaData, PackFileContainer container, BinaryWriter writer) + static void SerializeFileTable(List fileMetaData, PackFileContainer container, BinaryWriter writer) { + _logger.Here().Information("Starting SerializeFileTable"); + // Write file table // FileStartPosition // TimeStamp @@ -193,10 +216,14 @@ public static void SerializeFileTable(List fileMetaDat writer.Write(fileNameBytes); writer.Write((byte)0); // Zero terminator } + + _logger.Here().Information("Finished SerializeFileTable"); } - public static void SerializeFileBlob(string outputFileName, List fileMetaDataTabel, PackFileContainer container, BinaryWriter writer) + static void SerializeFileBlob(string outputFileName, List fileMetaDataTabel, PackFileContainer container, BinaryWriter writer) { + _logger.Here().Information("Starting SerializeFileBlob"); + foreach (var fileMetaData in fileMetaDataTabel) { var packFile = fileMetaData.PackFile; @@ -247,6 +274,8 @@ public static void SerializeFileBlob(string outputFileName, List Date: Sat, 6 Jun 2026 16:37:19 +0200 Subject: [PATCH 6/9] Code --- ...letonAnimationLookUpHelperKarlPackTests.cs | 3 +- .../DependencyInjectionContainer.cs | 4 +- .../PackFileContainerCacheHelper.cs | 37 ++- .../Utility/PackFileContainerLoader.cs | 262 ------------------ .../Utility/PackFileContainerLoader2.cs | 29 +- .../PackFileContainerTests_SearchFiles.cs | 7 +- .../PackFileContainerTests_TestBase.cs | 7 +- .../PackFileContainerCacheHelperTests.cs | 100 ++++--- .../Utility/PackFileContainerLoaderTests.cs | 3 +- .../TestUtility/PackFileSerivceTestHelper.cs | 5 +- 10 files changed, 118 insertions(+), 339 deletions(-) delete mode 100644 Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs index 1713f30b0..b768714de 100644 --- a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs @@ -6,6 +6,7 @@ using Shared.Core.Events.Global; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -32,7 +33,7 @@ public void LoadAndUnload_KarlPack_UpdatesSkeletonAndAnimationLookup() .Returns(() => containers.ToList()); var settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); - var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager()); + var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); var karlPackPath = PathHelper.GetDataFile("Karl_and_celestialgeneral.pack"); var karlContainer = loader.CreateFromPackFile(PackFileContainerType.Normal, karlPackPath, false); diff --git a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs index 625f1ef90..c91e23ecf 100644 --- a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs +++ b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs @@ -6,6 +6,7 @@ using Shared.Core.Events; using Shared.Core.Misc; using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -32,9 +33,9 @@ public override void Register(IServiceCollection services) services.AddSingleton(); services.AddScoped(); services.AddScoped(); - + services.AddScoped(); services.AddSingleton(); services.AddScoped(); @@ -43,6 +44,7 @@ public override void Register(IServiceCollection services) services.AddSingleton(); + services.AddScoped(); services.AddTransient(); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 8fd662361..1f971fbd7 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -9,16 +9,29 @@ namespace Shared.Core.PackFiles.Serialization.CacheDatabase { - internal static class PackFileContainerCacheHelper + interface IPackFileContainerCacheHelper { - private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerCacheHelper)); - public static string GetCacheFilePath(string gameName, string cacheId) + string ComputeFingerprint(List packFileNames); + DbContextOptions CreateDbOptions(SqliteConnection connection); + DbContextOptions CreateDbOptions(string dbFilePath); + DbContextOptions CreateDbOptionsFromConnectionString(string connectionString); + string GetCacheFilePath(string gameName, string cacheId); + CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint); + CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint); + void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions); + CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint); + } + + class PackFileContainerCacheHelper : IPackFileContainerCacheHelper + { + private readonly ILogger _logger = Logging.Create(); + public string GetCacheFilePath(string gameName, string cacheId) { var safeGameName = string.Join("_", gameName.Split(Path.GetInvalidFileNameChars())); return Path.Combine(DirectoryHelper.CacheDirectory, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); } - public static string ComputeFingerprint(List packFileNames) + public string ComputeFingerprint(List packFileNames) { using var sha = SHA256.Create(); var sb = new StringBuilder(); @@ -54,21 +67,21 @@ public static string ComputeFingerprint(List packFileNames) return Convert.ToHexString(hash); } - public static DbContextOptions CreateDbOptions(string dbFilePath) + public DbContextOptions CreateDbOptions(string dbFilePath) { return new DbContextOptionsBuilder() .UseSqlite($"Data Source={dbFilePath};Pooling=False") .Options; } - public static DbContextOptions CreateDbOptionsFromConnectionString(string connectionString) + public DbContextOptions CreateDbOptionsFromConnectionString(string connectionString) { return new DbContextOptionsBuilder() .UseSqlite(connectionString) .Options; } - public static DbContextOptions CreateDbOptions(SqliteConnection connection) + public DbContextOptions CreateDbOptions(SqliteConnection connection) { if (connection.State != System.Data.ConnectionState.Open) connection.Open(); @@ -80,7 +93,7 @@ public static DbContextOptions CreateDbOptions(SqliteConnection private const int CurrentSchemaVersion = 3; - public static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) + public void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) { _logger.Here().Information($"Saving cache for '{container.Name}' with {container.GetFileCount()} files"); @@ -170,7 +183,7 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db } } - private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) + private (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) { var relationalOptions = dbOptions.Extensions .OfType() @@ -186,7 +199,7 @@ private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSq return (new SqliteConnection(connectionString), true); } - public static CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) + public CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) { if (!File.Exists(dbFilePath)) { @@ -198,7 +211,7 @@ private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSq return LoadContainerFromCache(dbOptions, expectedFingerprint); } - public static CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint) + public CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint) { using var db = new CacheDbContext(dbOptions); @@ -244,7 +257,7 @@ private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSq return container; } - public static CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) + public CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) { if (!File.Exists(cacheFilePath)) { diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs deleted file mode 100644 index 1c94c7cb8..000000000 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ /dev/null @@ -1,262 +0,0 @@ -using System.Collections.Concurrent; -using System.Reflection; -using System.Text; -using Shared.Core.ErrorHandling; -using Shared.Core.PackFiles.Models; -using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Serialization; -using Shared.Core.PackFiles.Serialization.CacheDatabase; -using Shared.Core.Services; -using Shared.Core.Settings; - -namespace Shared.Core.PackFiles.Utility -{ - /* public interface IPackFileContainerLoader - { - IPackFileContainer? Load(string packFileSystemPath); - IPackFileContainer? LoadAllCaFiles(GameTypeEnum gameEnum); - IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath); - }*/ - - - - - - - -/* - - public class PackFileContainerLoader : IPackFileContainerLoader - { - static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); - private readonly ApplicationSettingsService _settingsService; - private readonly IStandardDialogs _standardDialogs; - private readonly LocalizationManager _localizationManager; - - public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager) - { - _settingsService = settingsService; - _standardDialogs = standardDialogs; - _localizationManager = localizationManager; - } - - public IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath) - { - if (Directory.Exists(packFileSystemPath) == false) - { - var location = Assembly.GetEntryAssembly()!.Location; - var loactionDir = Path.GetDirectoryName(location); - throw new Exception($"Unable to find folder {packFileSystemPath}. Curret systempath is {loactionDir}"); - } - - var containerName = Path.GetFileName(packFileSystemPath); - var container = new PackFileContainer(containerName) - { - SystemFilePath = packFileSystemPath, - }; - AddFolderContentToPackFile(container, packFileSystemPath, packFileSystemPath.ToLower() + "\\"); - return container; - } - - private static void AddFolderContentToPackFile(PackFileContainer container, string folderPath, string rootPath) - { - var files = Directory.GetFiles(folderPath); - foreach (var filePath in files) - { - var sanatizedFilePath = filePath.ToLower(); - var relativePath = sanatizedFilePath.Replace(rootPath, ""); - var fileName = Path.GetFileName(sanatizedFilePath); - - container.AddOrUpdateFile(relativePath, PackFile.CreateFromFileSystem(fileName, sanatizedFilePath)); - } - - var folders = Directory.GetDirectories(folderPath); - foreach (var folder in folders) - AddFolderContentToPackFile(container, folder, rootPath); - } - - public IPackFileContainer? Load(string packFileSystemPath) - { - try - { - if (!File.Exists(packFileSystemPath)) - { - _logger.Here().Error($"Trying to load file {packFileSystemPath}, which can not be located.", "Error"); - System.Windows.MessageBox.Show($"Unable to locate pack file \"{packFileSystemPath}\""); - return null; - } - - using var fileStream = File.OpenRead(packFileSystemPath); - using var reader = new BinaryReader(fileStream, Encoding.ASCII); - - var packFileSize = new FileInfo(packFileSystemPath).Length; - var pack = PackFileSerializerLoader.Load(packFileSystemPath, packFileSize, reader, new CustomPackDuplicateFileResolver()); - PackFileLog.LogPackCompression(pack); - - return pack; - } - catch (Exception e) - { - var errorMessage = $"Failed to load file {packFileSystemPath}. Error : {e.Message}"; - _logger.Here().Error(errorMessage); - throw new Exception(errorMessage, e); - } - } - - public IPackFileContainer? LoadAllCaFiles(GameTypeEnum gameEnum) - { - var game = GameInformationDatabase.GetGameById(gameEnum); - var gamePathInfo = _settingsService.CurrentSettings.GameDirectories.FirstOrDefault(x => x.Game == game.Type); - var gameName = game.DisplayName; - - if (gamePathInfo == null || string.IsNullOrWhiteSpace(gamePathInfo.Path)) - { - var errorMessage = $"Unable to load pack files for {gameName} because no game directory is configured."; - _logger.Here().Error(errorMessage); - return null; - } - - var gameDataFolder = gamePathInfo.Path; - - try - { - _logger.Here().Information($"Loading pack files for {gameName} located in {gameDataFolder}"); - var allCaPackFiles = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); - - var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(gameDataFolder, allCaPackFiles); - _logger.Here().Information($"Computed fingerprint for {gameName}: {fingerprint}"); - - var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName, fingerprint); - _logger.Here().Information($"Cache file path for {gameName}: {cacheFilePath}"); - - var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); - if (cached != null) - { - _logger.Here().Information($"Cache hit for {gameName} - loaded {cached.GetFileCount()} files from: {cacheFilePath}"); - return cached; - } - - _logger.Here().Information($"Cache miss for {gameName} at: {cacheFilePath}"); - - var cacheInvalidReason = GetCacheInvalidReason(cacheFilePath, fingerprint); - _logger.Here().Information($"Cache invalid reason for {gameName}: {cacheInvalidReason}"); - var reasonMessage = string.Format(_localizationManager.Get("PackFileCache.InvalidReason." + cacheInvalidReason), gameName); - var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); - var cacheDescription = _localizationManager.Get("PackFileCache.Description"); - _standardDialogs.ShowDialogBox( - reasonMessage + "\n\n" + buildingMessage + "\n\n" + cacheDescription, - _localizationManager.Get("PackFileCache.InvalidReason.Title")); - - PackFileContainer container; - using (_standardDialogs.ShowWaitCursor()) - { - container = LoadAllCaFilesFromDisk(gameDataFolder, gameName, allCaPackFiles, manifestFileFound); - - try - { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); - _logger.Here().Information($"Saved CA pack cache for {gameName} to {cacheFilePath}"); - } - catch (Exception cacheEx) - { - _logger.Here().Warning($"Failed to save CA pack cache for {gameName}: {cacheEx.Message}"); - } - } - - return container; - } - catch (Exception e) - { - _logger.Here().Error($"Trying to get all CA packs in {gameDataFolder}. Error : {e.ToString()}"); - return null; - } - } - - private static string GetCacheInvalidReason(string cacheFilePath, string fingerprint) - { - if (!File.Exists(cacheFilePath)) - return "NotFound"; - - try - { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, fingerprint); - if (result == null) - return "DataChanged"; - } - catch (Exception cacheReadException) - { - _logger.Here().Warning($"Failed to inspect CA pack cache '{cacheFilePath}': {cacheReadException.Message}"); - return "Corrupted"; - } - - return "DataChanged"; - } - - private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string gameName, List allCaPackFiles, bool manifestFileFound) - { - // When loading ca pack packs, we want to use the CA resolver as its faster. - // If there is no manifest file, we need to use the duplicate resolver as it loads all file in the folder. - // There might be custom mods in there that does not follow the rules! - IDuplicateFileResolver packfileResolver = new CaPackDuplicateFileResolver(); - if (manifestFileFound == false) - { - _logger.Here().Warning($"Loading pack files for {gameName}, which does not uses manifest.txt. If there are MODs in the game folder, this might cause issues!"); - packfileResolver = new CustomPackDuplicateFileResolver(); - } - - var packList = new ConcurrentBag(); - var packsCompressionStats = new ConcurrentDictionary(); - - Parallel.ForEach(allCaPackFiles, packFilePath => - { - var path = gameDataFolder + "\\" + packFilePath; - if (File.Exists(path)) - { - using var fileStream = File.OpenRead(path); - using var reader = new BinaryReader(fileStream, Encoding.ASCII); - - var packFileSize = new FileInfo(path).Length; - var pack = PackFileSerializerLoader.Load(path, packFileSize, reader, packfileResolver); - packList.Add(pack); - - PackFileLog.LogPackCompression(pack); - var packCompressionStats = PackFileLog.GetCompressionInformation(pack); - foreach (var kvp in packCompressionStats) - { - packsCompressionStats.AddOrUpdate( - kvp.Key, - _ => new CompressionInformation(kvp.Value.DiskSize, kvp.Value.UncompressedSize), - (_, existingStats) => new CompressionInformation( - existingStats.DiskSize + kvp.Value.DiskSize, - existingStats.UncompressedSize + kvp.Value.UncompressedSize)); - } - } - else - _logger.Here().Warning($"{gameName} pack file '{path}' not found, loading skipped"); - } - ); - - PackFileLog.LogPacksCompression(packsCompressionStats); - - var caPackFileContainer = new PackFileContainer($"All Game Packs - {gameName}"); - caPackFileContainer.IsCaPackFile = true; - caPackFileContainer.SystemFilePath = gameDataFolder; - var packFilesOrderedByGroup = packList.GroupBy(x => x.Header.LoadOrder).OrderBy(x => x.Key); - - foreach (var group in packFilesOrderedByGroup) - { - var packFilesOrderedByName = group.OrderBy(x => x.Name); - foreach (var packfile in packFilesOrderedByName) - { - if (string.IsNullOrWhiteSpace(packfile.SystemFilePath) == false) - caPackFileContainer.SourcePackFilePaths.Add(packfile.SystemFilePath); - caPackFileContainer.MergePackFileContainer(packfile); - } - } - - return caPackFileContainer; - } - }*/ -} diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs index 2c41833f3..d7a31ce2d 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Reflection; using System.Text; -using Microsoft.Xna.Framework; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; @@ -11,8 +10,7 @@ using Shared.Core.Settings; namespace Shared.Core.PackFiles.Utility -{ - +{ public enum PackFileContainerType { Cached, @@ -28,20 +26,22 @@ public interface IPackFileContainerLoader IPackFileContainer CreateFromSystemFolder(string folderPath); } - public class PackFileContainerLoader : IPackFileContainerLoader + class PackFileContainerLoader : IPackFileContainerLoader { static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); private readonly ApplicationSettingsService _settingsService; private readonly IStandardDialogs _standardDialogs; private readonly LocalizationManager _localizationManager; + private readonly IPackFileContainerCacheHelper _packFileContainerCacheHelper; - public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager) + public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager, IPackFileContainerCacheHelper packFileContainerCacheHelper) { // TODO : Handle context for logger _settingsService = settingsService; _standardDialogs = standardDialogs; _localizationManager = localizationManager; + _packFileContainerCacheHelper = packFileContainerCacheHelper; } public IPackFileContainer CreateFromSystemFolder(string packFileSystemPath) @@ -121,14 +121,13 @@ public IPackFileContainer CreateFromCollection(PackFileContainerType type, strin if(type == PackFileContainerType.Cached && loadAsReadOnly == false) throw new InvalidOperationException($"Cannot load as writable if loading from cache. Caching is only supported for read-only containers. PackFile {createdPackFileName}"); - - var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(fullPackFilePaths); + var fingerprint = _packFileContainerCacheHelper.ComputeFingerprint(fullPackFilePaths); var cachePrefix = createdPackFileName; - var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(cachePrefix, fingerprint); + var cacheFilePath = _packFileContainerCacheHelper.GetCacheFilePath(cachePrefix, fingerprint); if (type == PackFileContainerType.Cached) { - var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); + var cached = _packFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); if (cached != null) return cached; @@ -138,12 +137,10 @@ public IPackFileContainer CreateFromCollection(PackFileContainerType type, strin //var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); //var cacheDescription = _localizationManager.Get("PackFileCache.Description"); _standardDialogs.ShowDialogBox("Failed to load from cache - make better error later"); - } using (_standardDialogs.ShowWaitCursor()) { - var container = LoadPackFilesFromDisk(createdPackFileName, fullPackFilePaths, duplicateFileResolver); container.Name = createdPackFileName; container.IsCaPackFile = loadAsReadOnly; @@ -151,9 +148,13 @@ public IPackFileContainer CreateFromCollection(PackFileContainerType type, strin if (type == PackFileContainerType.Cached) { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); - // Do we want to set it to the cached version? It should be the same data, just in a different format. + var dbOptions = _packFileContainerCacheHelper.CreateDbOptions(cacheFilePath); + _packFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); + var cached = _packFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); + if(cached == null) + throw new Exception($"Failed to load from cache after saving cache. PackFile {createdPackFileName}"); + + return cached; } return container; diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs index b8278c865..a9fded383 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs @@ -78,9 +78,10 @@ public void SearchFiles_ExtensionFilter_MatchesFilenameSubstrings_ForLegacyWemVa keepAliveConnection = new SqliteConnection(connectionString); keepAliveConnection.Open(); - var dbOptions = PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); - PackFileContainerCacheHelper.SaveCache("variants_fp", sourceContainer, dbOptions); - container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "variants_fp")!; + var cacheHelper = new PackFileContainerCacheHelper(); + var dbOptions = cacheHelper.CreateDbOptionsFromConnectionString(connectionString); + cacheHelper.SaveCache("variants_fp", sourceContainer, dbOptions); + container = cacheHelper.LoadContainerFromCache(dbOptions, "variants_fp")!; } else { diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index 688746468..9fcdd1de8 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -60,9 +60,10 @@ public void Setup() _cacheKeepAliveConnection = new SqliteConnection(connectionString); _cacheKeepAliveConnection.Open(); - var dbOptions = PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); - PackFileContainerCacheHelper.SaveCache("test_fp", sourceContainer, dbOptions); - _container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp")!; + var cacheHelper = new PackFileContainerCacheHelper(); + var dbOptions = cacheHelper.CreateDbOptionsFromConnectionString(connectionString); + cacheHelper.SaveCache("test_fp", sourceContainer, dbOptions); + _container = cacheHelper.LoadContainerFromCache(dbOptions, "test_fp")!; } else { diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index 225eb4ece..6534fbb93 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -47,12 +47,14 @@ private DbContextOptions CreateTestDbOptions() keepAliveConnection.Open(); _inMemoryKeepAliveConnections.Add(keepAliveConnection); - return PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); + var cacheHelper = new PackFileContainerCacheHelper(); + return cacheHelper.CreateDbOptionsFromConnectionString(connectionString); } private DbContextOptions CreateFileDbOptions() { - return PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + var cacheHelper = new PackFileContainerCacheHelper(); + return cacheHelper.CreateDbOptions(_dbFilePath); } [Test] @@ -78,9 +80,10 @@ public void RoundTrip_PreservesMetadata() container.AddOrUpdateFile("folder\\file2.bin", new PackFile("file2.bin", source2)); // Act + var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fingerprint123", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fingerprint123"); + cacheHelper.SaveCache("fingerprint123", container, dbOptions); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fingerprint123"); // Assert Assert.That(loaded, Is.Not.Null); @@ -109,10 +112,11 @@ public void LoadCache_ReturnsCorrectFileData() new PackedFileSource(parent, 2048, 4096, false, true, CompressionFormat.Lz4, 8192))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + var cacheHelper = new PackFileContainerCacheHelper(); + cacheHelper.SaveCache("fp", container, dbOptions); // Act - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); // Assert Assert.That(loaded, Is.Not.Null); @@ -153,8 +157,9 @@ public void LoadCache_PreservesSourcePackFilePath() new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var cacheHelper = new PackFileContainerCacheHelper(); + cacheHelper.SaveCache("fp", container, dbOptions); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); var sourceA = (PackedFileSource)loaded!.FindFile("a.txt")!.DataSource; var sourceB = (PackedFileSource)loaded!.FindFile("b.txt")!.DataSource; @@ -165,7 +170,8 @@ public void LoadCache_PreservesSourcePackFilePath() [Test] public void LoadCache_ReturnsNullForMissingFile() { - var result = PackFileContainerCacheHelper.LoadContainerFromCache( + var cacheHelper = new PackFileContainerCacheHelper(); + var result = cacheHelper.LoadContainerFromCache( Path.Combine(_tempDir, "nonexistent.db"), "fp"); Assert.That(result, Is.Null); } @@ -178,10 +184,11 @@ public void LoadCache_ReturnsNullForWrongFingerprint() SystemFilePath = @"c:\game" }; + var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("correct_fp", container, dbOptions); + cacheHelper.SaveCache("correct_fp", container, dbOptions); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "wrong_fp"); + var result = cacheHelper.LoadContainerFromCache(dbOptions, "wrong_fp"); Assert.That(result, Is.Null); } @@ -194,8 +201,9 @@ public void ComputeFingerprint_DeterministicForSameInputs() File.WriteAllText(Path.Combine(packDir, "b.pack"), "data_b"); var packFiles = new List { Path.Combine(packDir, "a.pack"), Path.Combine(packDir, "b.pack") }; - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); + var cacheHelper = new PackFileContainerCacheHelper(); + var fp1 = cacheHelper.ComputeFingerprint(packFiles); + var fp2 = cacheHelper.ComputeFingerprint(packFiles); Assert.That(fp1, Is.EqualTo(fp2)); } @@ -208,12 +216,12 @@ public void ComputeFingerprint_ChangesWhenFileChanges() File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); var packFiles = new List { Path.Combine(packDir,"a.pack") }; - - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); + var cacheHelper = new PackFileContainerCacheHelper(); + var fp1 = cacheHelper.ComputeFingerprint(packFiles); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a_modified_longer"); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); + var fp2 = cacheHelper.ComputeFingerprint(packFiles); Assert.That(fp1, Is.Not.EqualTo(fp2)); } @@ -238,8 +246,9 @@ public void RoundTrip_FullCycle() // Act: save ? load var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("test_fp", container, dbOptions); - var restored = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp"); + var cacheHelper = new PackFileContainerCacheHelper(); + cacheHelper.SaveCache("test_fp", container, dbOptions); + var restored = cacheHelper.LoadContainerFromCache(dbOptions, "test_fp"); // Assert Assert.That(restored, Is.Not.Null); @@ -272,9 +281,10 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() new PackedFileSource(parent, 0, 100, false, false, CompressionFormat.None, 0))); var dbOptions = CreateFileDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp_try", container, dbOptions); + var cacheHelper = new PackFileContainerCacheHelper(); + cacheHelper.SaveCache("fp_try", container, dbOptions); - var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp_try"); + var result = cacheHelper.TryLoadFromCache(_dbFilePath, "fp_try"); Assert.That(result, Is.Not.Null); Assert.That(result.Name, Is.EqualTo("TryLoad Test")); } @@ -282,7 +292,8 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() [Test] public void TryLoadFromCache_ReturnsNullForMissingFile() { - var result = PackFileContainerCacheHelper.TryLoadFromCache( + var cacheHelper = new PackFileContainerCacheHelper(); + var result = cacheHelper.TryLoadFromCache( Path.Combine(_tempDir, "does_not_exist.db"), "fp"); Assert.That(result, Is.Null); } @@ -290,8 +301,9 @@ public void TryLoadFromCache_ReturnsNullForMissingFile() [Test] public void TryLoadFromCache_ReturnsNullForCorruptFile() { + var cacheHelper = new PackFileContainerCacheHelper(); File.WriteAllBytes(_dbFilePath, [0xFF, 0xFE, 0x00, 0x01]); - var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp"); + var result = cacheHelper.TryLoadFromCache(_dbFilePath, "fp"); Assert.That(result, Is.Null); } @@ -307,9 +319,10 @@ public void SaveCache_PreservesEncryptedFlag() container.AddOrUpdateFile("secret\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 0, 500, isEncrypted: true, isCompressed: false, CompressionFormat.None, 0))); + var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + cacheHelper.SaveCache("fp", container, dbOptions); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); var source = (PackedFileSource)loaded!.FindFile("secret\\data.bin")!.DataSource; Assert.That(source.IsEncrypted, Is.True); @@ -323,9 +336,10 @@ public void SaveCache_EmptyContainer_RoundTrips() SystemFilePath = @"c:\game\empty" }; + var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp_empty", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp_empty"); + cacheHelper.SaveCache("fp_empty", container, dbOptions); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp_empty"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.Name, Is.EqualTo("Empty Pack")); @@ -337,25 +351,26 @@ public void SaveCache_OverwritesExistingCache() { var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; var dbOptions = CreateFileDbOptions(); + var cacheHelper = new PackFileContainerCacheHelper(); // Save first version var container1 = new PackFileContainer("Version1") { SystemFilePath = @"c:\game" }; container1.AddOrUpdateFile("old.txt", new PackFile("old.txt", new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - PackFileContainerCacheHelper.SaveCache("fp1", container1, dbOptions); + cacheHelper.SaveCache("fp1", container1, dbOptions); // Save second version (same db path) var container2 = new PackFileContainer("Version2") { SystemFilePath = @"c:\game" }; container2.AddOrUpdateFile("new.txt", new PackFile("new.txt", new PackedFileSource(parent, 0, 20, false, false, CompressionFormat.None, 0))); - PackFileContainerCacheHelper.SaveCache("fp2", container2, dbOptions); + cacheHelper.SaveCache("fp2", container2, dbOptions); // Old fingerprint should fail - var oldResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp1"); + var oldResult = cacheHelper.LoadContainerFromCache(dbOptions, "fp1"); Assert.That(oldResult, Is.Null); // New fingerprint should work - var newResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp2"); + var newResult = cacheHelper.LoadContainerFromCache(dbOptions, "fp2"); Assert.That(newResult, Is.Not.Null); Assert.That(newResult.Name, Is.EqualTo("Version2")); Assert.That(newResult.GetFileCount(), Is.EqualTo(1)); @@ -379,10 +394,11 @@ public void SaveCache_StoresFolderPathCorrectly() new PackedFileSource(parent, 20, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + var cacheHelper = new PackFileContainerCacheHelper(); + cacheHelper.SaveCache("fp", container, dbOptions); // Verify via the CachedPackFileContainer's GetDirectoryContent - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); var rootFiles = loaded.GetDirectoryContent(""); @@ -406,9 +422,10 @@ public void SaveCache_SkipsNonPackedFileSources() container.AddOrUpdateFile("memory.txt", new PackFile("memory.txt", new MemorySource([1, 2, 3]))); + var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + cacheHelper.SaveCache("fp", container, dbOptions); + var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.GetFileCount(), Is.EqualTo(1)); @@ -424,13 +441,14 @@ public void ComputeFingerprint_IgnoresMissingPackFiles() Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "exists.pack"), "data"); + var cacheHelper = new PackFileContainerCacheHelper(); var packFiles = new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }; - var fp = PackFileContainerCacheHelper.ComputeFingerprint(packFiles); + var fp = cacheHelper.ComputeFingerprint(packFiles); Assert.That(fp, Is.Not.Null.And.Not.Empty); // Same result regardless of missing file in list - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); + var fp2 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); Assert.That(fp, Is.EqualTo(fp2)); } @@ -443,9 +461,10 @@ public void ComputeFingerprint_OrderIndependent() File.WriteAllText(Path.Combine(packDir, "beta.pack"), "bbb"); File.WriteAllText(Path.Combine(packDir, "gamma.pack"), "ccc"); - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); - var fp3 = PackFileContainerCacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); + var cacheHelper = new PackFileContainerCacheHelper(); + var fp1 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); + var fp2 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); + var fp3 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); Assert.That(fp1, Is.EqualTo(fp2)); Assert.That(fp2, Is.EqualTo(fp3)); @@ -454,7 +473,8 @@ public void ComputeFingerprint_OrderIndependent() [Test] public void GetCacheFilePath_SanitizesInvalidChars() { - var path = PackFileContainerCacheHelper.GetCacheFilePath("Game:Name/WithChars", "abc123"); + var cacheHelper = new PackFileContainerCacheHelper(); + var path = cacheHelper.GetCacheFilePath("Game:Name/WithChars", "abc123"); var fileName = Path.GetFileName(path); Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs index e08b85a74..f1404cdb3 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -1,5 +1,6 @@ using Moq; using Shared.Core.Misc; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -46,7 +47,7 @@ public void TearDown() Directory.Delete(_tempGameDir, true); } - private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager); + private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager, new PackFileContainerCacheHelper()); [Test] public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() diff --git a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs index 0f73a8f9f..bca42ecdb 100644 --- a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs +++ b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs @@ -1,5 +1,6 @@ using Moq; using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -12,7 +13,7 @@ public static class PackFileSerivceTestHelper public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = GameTypeEnum.Warhammer3) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager()); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); var container = loader.CreateFromSystemFolder(path); container.IsCaPackFile = true; pfs.AddContainer(container); @@ -23,7 +24,7 @@ public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = G public static IPackFileService CreateFromFolder(GameTypeEnum selectedGame, string path ) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager()); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); var container = loader.CreateFromSystemFolder(PathHelper.GetDataFolder(path)); container.IsCaPackFile = true; From 858905d1711373292da3ba5fa46cd994ae04310a Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sat, 6 Jun 2026 20:16:30 +0200 Subject: [PATCH 7/9] Code --- .../Containers/CachedPackFileContainer.cs | 230 +++++++++++++++++- .../PackFileContainerCacheHelper.cs | 204 +--------------- .../Utility/PackFileContainerLoader2.cs | 8 +- .../PackFileContainerTests_SearchFiles.cs | 15 +- .../PackFileContainerTests_TestBase.cs | 11 +- .../PackFileContainerCacheHelperTests.cs | 119 ++++----- .../InMemoryPackFileContainerCacheHelper.cs | 111 +++++++++ .../Utility/PackFileContainerLoaderTests.cs | 26 +- 8 files changed, 425 insertions(+), 299 deletions(-) create mode 100644 Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 924fedd5c..9230c3de6 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,6 +1,7 @@ -using System.Collections.Immutable; using System.Diagnostics; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization.CacheDatabase; @@ -9,26 +10,247 @@ namespace Shared.Core.PackFiles.Models.Containers { - internal class CachedPackFileContainer : IPackFileContainerInternal + internal enum CacheStorageMode { - private readonly ILogger _logger = Logging.Create(); + File, + InMemory + } + + internal class CachedPackFileContainer : IPackFileContainerInternal, IDisposable + { + private static readonly ILogger _logger = Logging.CreateStatic(typeof(CachedPackFileContainer)); + private const int CurrentSchemaVersion = 3; - private readonly CacheDbContext _db; + private CacheDbContext _db; + private readonly DbContextOptions _dbOptions; private readonly object _dbLock = new(); + public CacheStorageMode StorageMode { get; } + public string? DbFilePath { get; } public string Name { get; set; } public bool IsCaPackFile { get => true; set { } } public string SystemFilePath { get; set; } public HashSet SourcePackFilePaths { get; set; } = []; + /// + /// Creates a file-backed CachedPackFileContainer. + /// + public CachedPackFileContainer(string name, string dbFilePath) + { + Name = name; + SystemFilePath = string.Empty; + DbFilePath = dbFilePath; + StorageMode = CacheStorageMode.File; + _dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + _db = new CacheDbContext(_dbOptions); + _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + } + + /// + /// Creates an in-memory CachedPackFileContainer. + /// public CachedPackFileContainer(string name, DbContextOptions dbOptions) { Name = name; SystemFilePath = string.Empty; + DbFilePath = null; + StorageMode = CacheStorageMode.InMemory; + _dbOptions = dbOptions; _db = new CacheDbContext(dbOptions); _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; } + /// + /// Saves the contents of a PackFileContainer into this cached container's database. + /// + public void Save(string fingerprint, PackFileContainer source) + { + _logger.Here().Information($"Saving cache for '{source.Name}' with {source.GetFileCount()} files"); + + // Use EF to create the schema + using (var db = new CacheDbContext(_dbOptions)) + { + db.Database.EnsureDeleted(); + db.Database.EnsureCreated(); + } + + var (connection, shouldDisposeConnection) = GetSqliteConnection(_dbOptions); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + + try + { + using var transaction = connection.BeginTransaction(); + + using (var cmd = connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO CacheInfo (SchemaVersion, Fingerprint, ContainerName, SystemFilePath, SourcePackFilePaths) + VALUES ($schemaVersion, $fingerprint, $containerName, $systemFilePath, $sourcePackFilePaths)"; + cmd.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); + cmd.Parameters.AddWithValue("$fingerprint", fingerprint); + cmd.Parameters.AddWithValue("$containerName", source.Name); + cmd.Parameters.AddWithValue("$systemFilePath", source.SystemFilePath); + cmd.Parameters.AddWithValue("$sourcePackFilePaths", string.Join("|", source.SourcePackFilePaths)); + cmd.ExecuteNonQuery(); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO FileList (RelativePath, FileName, Extension, FolderPath, SourcePackFilePath, Offset, Size, IsEncrypted, IsCompressed, CompressionFormat, UncompressedSize) + VALUES ($relativePath, $fileName, $extension, $folderPath, $sourcePackFilePath, $offset, $size, $isEncrypted, $isCompressed, $compressionFormat, $uncompressedSize)"; + + var pRelativePath = cmd.Parameters.Add("$relativePath", SqliteType.Text); + var pFileName = cmd.Parameters.Add("$fileName", SqliteType.Text); + var pExtension = cmd.Parameters.Add("$extension", SqliteType.Text); + var pFolderPath = cmd.Parameters.Add("$folderPath", SqliteType.Text); + var pSourcePackFilePath = cmd.Parameters.Add("$sourcePackFilePath", SqliteType.Text); + var pOffset = cmd.Parameters.Add("$offset", SqliteType.Integer); + var pSize = cmd.Parameters.Add("$size", SqliteType.Integer); + var pIsEncrypted = cmd.Parameters.Add("$isEncrypted", SqliteType.Integer); + var pIsCompressed = cmd.Parameters.Add("$isCompressed", SqliteType.Integer); + var pCompressionFormat = cmd.Parameters.Add("$compressionFormat", SqliteType.Integer); + var pUncompressedSize = cmd.Parameters.Add("$uncompressedSize", SqliteType.Integer); + + cmd.Prepare(); + + foreach (var (relativePath, packFile) in source.GetAllFiles()) + { + if (packFile.DataSource is not PackedFileSource fileSource) + continue; + + var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); + var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); + + pRelativePath.Value = relativePath; + pFileName.Value = packFile.Name; + pExtension.Value = Path.GetExtension(relativePath).ToLower(); + pFolderPath.Value = folderPath; + pSourcePackFilePath.Value = fileSource.Parent.FilePath; + pOffset.Value = fileSource.Offset; + pSize.Value = fileSource.Size; + pIsEncrypted.Value = fileSource.IsEncrypted ? 1 : 0; + pIsCompressed.Value = fileSource.IsCompressed ? 1 : 0; + pCompressionFormat.Value = (int)fileSource.CompressionFormat; + pUncompressedSize.Value = (long)fileSource.UncompressedSize; + + cmd.ExecuteNonQuery(); + } + } + + transaction.Commit(); + _logger.Here().Information($"Cache saved successfully for '{source.Name}'"); + } + finally + { + if (shouldDisposeConnection) + connection.Dispose(); + } + + // Update this container's metadata from what was saved + Name = source.Name; + SystemFilePath = source.SystemFilePath ?? string.Empty; + SourcePackFilePaths = new HashSet(source.SourcePackFilePaths); + + // Recreate _db so this instance can be queried after Save() + _db.Dispose(); + _db = new CacheDbContext(_dbOptions); + _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + } + + /// + /// Loads a CachedPackFileContainer from a file-backed database. + /// Returns null if the file doesn't exist, the DB is corrupt, or the fingerprint doesn't match. + /// + public static CachedPackFileContainer? CreateFromFingerPrint(string dbFilePath, string expectedFingerprint) + { + if (!File.Exists(dbFilePath)) + { + _logger.Here().Information($"No cache file found at '{dbFilePath}'"); + return null; + } + + var dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + + return CreateFromFingerPrint(dbOptions, expectedFingerprint); + } + + /// + /// Loads a CachedPackFileContainer from the given DbContextOptions (file or in-memory). + /// Returns null if the DB is corrupt or the fingerprint doesn't match. + /// + public static CachedPackFileContainer? CreateFromFingerPrint(DbContextOptions dbOptions, string expectedFingerprint) + { + using var db = new CacheDbContext(dbOptions); + + try + { + db.Database.EnsureCreated(); + } + catch (Exception ex) + { + _logger.Here().Warning($"Failed to open cache database: {ex.Message}"); + return null; + } + + CacheInfoEntity? cacheInfo; + try + { + cacheInfo = db.CacheInfo.FirstOrDefault(); + } + catch (Exception ex) + { + _logger.Here().Warning($"Failed to read cache info: {ex.Message}"); + return null; + } + + if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) + { + _logger.Here().Information($"Cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); + return null; + } + + var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) + { + SystemFilePath = cacheInfo.SystemFilePath, + }; + + if (!string.IsNullOrEmpty(cacheInfo.SourcePackFilePaths)) + { + foreach (var path in cacheInfo.SourcePackFilePaths.Split('|')) + container.SourcePackFilePaths.Add(path); + } + + _logger.Here().Information($"Loaded container '{container.Name}' from cache"); + return container; + } + + private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) + { + var relationalOptions = dbOptions.Extensions + .OfType() + .FirstOrDefault(); + + if (relationalOptions?.Connection is SqliteConnection sqliteConnection) + return (sqliteConnection, false); + + var connectionString = relationalOptions?.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + throw new InvalidOperationException("Unable to resolve SQLite connection from DbContextOptions."); + + return (new SqliteConnection(connectionString), true); + } + + public void Dispose() + { + _db.Dispose(); + } + public int GetFileCount() { lock (_dbLock) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 1f971fbd7..9b20bd1a9 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -1,25 +1,17 @@ -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; using Shared.Core.ErrorHandling; using Shared.Core.Misc; using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Models.FileSources; namespace Shared.Core.PackFiles.Serialization.CacheDatabase { interface IPackFileContainerCacheHelper { string ComputeFingerprint(List packFileNames); - DbContextOptions CreateDbOptions(SqliteConnection connection); - DbContextOptions CreateDbOptions(string dbFilePath); - DbContextOptions CreateDbOptionsFromConnectionString(string connectionString); string GetCacheFilePath(string gameName, string cacheId); - CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint); - CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint); - void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions); CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint); + CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath); } class PackFileContainerCacheHelper : IPackFileContainerCacheHelper @@ -67,194 +59,18 @@ public string ComputeFingerprint(List packFileNames) return Convert.ToHexString(hash); } - public DbContextOptions CreateDbOptions(string dbFilePath) + public CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath) { - return new DbContextOptionsBuilder() - .UseSqlite($"Data Source={dbFilePath};Pooling=False") - .Options; - } - - public DbContextOptions CreateDbOptionsFromConnectionString(string connectionString) - { - return new DbContextOptionsBuilder() - .UseSqlite(connectionString) - .Options; - } - - public DbContextOptions CreateDbOptions(SqliteConnection connection) - { - if (connection.State != System.Data.ConnectionState.Open) - connection.Open(); - - return new DbContextOptionsBuilder() - .UseSqlite(connection) - .Options; - } - - private const int CurrentSchemaVersion = 3; - - public void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) - { - _logger.Here().Information($"Saving cache for '{container.Name}' with {container.GetFileCount()} files"); - - // Use EF only to create the schema, then dispose to release memory - using (var db = new CacheDbContext(dbOptions)) - { - db.Database.EnsureDeleted(); - db.Database.EnsureCreated(); - } - - // Use raw SQLite for bulk insert - avoids EF change tracker memory and perf overhead. - // If dbOptions carries an explicit SqliteConnection (for tests/in-memory), reuse it. - var (connection, shouldDisposeConnection) = GetSqliteConnection(dbOptions); - if (connection.State != System.Data.ConnectionState.Open) - connection.Open(); - - try - { - using var transaction = connection.BeginTransaction(); - - // Insert CacheInfo - using (var cmd = connection.CreateCommand()) - { - cmd.Transaction = transaction; - cmd.CommandText = @"INSERT INTO CacheInfo (SchemaVersion, Fingerprint, ContainerName, SystemFilePath, SourcePackFilePaths) - VALUES ($schemaVersion, $fingerprint, $containerName, $systemFilePath, $sourcePackFilePaths)"; - cmd.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); - cmd.Parameters.AddWithValue("$fingerprint", fingerprint); - cmd.Parameters.AddWithValue("$containerName", container.Name); - cmd.Parameters.AddWithValue("$systemFilePath", container.SystemFilePath); - cmd.Parameters.AddWithValue("$sourcePackFilePaths", string.Join("|", container.SourcePackFilePaths)); - cmd.ExecuteNonQuery(); - } - - // Bulk insert files using a prepared statement - using (var cmd = connection.CreateCommand()) - { - cmd.Transaction = transaction; - cmd.CommandText = @"INSERT INTO FileList (RelativePath, FileName, Extension, FolderPath, SourcePackFilePath, Offset, Size, IsEncrypted, IsCompressed, CompressionFormat, UncompressedSize) - VALUES ($relativePath, $fileName, $extension, $folderPath, $sourcePackFilePath, $offset, $size, $isEncrypted, $isCompressed, $compressionFormat, $uncompressedSize)"; - - var pRelativePath = cmd.Parameters.Add("$relativePath", SqliteType.Text); - var pFileName = cmd.Parameters.Add("$fileName", SqliteType.Text); - var pExtension = cmd.Parameters.Add("$extension", SqliteType.Text); - var pFolderPath = cmd.Parameters.Add("$folderPath", SqliteType.Text); - var pSourcePackFilePath = cmd.Parameters.Add("$sourcePackFilePath", SqliteType.Text); - var pOffset = cmd.Parameters.Add("$offset", SqliteType.Integer); - var pSize = cmd.Parameters.Add("$size", SqliteType.Integer); - var pIsEncrypted = cmd.Parameters.Add("$isEncrypted", SqliteType.Integer); - var pIsCompressed = cmd.Parameters.Add("$isCompressed", SqliteType.Integer); - var pCompressionFormat = cmd.Parameters.Add("$compressionFormat", SqliteType.Integer); - var pUncompressedSize = cmd.Parameters.Add("$uncompressedSize", SqliteType.Integer); - - cmd.Prepare(); - - foreach (var (relativePath, packFile) in container.GetAllFiles()) - { - if (packFile.DataSource is not PackedFileSource source) - continue; - - var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); - var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); - - pRelativePath.Value = relativePath; - pFileName.Value = packFile.Name; - pExtension.Value = Path.GetExtension(relativePath).ToLower(); - pFolderPath.Value = folderPath; - pSourcePackFilePath.Value = source.Parent.FilePath; - pOffset.Value = source.Offset; - pSize.Value = source.Size; - pIsEncrypted.Value = source.IsEncrypted ? 1 : 0; - pIsCompressed.Value = source.IsCompressed ? 1 : 0; - pCompressionFormat.Value = (int)source.CompressionFormat; - pUncompressedSize.Value = (long)source.UncompressedSize; - - cmd.ExecuteNonQuery(); - } - } - - transaction.Commit(); - _logger.Here().Information($"Cache saved successfully for '{container.Name}'"); - } - finally - { - if (shouldDisposeConnection) - connection.Dispose(); - } - } - - private (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) - { - var relationalOptions = dbOptions.Extensions - .OfType() - .FirstOrDefault(); - - if (relationalOptions?.Connection is SqliteConnection sqliteConnection) - return (sqliteConnection, false); - - var connectionString = relationalOptions?.ConnectionString; - if (string.IsNullOrWhiteSpace(connectionString)) - throw new InvalidOperationException("Unable to resolve SQLite connection from DbContextOptions."); - - return (new SqliteConnection(connectionString), true); - } - - public CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) - { - if (!File.Exists(dbFilePath)) - { - _logger.Here().Information($"No cache file found at '{dbFilePath}'"); - return null; - } - - var dbOptions = CreateDbOptions(dbFilePath); - return LoadContainerFromCache(dbOptions, expectedFingerprint); - } - - public CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint) - { - using var db = new CacheDbContext(dbOptions); - - try - { - db.Database.EnsureCreated(); - } - catch (Exception ex) - { - _logger.Here().Warning($"Failed to open cache database: {ex.Message}"); - return null; - } - - CacheInfoEntity? cacheInfo; - try - { - cacheInfo = db.CacheInfo.FirstOrDefault(); - } - catch (Exception ex) + using (var temp = new CachedPackFileContainer(container.Name, cacheFilePath)) { - _logger.Here().Warning($"Failed to read cache info: {ex.Message}"); - return null; - } - - if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) - { - _logger.Here().Information($"Cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); - return null; + temp.Save(fingerprint, container); } - var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) - { - SystemFilePath = cacheInfo.SystemFilePath, - }; - - if (!string.IsNullOrEmpty(cacheInfo.SourcePackFilePaths)) - { - foreach (var path in cacheInfo.SourcePackFilePaths.Split('|')) - container.SourcePackFilePaths.Add(path); - } + var loaded = CachedPackFileContainer.CreateFromFingerPrint(cacheFilePath, fingerprint); + if (loaded == null) + throw new Exception($"Failed to load from cache after saving. CacheFile: {cacheFilePath}"); - _logger.Here().Information($"Loaded container '{container.Name}' from cache"); - return container; + return loaded; } public CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) @@ -268,7 +84,7 @@ public void SaveCache(string fingerprint, PackFileContainer container, DbContext try { _logger.Here().Information($"Attempting to load cache from: {cacheFilePath} with fingerprint: {fingerprint}"); - var result = LoadContainerFromCache(cacheFilePath, fingerprint); + var result = CachedPackFileContainer.CreateFromFingerPrint(cacheFilePath, fingerprint); if (result == null) _logger.Here().Information($"Cache load returned null (fingerprint/schema mismatch) for: {cacheFilePath}"); return result; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs index d7a31ce2d..b0daa31e9 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs @@ -148,13 +148,7 @@ public IPackFileContainer CreateFromCollection(PackFileContainerType type, strin if (type == PackFileContainerType.Cached) { - var dbOptions = _packFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - _packFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); - var cached = _packFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); - if(cached == null) - throw new Exception($"Failed to load from cache after saving cache. PackFile {createdPackFileName}"); - - return cached; + return _packFileContainerCacheHelper.SaveAndLoadCache(fingerprint, container, cacheFilePath); } return container; diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs index a9fded383..86cadcf4b 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs @@ -1,9 +1,10 @@ -using Shared.Core.PackFiles.Models; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; -using Microsoft.Data.Sqlite; namespace Shared.CoreTest.PackFiles.Models.Containers { @@ -78,10 +79,12 @@ public void SearchFiles_ExtensionFilter_MatchesFilenameSubstrings_ForLegacyWemVa keepAliveConnection = new SqliteConnection(connectionString); keepAliveConnection.Open(); - var cacheHelper = new PackFileContainerCacheHelper(); - var dbOptions = cacheHelper.CreateDbOptionsFromConnectionString(connectionString); - cacheHelper.SaveCache("variants_fp", sourceContainer, dbOptions); - container = cacheHelper.LoadContainerFromCache(dbOptions, "variants_fp")!; + var dbOptions = new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; + var cached = new CachedPackFileContainer("variants_cache", dbOptions); + cached.Save("variants_fp", sourceContainer); + container = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "variants_fp")!; } else { diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index 9fcdd1de8..41ab62e29 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -1,4 +1,5 @@ using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; @@ -60,10 +61,12 @@ public void Setup() _cacheKeepAliveConnection = new SqliteConnection(connectionString); _cacheKeepAliveConnection.Open(); - var cacheHelper = new PackFileContainerCacheHelper(); - var dbOptions = cacheHelper.CreateDbOptionsFromConnectionString(connectionString); - cacheHelper.SaveCache("test_fp", sourceContainer, dbOptions); - _container = cacheHelper.LoadContainerFromCache(dbOptions, "test_fp")!; + var dbOptions = new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; + var cached = new CachedPackFileContainer("test_cache", dbOptions); + cached.Save("test_fp", sourceContainer); + _container = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "test_fp")!; } else { diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index 6534fbb93..9947d4eb4 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -1,4 +1,4 @@ -using Microsoft.Data.Sqlite; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; @@ -47,14 +47,22 @@ private DbContextOptions CreateTestDbOptions() keepAliveConnection.Open(); _inMemoryKeepAliveConnections.Add(keepAliveConnection); - var cacheHelper = new PackFileContainerCacheHelper(); - return cacheHelper.CreateDbOptionsFromConnectionString(connectionString); + return new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; } private DbContextOptions CreateFileDbOptions() { - var cacheHelper = new PackFileContainerCacheHelper(); - return cacheHelper.CreateDbOptions(_dbFilePath); + return new DbContextOptionsBuilder() + .UseSqlite($"Data Source={_dbFilePath};Pooling=False") + .Options; + } + + private static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) + { + using var cached = new CachedPackFileContainer(container.Name, dbOptions); + cached.Save(fingerprint, container); } [Test] @@ -80,10 +88,9 @@ public void RoundTrip_PreservesMetadata() container.AddOrUpdateFile("folder\\file2.bin", new PackFile("file2.bin", source2)); // Act - var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - cacheHelper.SaveCache("fingerprint123", container, dbOptions); - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fingerprint123"); + SaveCache("fingerprint123", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fingerprint123"); // Assert Assert.That(loaded, Is.Not.Null); @@ -112,11 +119,10 @@ public void LoadCache_ReturnsCorrectFileData() new PackedFileSource(parent, 2048, 4096, false, true, CompressionFormat.Lz4, 8192))); var dbOptions = CreateTestDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - cacheHelper.SaveCache("fp", container, dbOptions); + SaveCache("fp", container, dbOptions); // Act - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); // Assert Assert.That(loaded, Is.Not.Null); @@ -157,9 +163,8 @@ public void LoadCache_PreservesSourcePackFilePath() new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - cacheHelper.SaveCache("fp", container, dbOptions); - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); var sourceA = (PackedFileSource)loaded!.FindFile("a.txt")!.DataSource; var sourceB = (PackedFileSource)loaded!.FindFile("b.txt")!.DataSource; @@ -170,8 +175,7 @@ public void LoadCache_PreservesSourcePackFilePath() [Test] public void LoadCache_ReturnsNullForMissingFile() { - var cacheHelper = new PackFileContainerCacheHelper(); - var result = cacheHelper.LoadContainerFromCache( + var result = CachedPackFileContainer.CreateFromFingerPrint( Path.Combine(_tempDir, "nonexistent.db"), "fp"); Assert.That(result, Is.Null); } @@ -183,12 +187,10 @@ public void LoadCache_ReturnsNullForWrongFingerprint() { SystemFilePath = @"c:\game" }; - - var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - cacheHelper.SaveCache("correct_fp", container, dbOptions); + SaveCache("correct_fp", container, dbOptions); - var result = cacheHelper.LoadContainerFromCache(dbOptions, "wrong_fp"); + var result = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "wrong_fp"); Assert.That(result, Is.Null); } @@ -200,10 +202,8 @@ public void ComputeFingerprint_DeterministicForSameInputs() File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); File.WriteAllText(Path.Combine(packDir, "b.pack"), "data_b"); var packFiles = new List { Path.Combine(packDir, "a.pack"), Path.Combine(packDir, "b.pack") }; - - var cacheHelper = new PackFileContainerCacheHelper(); - var fp1 = cacheHelper.ComputeFingerprint(packFiles); - var fp2 = cacheHelper.ComputeFingerprint(packFiles); + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp1, Is.EqualTo(fp2)); } @@ -216,12 +216,11 @@ public void ComputeFingerprint_ChangesWhenFileChanges() File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); var packFiles = new List { Path.Combine(packDir,"a.pack") }; - var cacheHelper = new PackFileContainerCacheHelper(); - var fp1 = cacheHelper.ComputeFingerprint(packFiles); + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a_modified_longer"); - var fp2 = cacheHelper.ComputeFingerprint(packFiles); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp1, Is.Not.EqualTo(fp2)); } @@ -246,9 +245,8 @@ public void RoundTrip_FullCycle() // Act: save ? load var dbOptions = CreateTestDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - cacheHelper.SaveCache("test_fp", container, dbOptions); - var restored = cacheHelper.LoadContainerFromCache(dbOptions, "test_fp"); + SaveCache("test_fp", container, dbOptions); + var restored = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "test_fp"); // Assert Assert.That(restored, Is.Not.Null); @@ -281,10 +279,9 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() new PackedFileSource(parent, 0, 100, false, false, CompressionFormat.None, 0))); var dbOptions = CreateFileDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - cacheHelper.SaveCache("fp_try", container, dbOptions); + SaveCache("fp_try", container, dbOptions); - var result = cacheHelper.TryLoadFromCache(_dbFilePath, "fp_try"); + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache(_dbFilePath, "fp_try"); Assert.That(result, Is.Not.Null); Assert.That(result.Name, Is.EqualTo("TryLoad Test")); } @@ -292,8 +289,7 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() [Test] public void TryLoadFromCache_ReturnsNullForMissingFile() { - var cacheHelper = new PackFileContainerCacheHelper(); - var result = cacheHelper.TryLoadFromCache( + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache( Path.Combine(_tempDir, "does_not_exist.db"), "fp"); Assert.That(result, Is.Null); } @@ -301,9 +297,8 @@ public void TryLoadFromCache_ReturnsNullForMissingFile() [Test] public void TryLoadFromCache_ReturnsNullForCorruptFile() { - var cacheHelper = new PackFileContainerCacheHelper(); File.WriteAllBytes(_dbFilePath, [0xFF, 0xFE, 0x00, 0x01]); - var result = cacheHelper.TryLoadFromCache(_dbFilePath, "fp"); + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache(_dbFilePath, "fp"); Assert.That(result, Is.Null); } @@ -318,11 +313,9 @@ public void SaveCache_PreservesEncryptedFlag() var parent = new PackedFileSourceParent { FilePath = @"c:\game\encrypted.pack" }; container.AddOrUpdateFile("secret\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 0, 500, isEncrypted: true, isCompressed: false, CompressionFormat.None, 0))); - - var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - cacheHelper.SaveCache("fp", container, dbOptions); - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); var source = (PackedFileSource)loaded!.FindFile("secret\\data.bin")!.DataSource; Assert.That(source.IsEncrypted, Is.True); @@ -335,11 +328,9 @@ public void SaveCache_EmptyContainer_RoundTrips() { SystemFilePath = @"c:\game\empty" }; - - var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - cacheHelper.SaveCache("fp_empty", container, dbOptions); - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp_empty"); + SaveCache("fp_empty", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp_empty"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.Name, Is.EqualTo("Empty Pack")); @@ -351,26 +342,24 @@ public void SaveCache_OverwritesExistingCache() { var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; var dbOptions = CreateFileDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - // Save first version var container1 = new PackFileContainer("Version1") { SystemFilePath = @"c:\game" }; container1.AddOrUpdateFile("old.txt", new PackFile("old.txt", new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - cacheHelper.SaveCache("fp1", container1, dbOptions); + SaveCache("fp1", container1, dbOptions); // Save second version (same db path) var container2 = new PackFileContainer("Version2") { SystemFilePath = @"c:\game" }; container2.AddOrUpdateFile("new.txt", new PackFile("new.txt", new PackedFileSource(parent, 0, 20, false, false, CompressionFormat.None, 0))); - cacheHelper.SaveCache("fp2", container2, dbOptions); + SaveCache("fp2", container2, dbOptions); // Old fingerprint should fail - var oldResult = cacheHelper.LoadContainerFromCache(dbOptions, "fp1"); + var oldResult = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp1"); Assert.That(oldResult, Is.Null); // New fingerprint should work - var newResult = cacheHelper.LoadContainerFromCache(dbOptions, "fp2"); + var newResult = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp2"); Assert.That(newResult, Is.Not.Null); Assert.That(newResult.Name, Is.EqualTo("Version2")); Assert.That(newResult.GetFileCount(), Is.EqualTo(1)); @@ -394,11 +383,10 @@ public void SaveCache_StoresFolderPathCorrectly() new PackedFileSource(parent, 20, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - var cacheHelper = new PackFileContainerCacheHelper(); - cacheHelper.SaveCache("fp", container, dbOptions); + SaveCache("fp", container, dbOptions); // Verify via the CachedPackFileContainer's GetDirectoryContent - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); var rootFiles = loaded.GetDirectoryContent(""); @@ -421,11 +409,9 @@ public void SaveCache_SkipsNonPackedFileSources() new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); container.AddOrUpdateFile("memory.txt", new PackFile("memory.txt", new MemorySource([1, 2, 3]))); - - var cacheHelper = new PackFileContainerCacheHelper(); var dbOptions = CreateTestDbOptions(); - cacheHelper.SaveCache("fp", container, dbOptions); - var loaded = cacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.GetFileCount(), Is.EqualTo(1)); @@ -440,15 +426,13 @@ public void ComputeFingerprint_IgnoresMissingPackFiles() var packDir = Path.Combine(_tempDir, "partial"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "exists.pack"), "data"); - - var cacheHelper = new PackFileContainerCacheHelper(); var packFiles = new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }; - var fp = cacheHelper.ComputeFingerprint(packFiles); + var fp = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp, Is.Not.Null.And.Not.Empty); // Same result regardless of missing file in list - var fp2 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); Assert.That(fp, Is.EqualTo(fp2)); } @@ -460,11 +444,9 @@ public void ComputeFingerprint_OrderIndependent() File.WriteAllText(Path.Combine(packDir, "alpha.pack"), "aaa"); File.WriteAllText(Path.Combine(packDir, "beta.pack"), "bbb"); File.WriteAllText(Path.Combine(packDir, "gamma.pack"), "ccc"); - - var cacheHelper = new PackFileContainerCacheHelper(); - var fp1 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); - var fp2 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); - var fp3 = cacheHelper.ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); + var fp3 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); Assert.That(fp1, Is.EqualTo(fp2)); Assert.That(fp2, Is.EqualTo(fp3)); @@ -473,8 +455,7 @@ public void ComputeFingerprint_OrderIndependent() [Test] public void GetCacheFilePath_SanitizesInvalidChars() { - var cacheHelper = new PackFileContainerCacheHelper(); - var path = cacheHelper.GetCacheFilePath("Game:Name/WithChars", "abc123"); + var path = (new PackFileContainerCacheHelper()).GetCacheFilePath("Game:Name/WithChars", "abc123"); var fileName = Path.GetFileName(path); Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs new file mode 100644 index 000000000..7cb71feaf --- /dev/null +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs @@ -0,0 +1,111 @@ +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles.Models.Containers; +using Shared.Core.PackFiles.Serialization.CacheDatabase; + +namespace Shared.CoreTest.PackFiles.Utility +{ + /// + /// An in-memory implementation of IPackFileContainerCacheHelper that avoids all filesystem I/O. + /// Uses shared in-memory SQLite databases keyed by a logical cache file path. + /// + internal class InMemoryPackFileContainerCacheHelper : IPackFileContainerCacheHelper + { + private readonly PackFileContainerCacheHelper _fingerprintHelper = new(); + + // Keeps shared in-memory SQLite connections alive (shared cache requires at least one open connection) + private readonly Dictionary _keepAliveConnections = new(StringComparer.OrdinalIgnoreCase); + + // Tracks which logical cache paths have been "saved" (i.e. exist) + private readonly HashSet _existingCaches = new(StringComparer.OrdinalIgnoreCase); + + // Tracks which caches have been "corrupted" (for simulating corruption scenarios) + private readonly HashSet _corruptedCaches = new(StringComparer.OrdinalIgnoreCase); + + public string ComputeFingerprint(List packFileNames) + => _fingerprintHelper.ComputeFingerprint(packFileNames); + + public string GetCacheFilePath(string gameName, string cacheId) + { + // Return a logical key, not a real filesystem path + return $"inmemory://{gameName}_{cacheId}"; + } + + public CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath) + { + var dbOptions = GetOrCreateDbOptions(cacheFilePath); + using (var temp = new CachedPackFileContainer(container.Name, dbOptions)) + { + temp.Save(fingerprint, container); + } + + _existingCaches.Add(cacheFilePath); + _corruptedCaches.Remove(cacheFilePath); + + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, fingerprint); + if (loaded == null) + throw new Exception($"Failed to load from cache after saving. CacheFile: {cacheFilePath}"); + + return loaded; + } + + public CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) + { + if (!_existingCaches.Contains(cacheFilePath)) + return null; + + if (_corruptedCaches.Contains(cacheFilePath)) + return null; + + try + { + var dbOptions = GetOrCreateDbOptions(cacheFilePath); + return CachedPackFileContainer.CreateFromFingerPrint(dbOptions, fingerprint); + } + catch + { + return null; + } + } + + /// + /// Simulates cache corruption for a given logical path. + /// + public void CorruptCache(string cacheFilePath) + { + _corruptedCaches.Add(cacheFilePath); + } + + public void Dispose() + { + foreach (var conn in _keepAliveConnections.Values) + conn.Dispose(); + + _keepAliveConnections.Clear(); + _existingCaches.Clear(); + _corruptedCaches.Clear(); + } + + private DbContextOptions GetOrCreateDbOptions(string logicalPath) + { + if (!_keepAliveConnections.TryGetValue(logicalPath, out var connection)) + { + var dbName = "InMemCache_" + Guid.NewGuid().ToString("N"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbName, + Mode = SqliteOpenMode.Memory, + Cache = SqliteCacheMode.Shared + }.ToString(); + + connection = new SqliteConnection(connectionString); + connection.Open(); + _keepAliveConnections[logicalPath] = connection; + } + + return new DbContextOptionsBuilder() + .UseSqlite(connection) + .Options; + } + } +} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs index f1404cdb3..a0c86d2b5 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -1,6 +1,5 @@ using Moq; using Shared.Core.Misc; -using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -15,12 +14,11 @@ internal class PackFileContainerLoaderTests private Mock _waitCursor; private LocalizationManager _localizationManager; private ApplicationSettingsService _settingsService; + private InMemoryPackFileContainerCacheHelper _cacheHelper; [SetUp] public void Setup() { - DirectoryHelper.EnsureCreated(); - _tempGameDir = Path.Combine(Path.GetTempPath(), "AE_LoaderTest_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_tempGameDir); @@ -38,16 +36,20 @@ public void Setup() _settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); _settingsService.CurrentSettings.GameDirectories.Add(new ApplicationSettings.GamePathPair(GameTypeEnum.Warhammer3, _tempGameDir)); + + _cacheHelper = new InMemoryPackFileContainerCacheHelper(); } [TearDown] public void TearDown() { + _cacheHelper.Dispose(); + if (Directory.Exists(_tempGameDir)) Directory.Delete(_tempGameDir, true); } - private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager, new PackFileContainerCacheHelper()); + private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager, _cacheHelper); [Test] public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() @@ -105,17 +107,11 @@ public void LoadAllCaFiles_CacheCorrupted_ShowsCorruptedDialog() var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); - // Find and corrupt the cache file - var cacheDir = DirectoryHelper.CacheDirectory; - var cacheFiles = Directory.GetFiles(cacheDir, "*.db", SearchOption.AllDirectories); - Assert.That(cacheFiles.Length, Is.GreaterThan(0), "Cache file should have been created"); - - // Corrupt all matching cache files - foreach (var cacheFile in cacheFiles) - { - if (cacheFile.Contains("Warhammer")) - File.WriteAllText(cacheFile, "CORRUPTED DATA"); - } + // Corrupt the cache via the in-memory helper + var packFiles = Directory.GetFiles(_tempGameDir, "*.pack").ToList(); + var fingerprint = _cacheHelper.ComputeFingerprint(packFiles); + var cacheFilePath = _cacheHelper.GetCacheFilePath("All Game Packs - Warhammer III", fingerprint); + _cacheHelper.CorruptCache(cacheFilePath); _dialogs.Invocations.Clear(); From c1eef7fa6e5d203187dc7a317571ed57857eca99 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sat, 6 Jun 2026 20:32:36 +0200 Subject: [PATCH 8/9] Code --- .../Containers/CachedPackFileContainer.cs | 68 ++++++++++++++++ .../PackFileContainerTests_TestBase.cs | 81 ++++++++----------- 2 files changed, 101 insertions(+), 48 deletions(-) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 9230c3de6..321a53d53 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -24,6 +24,7 @@ internal class CachedPackFileContainer : IPackFileContainerInternal, IDisposable private CacheDbContext _db; private readonly DbContextOptions _dbOptions; private readonly object _dbLock = new(); + private SqliteConnection? _keepAliveConnection; public CacheStorageMode StorageMode { get; } public string? DbFilePath { get; } @@ -246,9 +247,76 @@ private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSq return (new SqliteConnection(connectionString), true); } + /// + /// Creates a CachedPackFileContainer from a list of file entries. Useful for testing. + /// + public static CachedPackFileContainer CreateFromFileList( + string containerName, + (string RelativePath, string FileName, long Offset, long Size, bool IsEncrypted, bool IsCompressed, CompressionFormat CompressionFormat, uint UncompressedSize)[] files, + bool useInMemoryDb, + string? dbFilePath = null, + string systemFilePath = "", + string? sourcePackFilePath = null) + { + var packParent = new PackedFileSourceParent { FilePath = sourcePackFilePath ?? @"c:\game\data\pack1.pack" }; + + var source = new PackFileContainer(containerName) + { + IsCaPackFile = true, + SystemFilePath = systemFilePath + }; + + if (sourcePackFilePath != null) + source.SourcePackFilePaths.Add(sourcePackFilePath); + else + source.SourcePackFilePaths.Add(packParent.FilePath); + + foreach (var file in files) + source.AddOrUpdateFile(file.RelativePath, new PackFile(file.FileName, new PackedFileSource(packParent, file.Offset, file.Size, file.IsEncrypted, file.IsCompressed, file.CompressionFormat, file.UncompressedSize))); + + DbContextOptions dbOptions; + SqliteConnection? keepAlive = null; + if (useInMemoryDb) + { + var dbName = "CachedContainer_" + Guid.NewGuid().ToString("N"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbName, + Mode = SqliteOpenMode.Memory, + Cache = SqliteCacheMode.Shared + }.ToString(); + + // Open a keep-alive connection for shared in-memory SQLite + keepAlive = new SqliteConnection(connectionString); + keepAlive.Open(); + + dbOptions = new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; + } + else + { + dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + } + + var fingerprint = "factory_fp"; + using (var temp = new CachedPackFileContainer(containerName, dbOptions)) + { + temp.Save(fingerprint, source); + } + + var result = CreateFromFingerPrint(dbOptions, fingerprint)!; + result._keepAliveConnection = keepAlive; + return result; + } + public void Dispose() { _db.Dispose(); + _keepAliveConnection?.Dispose(); + _keepAliveConnection = null; } public int GetFileCount() diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index 41ab62e29..956603bec 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -1,9 +1,6 @@ -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; -using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; namespace Shared.CoreTest.PackFiles.Models.Containers @@ -12,10 +9,28 @@ internal abstract class PackFileContainerTests_TestBase { private readonly bool _useCachedContainer; - private SqliteConnection? _cacheKeepAliveConnection; protected IPackFileContainerInternal _container = null!; protected bool IsCachedContainer => _useCachedContainer; + protected static readonly (string RelativePath, string FileName, long Offset, long Size, bool IsEncrypted, bool IsCompressed, CompressionFormat CompressionFormat, uint UncompressedSize)[] TestFiles = + [ + ("folder\\file.txt", "file.txt", 100, 200, false, false, CompressionFormat.None, 0), + ("other\\data.bin", "data.bin", 300, 400, false, true, CompressionFormat.Lz4, 800), + ("audio\\sound.wem", "sound.wem", 700, 500, false, false, CompressionFormat.None, 0), + ("root_file.txt", "root_file.txt", 0, 10, false, false, CompressionFormat.None, 0), + ("models\\unit.model", "unit.model", 10, 20, false, false, CompressionFormat.None, 0), + ("models\\vehicle.model", "vehicle.model", 30, 40, false, false, CompressionFormat.None, 0), + ("models\\textures\\diffuse.dds", "diffuse.dds", 70, 50, false, false, CompressionFormat.None, 0), + ("models\\textures\\normal.dds", "normal.dds", 120, 60, false, false, CompressionFormat.None, 0), + ("models\\textures\\specular\\gloss.dds", "gloss.dds", 180, 30, false, false, CompressionFormat.None, 0), + ("audio\\music.wem", "music.wem", 210, 100, false, false, CompressionFormat.None, 0), + ("audio\\battle_sound.wem", "battle_sound.wem", 400, 300, false, false, CompressionFormat.None, 0), + ("scripts\\campaign_script.lua", "campaign_script.lua", 850, 80, false, false, CompressionFormat.None, 0), + ("folder_a\\shared.txt", "shared.txt", 900, 10, false, false, CompressionFormat.None, 0), + ("folder_b\\shared.txt", "shared.txt", 910, 20, false, false, CompressionFormat.None, 0), + ("compressed\\data.bin", "data.bin", 1000, 500, true, true, CompressionFormat.Lz4, 2000), + ]; + protected PackFileContainerTests_TestBase(Type containerType) { _useCachedContainer = containerType == typeof(CachedPackFileContainer); @@ -24,52 +39,23 @@ protected PackFileContainerTests_TestBase(Type containerType) [SetUp] public void Setup() { - var sourceContainer = new PackFileContainer("TestCache") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game\data" - }; - sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); - - var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; - sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", new PackedFileSource(parent, 100, 200, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 300, 400, false, true, CompressionFormat.Lz4, 800))); - sourceContainer.AddOrUpdateFile("audio\\sound.wem", new PackFile("sound.wem", new PackedFileSource(parent, 700, 500, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt",new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\unit.model", new PackFile("unit.model", new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\vehicle.model", new PackFile("vehicle.model", new PackedFileSource(parent, 30, 40, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\diffuse.dds", new PackFile("diffuse.dds",new PackedFileSource(parent, 70, 50, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\normal.dds", new PackFile("normal.dds", new PackedFileSource(parent, 120, 60, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\specular\\gloss.dds", new PackFile("gloss.dds",new PackedFileSource(parent, 180, 30, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\music.wem", new PackFile("music.wem", new PackedFileSource(parent, 210, 100, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\battle_sound.wem", new PackFile("battle_sound.wem", new PackedFileSource(parent, 400, 300, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("scripts\\campaign_script.lua", new PackFile("campaign_script.lua",new PackedFileSource(parent, 850, 80, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_a\\shared.txt", new PackFile("shared.txt",new PackedFileSource(parent, 900, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_b\\shared.txt", new PackFile("shared.txt", new PackedFileSource(parent, 910, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("compressed\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 1000, 500, true, true, CompressionFormat.Lz4, 2000))); - if (_useCachedContainer) { - var dbName = "CachedContainerTests_" + Guid.NewGuid().ToString("N"); - var connectionString = new SqliteConnectionStringBuilder - { - DataSource = dbName, - Mode = SqliteOpenMode.Memory, - Cache = SqliteCacheMode.Shared - }.ToString(); - - _cacheKeepAliveConnection = new SqliteConnection(connectionString); - _cacheKeepAliveConnection.Open(); - - var dbOptions = new DbContextOptionsBuilder() - .UseSqlite(connectionString) - .Options; - var cached = new CachedPackFileContainer("test_cache", dbOptions); - cached.Save("test_fp", sourceContainer); - _container = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "test_fp")!; + _container = CachedPackFileContainer.CreateFromFileList("TestCache", TestFiles, useInMemoryDb: true, systemFilePath: @"c:\game\data", sourcePackFilePath: @"c:\game\data\pack1.pack"); } else { + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; + var sourceContainer = new PackFileContainer("TestCache") + { + IsCaPackFile = true, + SystemFilePath = @"c:\game\data" + }; + sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + + foreach (var file in TestFiles) + sourceContainer.AddOrUpdateFile(file.RelativePath, new PackFile(file.FileName, new PackedFileSource(parent, file.Offset, file.Size, file.IsEncrypted, file.IsCompressed, file.CompressionFormat, file.UncompressedSize))); + _container = sourceContainer; } } @@ -77,8 +63,7 @@ public void Setup() [TearDown] public void TearDown() { - _cacheKeepAliveConnection?.Dispose(); - _cacheKeepAliveConnection = null; + (_container as IDisposable)?.Dispose(); } protected void IgnoreIfNotCached(string scenario) From a1bb047f6b893fce89d13eab20d59b29b56474cf Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sat, 6 Jun 2026 20:58:59 +0200 Subject: [PATCH 9/9] Code --- .../PackFileContainerTests_GetAllFiles.cs | 2 +- ...kFileContainerTests_GetAllFilesByFolder.cs | 5 +- .../PackFileContainerTests_GetFileCount.cs | 2 +- .../PackFileContainerTests_SearchFiles.cs | 73 +++---------------- .../PackFileContainerTests_TestBase.cs | 3 + 5 files changed, 18 insertions(+), 67 deletions(-) diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs index 69b345954..5c00a013a 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs @@ -14,7 +14,7 @@ public PackFileContainerTests_GetAllFiles(Type containerType) : base(containerTy public void GetAllFiles_ReturnsExpectedCountAndKeys() { var files = _container.GetAllFiles(); - Assert.That(files.Count, Is.EqualTo(15)); + Assert.That(files.Count, Is.EqualTo(18)); Assert.That(files.ContainsKey("folder\\file.txt"), Is.True); Assert.That(files.ContainsKey("scripts\\campaign_script.lua"), Is.True); } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs index 9f1c4fcbb..2a6b0ca61 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs @@ -56,7 +56,10 @@ public void GetAllFilesByFolder_FolderContainsCorrectFiles() Assert.That(audioFiles, Does.Contain("sound.wem")); Assert.That(audioFiles, Does.Contain("music.wem")); Assert.That(audioFiles, Does.Contain("battle_sound.wem")); - Assert.That(audioFiles.Count, Is.EqualTo(3)); + Assert.That(audioFiles, Does.Contain("voice.wem_temp")); + Assert.That(audioFiles, Does.Contain("voice.wem.{sdf}")); + Assert.That(audioFiles, Does.Contain("voice.txt")); + Assert.That(audioFiles.Count, Is.EqualTo(6)); } [Test] diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs index 8b5e2355b..2c9714d35 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs @@ -11,7 +11,7 @@ public PackFileContainerTests_GetFileCount(Type containerType) : base(containerT [Test] public void GetFileCount_ReturnsMasterDatasetCount() { - Assert.That(_container.GetFileCount(), Is.EqualTo(15)); + Assert.That(_container.GetFileCount(), Is.EqualTo(18)); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs index 86cadcf4b..ae091210b 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs @@ -1,10 +1,4 @@ -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; -using Shared.Core.PackFiles.Models; -using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization.CacheDatabase; -using Shared.Core.PackFiles.Utility; +using Shared.Core.PackFiles.Models.Containers; namespace Shared.CoreTest.PackFiles.Models.Containers { @@ -18,7 +12,7 @@ public PackFileContainerTests_SearchFiles(Type containerType) : base(containerTy public void SearchFiles_NullFilters_ReturnsAll() { var results = _container.SearchFiles(null, null); - Assert.That(results.Count, Is.EqualTo(15)); + Assert.That(results.Count, Is.EqualTo(18)); } [Test] @@ -41,68 +35,19 @@ public void SearchFiles_ResultsArePathSorted() public void SearchFiles_EmptyExtensionList_TreatedAsNoFilter() { var results = _container.SearchFiles(null, []); - Assert.That(results.Count, Is.EqualTo(15)); + Assert.That(results.Count, Is.EqualTo(18)); } [Test] public void SearchFiles_ExtensionFilter_MatchesFilenameSubstrings_ForLegacyWemVariants() { - var sourceContainer = new PackFileContainer("SearchFilesVariants") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game\data" - }; - sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + var results = _container.SearchFiles(null, [".wem"]); + var paths = results.Select(x => x.Path).ToList(); - var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; - sourceContainer.AddOrUpdateFile("audio\\voice.wem", new PackFile("voice.wem", new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.wem_temp", new PackFile("voice.wem_temp", new PackedFileSource(parent, 10, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.wem.{sdf}", new PackFile("voice.wem.{sdf}", new PackedFileSource(parent, 20, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.txt", new PackFile("voice.txt", new PackedFileSource(parent, 30, 10, false, false, CompressionFormat.None, 0))); - - SqliteConnection? keepAliveConnection = null; - IPackFileContainer container; - try - { - if (IsCachedContainer) - { - var dbName = "SearchFilesVariants_" + Guid.NewGuid().ToString("N"); - var connectionString = new SqliteConnectionStringBuilder - { - DataSource = dbName, - Mode = SqliteOpenMode.Memory, - Cache = SqliteCacheMode.Shared - }.ToString(); - - // Keep one connection alive so the shared in-memory DB remains available - // while SaveCache and LoadContainerFromCache use separate EF/raw connections. - keepAliveConnection = new SqliteConnection(connectionString); - keepAliveConnection.Open(); - - var dbOptions = new DbContextOptionsBuilder() - .UseSqlite(connectionString) - .Options; - var cached = new CachedPackFileContainer("variants_cache", dbOptions); - cached.Save("variants_fp", sourceContainer); - container = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "variants_fp")!; - } - else - { - container = sourceContainer; - } - - var results = container.SearchFiles(null, [".wem"]); - var paths = results.Select(x => x.Path).ToList(); - - Assert.That(paths, Does.Contain(@"audio\voice.wem")); - Assert.That(paths, Does.Contain(@"audio\voice.wem_temp")); - Assert.That(paths, Does.Contain(@"audio\voice.wem.{sdf}")); - Assert.That(paths, Does.Not.Contain(@"audio\voice.txt")); - } - finally - { - keepAliveConnection?.Dispose(); - } + Assert.That(paths, Does.Contain(@"audio\sound.wem")); + Assert.That(paths, Does.Contain(@"audio\voice.wem_temp")); + Assert.That(paths, Does.Contain(@"audio\voice.wem.{sdf}")); + Assert.That(paths, Does.Not.Contain(@"audio\voice.txt")); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index 956603bec..29ab66839 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -29,6 +29,9 @@ protected static readonly (string RelativePath, string FileName, long Offset, lo ("folder_a\\shared.txt", "shared.txt", 900, 10, false, false, CompressionFormat.None, 0), ("folder_b\\shared.txt", "shared.txt", 910, 20, false, false, CompressionFormat.None, 0), ("compressed\\data.bin", "data.bin", 1000, 500, true, true, CompressionFormat.Lz4, 2000), + ("audio\\voice.wem_temp", "voice.wem_temp", 1500, 10, false, false, CompressionFormat.None, 0), + ("audio\\voice.wem.{sdf}", "voice.wem.{sdf}", 1510, 10, false, false, CompressionFormat.None, 0), + ("audio\\voice.txt", "voice.txt", 1520, 10, false, false, CompressionFormat.None, 0), ]; protected PackFileContainerTests_TestBase(Type containerType)