diff --git a/AssetEditor/App.xaml.cs b/AssetEditor/App.xaml.cs index a243aeade..c9ad852a4 100644 --- a/AssetEditor/App.xaml.cs +++ b/AssetEditor/App.xaml.cs @@ -100,7 +100,7 @@ private void LoadCAPackFiles(ApplicationSettingsService settingsService) { var packfileService = _serviceProvider.GetRequiredService(); var containerLoader = _serviceProvider.GetRequiredService(); - var loadRes = containerLoader.LoadAllCaFiles(settingsService.CurrentSettings.CurrentGame); + var loadRes = containerLoader.CreateFromGameEnum(PackFileContainerType.Cached, settingsService.CurrentSettings.CurrentGame); if (loadRes == null) MessageBox.Show($"Unable to load all CA packfiles in {gamePath}"); diff --git a/AssetEditor/UiCommands/OpenGamePackCommand.cs b/AssetEditor/UiCommands/OpenGamePackCommand.cs index 61381680a..e1c9129a4 100644 --- a/AssetEditor/UiCommands/OpenGamePackCommand.cs +++ b/AssetEditor/UiCommands/OpenGamePackCommand.cs @@ -50,7 +50,7 @@ public void Execute() using (new WaitCursor()) { - var res = _packFileContainerLoader.LoadAllCaFiles(_game); + var res = _packFileContainerLoader.CreateFromGameEnum(PackFileContainerType.Cached, _game); _packFileService.AddContainer(res); } } diff --git a/AssetEditor/UiCommands/OpenPackFileCommand.cs b/AssetEditor/UiCommands/OpenPackFileCommand.cs index 274eac54b..9862b81b8 100644 --- a/AssetEditor/UiCommands/OpenPackFileCommand.cs +++ b/AssetEditor/UiCommands/OpenPackFileCommand.cs @@ -25,7 +25,7 @@ public void Execute() if (dialog.ShowDialog() != DialogResult.OK) return; - var container = _packFileContainerLoader.Load(dialog.FileName); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, dialog.FileName, false); _packFileService.AddContainer(container, true); } } diff --git a/AssetEditor/ViewModels/MenuBarViewModel.cs b/AssetEditor/ViewModels/MenuBarViewModel.cs index 31c959e7c..2ef5629e2 100644 --- a/AssetEditor/ViewModels/MenuBarViewModel.cs +++ b/AssetEditor/ViewModels/MenuBarViewModel.cs @@ -168,7 +168,7 @@ void CreateRecentPackFilesItems() path, () => { - var container = _packFileContainerLoader.Load(path); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, path, false); if (container == null) { System.Windows.MessageBox.Show($"Unable to load packfiles {path}"); diff --git a/Editors/Ipc/IpcEditor/ExternalPackLoader.cs b/Editors/Ipc/IpcEditor/ExternalPackLoader.cs index 22ff291ac..c26a745c8 100644 --- a/Editors/Ipc/IpcEditor/ExternalPackLoader.cs +++ b/Editors/Ipc/IpcEditor/ExternalPackLoader.cs @@ -40,7 +40,7 @@ public Task EnsureLoadedAsync(string packPathOnDisk, Cancellatio try { - var container = _packFileContainerLoader.Load(normalizedDiskPath); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, normalizedDiskPath, true); if (container == null) return Task.FromResult(PackLoadResult.Fail("Pack file could not be loaded")); diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs index e5e3430dd..c04354270 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Karl.cs @@ -34,7 +34,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs index 842a5c5bb..a2d65923c 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_MeshFitter.cs @@ -46,7 +46,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\AnimationTransfer_bear.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs index 97e8f3523..6fe72502e 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Ox.cs @@ -33,7 +33,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\cinderbreath.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs index 63bc839f1..bb6395206 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_Rat.cs @@ -45,7 +45,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs index 77629ba19..012d39321 100644 --- a/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs +++ b/Editors/Kitbashing/KitbasherEditor/DevConfig/Kitbash_RomeShield.cs @@ -33,7 +33,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.CurrentGame = GameTypeEnum.Rome2; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Rome_Man_And_Shield_Pack"; - var container = _packFileContainerLoader.LoadSystemFolderAsPackFileContainer(packFile); + var container = _packFileContainerLoader.CreateFromSystemFolder(packFile); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs b/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs index 48a0f57d2..263b437cb 100644 --- a/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs +++ b/Editors/MetaDataEditor/AnimationMeta/DevConfig/AnimMetaTool.cs @@ -26,7 +26,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container!.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs b/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs index 5ea470a3f..bdd07184a 100644 --- a/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs +++ b/Editors/MetaDataEditor/AnimationMeta/DevConfig/SuperView_Rat.cs @@ -28,7 +28,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.CurrentGame = GameTypeEnum.Warhammer3; currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Throt.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container!.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/Editors/Reports/DeepSearch/DeepSearchReport.cs b/Editors/Reports/DeepSearch/DeepSearchReport.cs index f5515c4c8..3110236f0 100644 --- a/Editors/Reports/DeepSearch/DeepSearchReport.cs +++ b/Editors/Reports/DeepSearch/DeepSearchReport.cs @@ -73,7 +73,7 @@ public List DeepSearch(string searchStr, bool caseSensetive) { using (var reader = new BinaryReader(fileStram, Encoding.ASCII)) { - var pfc = _loader.Load(packFilePath); + var pfc = _loader.CreateFromPackFile(PackFileContainerType.Normal, packFilePath, true); _logger.Here().Information($"Searching through packfile {currentIndex}/{files.Count} - {packFilePath} {pfc.GetFileCount()} files"); diff --git a/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs b/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs index a88fda5c8..2e2c144f1 100644 --- a/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs +++ b/Editors/SkeletonEditor/Editor.VisualSkeletonEditor/DevConfig/SkeletonTool.cs @@ -28,7 +28,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); Guard.IsNotNull(container, "Failed to load pack file container for development config."); container.IsCaPackFile = true; _packFileService.AddContainer(container); diff --git a/Editors/TextureEditor/DevConfig/Texture_Karl.cs b/Editors/TextureEditor/DevConfig/Texture_Karl.cs index d50766aac..a520503ea 100644 --- a/Editors/TextureEditor/DevConfig/Texture_Karl.cs +++ b/Editors/TextureEditor/DevConfig/Texture_Karl.cs @@ -31,7 +31,7 @@ public void OverrideSettings(ApplicationSettings currentSettings) { currentSettings.LoadCaPacksByDefault = false; var packFile = ResourceLoader.GetDevelopmentDataFolder() + "\\Karl_and_celestialgeneral.pack"; - var container = _packFileContainerLoader.Load(packFile); + var container = _packFileContainerLoader.CreateFromPackFile(PackFileContainerType.Normal, packFile, true); container.IsCaPackFile = true; _packFileService.AddContainer(container); } diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs index 73a43df35..b768714de 100644 --- a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs @@ -6,6 +6,7 @@ using Shared.Core.Events.Global; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -32,9 +33,9 @@ public void LoadAndUnload_KarlPack_UpdatesSkeletonAndAnimationLookup() .Returns(() => containers.ToList()); var settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); - var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager()); + var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); var karlPackPath = PathHelper.GetDataFile("Karl_and_celestialgeneral.pack"); - var karlContainer = loader.Load(karlPackPath); + var karlContainer = loader.CreateFromPackFile(PackFileContainerType.Normal, karlPackPath, false); Assert.That(karlContainer, Is.Not.Null); diff --git a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs index 625f1ef90..c91e23ecf 100644 --- a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs +++ b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs @@ -6,6 +6,7 @@ using Shared.Core.Events; using Shared.Core.Misc; using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -32,9 +33,9 @@ public override void Register(IServiceCollection services) services.AddSingleton(); services.AddScoped(); services.AddScoped(); - + services.AddScoped(); services.AddSingleton(); services.AddScoped(); @@ -43,6 +44,7 @@ public override void Register(IServiceCollection services) services.AddSingleton(); + services.AddScoped(); services.AddTransient(); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 0c81f2558..321a53d53 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,6 +1,7 @@ -using System.Diagnostics; -using System.Windows.Forms; +using System.Diagnostics; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization.CacheDatabase; @@ -9,26 +10,315 @@ namespace Shared.Core.PackFiles.Models.Containers { - internal class CachedPackFileContainer : IPackFileContainerInternal + internal enum CacheStorageMode { - private readonly ILogger _logger = Logging.Create(); + File, + InMemory + } + + internal class CachedPackFileContainer : IPackFileContainerInternal, IDisposable + { + private static readonly ILogger _logger = Logging.CreateStatic(typeof(CachedPackFileContainer)); + private const int CurrentSchemaVersion = 3; - private readonly CacheDbContext _db; + private CacheDbContext _db; + private readonly DbContextOptions _dbOptions; private readonly object _dbLock = new(); + private SqliteConnection? _keepAliveConnection; + public CacheStorageMode StorageMode { get; } + public string? DbFilePath { get; } public string Name { get; set; } public bool IsCaPackFile { get => true; set { } } public string SystemFilePath { get; set; } public HashSet SourcePackFilePaths { get; set; } = []; + /// + /// Creates a file-backed CachedPackFileContainer. + /// + public CachedPackFileContainer(string name, string dbFilePath) + { + Name = name; + SystemFilePath = string.Empty; + DbFilePath = dbFilePath; + StorageMode = CacheStorageMode.File; + _dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + _db = new CacheDbContext(_dbOptions); + _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + } + + /// + /// Creates an in-memory CachedPackFileContainer. + /// public CachedPackFileContainer(string name, DbContextOptions dbOptions) { Name = name; SystemFilePath = string.Empty; + DbFilePath = null; + StorageMode = CacheStorageMode.InMemory; + _dbOptions = dbOptions; _db = new CacheDbContext(dbOptions); _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; } + /// + /// Saves the contents of a PackFileContainer into this cached container's database. + /// + public void Save(string fingerprint, PackFileContainer source) + { + _logger.Here().Information($"Saving cache for '{source.Name}' with {source.GetFileCount()} files"); + + // Use EF to create the schema + using (var db = new CacheDbContext(_dbOptions)) + { + db.Database.EnsureDeleted(); + db.Database.EnsureCreated(); + } + + var (connection, shouldDisposeConnection) = GetSqliteConnection(_dbOptions); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + + try + { + using var transaction = connection.BeginTransaction(); + + using (var cmd = connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO CacheInfo (SchemaVersion, Fingerprint, ContainerName, SystemFilePath, SourcePackFilePaths) + VALUES ($schemaVersion, $fingerprint, $containerName, $systemFilePath, $sourcePackFilePaths)"; + cmd.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); + cmd.Parameters.AddWithValue("$fingerprint", fingerprint); + cmd.Parameters.AddWithValue("$containerName", source.Name); + cmd.Parameters.AddWithValue("$systemFilePath", source.SystemFilePath); + cmd.Parameters.AddWithValue("$sourcePackFilePaths", string.Join("|", source.SourcePackFilePaths)); + cmd.ExecuteNonQuery(); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO FileList (RelativePath, FileName, Extension, FolderPath, SourcePackFilePath, Offset, Size, IsEncrypted, IsCompressed, CompressionFormat, UncompressedSize) + VALUES ($relativePath, $fileName, $extension, $folderPath, $sourcePackFilePath, $offset, $size, $isEncrypted, $isCompressed, $compressionFormat, $uncompressedSize)"; + + var pRelativePath = cmd.Parameters.Add("$relativePath", SqliteType.Text); + var pFileName = cmd.Parameters.Add("$fileName", SqliteType.Text); + var pExtension = cmd.Parameters.Add("$extension", SqliteType.Text); + var pFolderPath = cmd.Parameters.Add("$folderPath", SqliteType.Text); + var pSourcePackFilePath = cmd.Parameters.Add("$sourcePackFilePath", SqliteType.Text); + var pOffset = cmd.Parameters.Add("$offset", SqliteType.Integer); + var pSize = cmd.Parameters.Add("$size", SqliteType.Integer); + var pIsEncrypted = cmd.Parameters.Add("$isEncrypted", SqliteType.Integer); + var pIsCompressed = cmd.Parameters.Add("$isCompressed", SqliteType.Integer); + var pCompressionFormat = cmd.Parameters.Add("$compressionFormat", SqliteType.Integer); + var pUncompressedSize = cmd.Parameters.Add("$uncompressedSize", SqliteType.Integer); + + cmd.Prepare(); + + foreach (var (relativePath, packFile) in source.GetAllFiles()) + { + if (packFile.DataSource is not PackedFileSource fileSource) + continue; + + var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); + var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); + + pRelativePath.Value = relativePath; + pFileName.Value = packFile.Name; + pExtension.Value = Path.GetExtension(relativePath).ToLower(); + pFolderPath.Value = folderPath; + pSourcePackFilePath.Value = fileSource.Parent.FilePath; + pOffset.Value = fileSource.Offset; + pSize.Value = fileSource.Size; + pIsEncrypted.Value = fileSource.IsEncrypted ? 1 : 0; + pIsCompressed.Value = fileSource.IsCompressed ? 1 : 0; + pCompressionFormat.Value = (int)fileSource.CompressionFormat; + pUncompressedSize.Value = (long)fileSource.UncompressedSize; + + cmd.ExecuteNonQuery(); + } + } + + transaction.Commit(); + _logger.Here().Information($"Cache saved successfully for '{source.Name}'"); + } + finally + { + if (shouldDisposeConnection) + connection.Dispose(); + } + + // Update this container's metadata from what was saved + Name = source.Name; + SystemFilePath = source.SystemFilePath ?? string.Empty; + SourcePackFilePaths = new HashSet(source.SourcePackFilePaths); + + // Recreate _db so this instance can be queried after Save() + _db.Dispose(); + _db = new CacheDbContext(_dbOptions); + _db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + } + + /// + /// Loads a CachedPackFileContainer from a file-backed database. + /// Returns null if the file doesn't exist, the DB is corrupt, or the fingerprint doesn't match. + /// + public static CachedPackFileContainer? CreateFromFingerPrint(string dbFilePath, string expectedFingerprint) + { + if (!File.Exists(dbFilePath)) + { + _logger.Here().Information($"No cache file found at '{dbFilePath}'"); + return null; + } + + var dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + + return CreateFromFingerPrint(dbOptions, expectedFingerprint); + } + + /// + /// Loads a CachedPackFileContainer from the given DbContextOptions (file or in-memory). + /// Returns null if the DB is corrupt or the fingerprint doesn't match. + /// + public static CachedPackFileContainer? CreateFromFingerPrint(DbContextOptions dbOptions, string expectedFingerprint) + { + using var db = new CacheDbContext(dbOptions); + + try + { + db.Database.EnsureCreated(); + } + catch (Exception ex) + { + _logger.Here().Warning($"Failed to open cache database: {ex.Message}"); + return null; + } + + CacheInfoEntity? cacheInfo; + try + { + cacheInfo = db.CacheInfo.FirstOrDefault(); + } + catch (Exception ex) + { + _logger.Here().Warning($"Failed to read cache info: {ex.Message}"); + return null; + } + + if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) + { + _logger.Here().Information($"Cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); + return null; + } + + var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) + { + SystemFilePath = cacheInfo.SystemFilePath, + }; + + if (!string.IsNullOrEmpty(cacheInfo.SourcePackFilePaths)) + { + foreach (var path in cacheInfo.SourcePackFilePaths.Split('|')) + container.SourcePackFilePaths.Add(path); + } + + _logger.Here().Information($"Loaded container '{container.Name}' from cache"); + return container; + } + + private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) + { + var relationalOptions = dbOptions.Extensions + .OfType() + .FirstOrDefault(); + + if (relationalOptions?.Connection is SqliteConnection sqliteConnection) + return (sqliteConnection, false); + + var connectionString = relationalOptions?.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + throw new InvalidOperationException("Unable to resolve SQLite connection from DbContextOptions."); + + return (new SqliteConnection(connectionString), true); + } + + /// + /// Creates a CachedPackFileContainer from a list of file entries. Useful for testing. + /// + public static CachedPackFileContainer CreateFromFileList( + string containerName, + (string RelativePath, string FileName, long Offset, long Size, bool IsEncrypted, bool IsCompressed, CompressionFormat CompressionFormat, uint UncompressedSize)[] files, + bool useInMemoryDb, + string? dbFilePath = null, + string systemFilePath = "", + string? sourcePackFilePath = null) + { + var packParent = new PackedFileSourceParent { FilePath = sourcePackFilePath ?? @"c:\game\data\pack1.pack" }; + + var source = new PackFileContainer(containerName) + { + IsCaPackFile = true, + SystemFilePath = systemFilePath + }; + + if (sourcePackFilePath != null) + source.SourcePackFilePaths.Add(sourcePackFilePath); + else + source.SourcePackFilePaths.Add(packParent.FilePath); + + foreach (var file in files) + source.AddOrUpdateFile(file.RelativePath, new PackFile(file.FileName, new PackedFileSource(packParent, file.Offset, file.Size, file.IsEncrypted, file.IsCompressed, file.CompressionFormat, file.UncompressedSize))); + + DbContextOptions dbOptions; + SqliteConnection? keepAlive = null; + if (useInMemoryDb) + { + var dbName = "CachedContainer_" + Guid.NewGuid().ToString("N"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbName, + Mode = SqliteOpenMode.Memory, + Cache = SqliteCacheMode.Shared + }.ToString(); + + // Open a keep-alive connection for shared in-memory SQLite + keepAlive = new SqliteConnection(connectionString); + keepAlive.Open(); + + dbOptions = new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; + } + else + { + dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; + } + + var fingerprint = "factory_fp"; + using (var temp = new CachedPackFileContainer(containerName, dbOptions)) + { + temp.Save(fingerprint, source); + } + + var result = CreateFromFingerPrint(dbOptions, fingerprint)!; + result._keepAliveConnection = keepAlive; + return result; + } + + public void Dispose() + { + _db.Dispose(); + _keepAliveConnection?.Dispose(); + _keepAliveConnection = null; + } + public int GetFileCount() { lock (_dbLock) @@ -132,23 +422,45 @@ public bool ContainsFile(string path) return entries.Select(e => (e.RelativePath, ToPackFile(e))).ToList(); } + public SortedDictionary> GetAllFilesByFolder() + { + var time = Stopwatch.StartNew(); + lock (_dbLock) + { + + var rows = _db.Files + .Select(f => new + { + f.FolderPath, + f.FileName + }) + .OrderBy(f => f.FolderPath) + .ThenBy(f => f.FileName) + .ToList(); + + var result = rows + .GroupBy(f => f.FolderPath) + .ToList(); + + var sorted = new SortedDictionary>(StringComparer.InvariantCultureIgnoreCase); + foreach (var re in result) + { + sorted[re.Key] = re.Select(f => f.FileName).OrderByDescending(f=>f).ToList(); + + + } + + _logger.Here().Information("Getting all files from cached container took {ElapsedMilliseconds} ms", time.ElapsedMilliseconds); + return sorted; + } + } + public Dictionary GetAllFiles() { var time = Stopwatch.StartNew(); List entries; lock (_dbLock) { - //var x = _db.Files - // .GroupBy(x=>x.FolderPath) - // .Select(g => new - // { - // Folder = g.Key, - // FileList = g.Select(x => x.FileName).ToList() - // }) - // .ToList(); - // - // - entries = _db.Files.ToList(); } _logger.Here().Information("Getting all files from cached container took {ElapsedMilliseconds} ms", time.ElapsedMilliseconds); @@ -221,37 +533,6 @@ public Dictionary GetAllFiles() return files; } - public List GetSubDirectories(string directoryPath) - { - var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; - var prefixLength = prefix.Length; - - List folderPaths; - lock (_dbLock) - { - folderPaths = _db.Files - .Where(f => string.IsNullOrEmpty(directoryPath) - ? f.FolderPath != "" - : f.FolderPath.StartsWith(prefix)) - .Select(f => f.FolderPath) - .ToList(); - } - - return folderPaths - .Select(folderPath => string.IsNullOrEmpty(directoryPath) - ? folderPath - : folderPath.Substring(prefixLength)) - .Select(candidate => - { - var separatorIndex = candidate.IndexOf('\\'); - return separatorIndex == -1 ? candidate : candidate.Substring(0, separatorIndex); - }) - .Where(folderName => !string.IsNullOrWhiteSpace(folderName)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - } - public void AddOrUpdateFile(string path, PackFile file) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index 4978d807e..f65101db8 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -1,4 +1,5 @@ using CommunityToolkit.Diagnostics; +using Shared.Core.ErrorHandling; using Shared.Core.Misc; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization; @@ -9,6 +10,8 @@ namespace Shared.Core.PackFiles.Models.Containers { internal class PackFileContainer : IPackFileContainerInternal { + private static readonly ILogger _logger = Logging.Create(); + public string Name { get; set; } public PFHeader Header { get; set; } public bool IsCaPackFile { get; set; } = false; @@ -25,7 +28,6 @@ public PackFileContainer(string name) Header = new PFHeader(v, PackFileCAType.MOD); } - public int GetFileCount() => FileList.Count; public void AddOrUpdateFile(string path, PackFile file) @@ -53,25 +55,6 @@ public void AddOrUpdateFile(string path, PackFile file) return results; } - public List GetSubDirectories(string directoryPath) - { - var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; - var subFolders = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var path in FileList.Keys) - { - if (!string.IsNullOrEmpty(prefix) && !path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) - continue; - - var remainder = string.IsNullOrEmpty(prefix) ? path : path.Substring(prefix.Length); - var separatorIndex = remainder.IndexOf(Path.DirectorySeparatorChar); - if (separatorIndex > 0) - subFolders.Add(remainder.Substring(0, separatorIndex)); - } - - return subFolders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(); - } - public List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention) { extention = extention.ToLower(); @@ -118,8 +101,6 @@ public List GetSubDirectories(string directoryPath) return results; } - - public void MergePackFileContainer(PackFileContainer other) { foreach (var item in other.GetAllFiles()) @@ -251,8 +232,21 @@ public virtual void SaveFileData(PackFile file, byte[] data) public virtual void SaveToDisk(string path, bool createBackup, GameInformation gameInformation) { + _logger.Here().Information("Saving pack file to disk at {Path}, createBackup = {createBackup} Game = {game}", path, createBackup, gameInformation.DisplayName); + if (File.Exists(path) && DirectoryHelper.IsFileLocked(path)) - throw new IOException($"Cannot access {path} because another process has locked it, most likely the game."); + { + var msg = $"Cannot access {path} because another process has locked it, most likely the game."; + _logger.Here().Error(msg); + throw new IOException(msg); + } + + if (File.Exists(path + "_temp") && DirectoryHelper.IsFileLocked(path + "_temp")) + { + var msg = $"Cannot access {path + "_temp"} because another process has locked it, most likely the game."; + _logger.Here().Error(msg); + throw new IOException(msg); + } if (createBackup) SaveUtility.CreateFileBackup(path); @@ -272,11 +266,16 @@ public virtual void SaveToDisk(string path, bool createBackup, GameInformation g PackFileSerializerWriter.SaveToByteArray(path, this, writer, gameInformation); } + _logger.Here().Information("Finished writing pack file to temporary file, now replacing original file with the new one"); File.Delete(path); + + _logger.Here().Information("Original file deleted, now moving temporary file to original file path"); File.Move(path + "_temp", path); SystemFilePath = path; OriginalLoadByteSize = new FileInfo(path).Length; + + _logger.Here().Information("SaveToDisk completed"); } private static string BuildPackPath(string? directoryPath, string fileName) @@ -293,5 +292,27 @@ private static string BuildPackPath(string? directoryPath, string fileName) return PathNormalization.NormalizeFileName(fullPath); } + + public SortedDictionary> GetAllFilesByFolder() + { + var result = new SortedDictionary>(StringComparer.InvariantCultureIgnoreCase); + + foreach (var fullPath in FileList.Keys) + { + var lastSep = fullPath.LastIndexOf('\\'); + var folder = lastSep == -1 ? string.Empty : fullPath.Substring(0, lastSep); + var fileName = lastSep == -1 ? fullPath : fullPath.Substring(lastSep + 1); + + if (!result.TryGetValue(folder, out var files)) + { + files = new List(); + result[folder] = files; + } + + files.Add(fileName); + } + + return result; + } } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs index 04ec72cf6..37c37a8b4 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs @@ -13,8 +13,8 @@ public interface IPackFileContainer string? GetFullPath(PackFile file); Dictionary GetAllFiles(); - - + SortedDictionary> GetAllFilesByFolder(); + List<(string Path, PackFile File)> SearchFiles(string? textFilter, IReadOnlyList? extensions); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs index e1cc13812..ddb52d970 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainerInternal.cs @@ -15,7 +15,8 @@ internal interface IPackFileContainerInternal : IPackFileContainer void SaveToDisk(string path, bool createBackup, GameInformation gameInformation); List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention); - List GetSubDirectories(string directoryPath); List<(string Path, PackFile File)> GetDirectoryContent(string directoryPath); + + } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 53059ee8d..9b20bd1a9 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -1,34 +1,44 @@ -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; using Shared.Core.ErrorHandling; using Shared.Core.Misc; using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Models.FileSources; namespace Shared.Core.PackFiles.Serialization.CacheDatabase { - internal static class PackFileContainerCacheHelper + interface IPackFileContainerCacheHelper { - private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerCacheHelper)); - public static string GetCacheFilePath(string gameDataFolder, string gameName, string cacheId) + string ComputeFingerprint(List packFileNames); + string GetCacheFilePath(string gameName, string cacheId); + CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint); + CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath); + } + + class PackFileContainerCacheHelper : IPackFileContainerCacheHelper + { + private readonly ILogger _logger = Logging.Create(); + public string GetCacheFilePath(string gameName, string cacheId) { var safeGameName = string.Join("_", gameName.Split(Path.GetInvalidFileNameChars())); return Path.Combine(DirectoryHelper.CacheDirectory, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); } - public static string ComputeFingerprint(string gameDataFolder, List packFileNames) + public string ComputeFingerprint(List packFileNames) { using var sha = SHA256.Create(); var sb = new StringBuilder(); + if (packFileNames.Count == 0) + throw new Exception("Trying to compute CachecFingerPrint, but no files provied"); + + var foundFiles = 0; foreach (var packFileName in packFileNames.OrderBy(x => x, StringComparer.OrdinalIgnoreCase)) { - var fullPath = Path.Combine(gameDataFolder, packFileName); - if (File.Exists(fullPath)) + if (File.Exists(packFileName)) { - var info = new FileInfo(fullPath); + foundFiles++; + + var info = new FileInfo(packFileName); sb.Append(packFileName); sb.Append('|'); sb.Append(info.Length); @@ -36,213 +46,34 @@ public static string ComputeFingerprint(string gameDataFolder, List pack sb.Append(info.LastWriteTimeUtc.Ticks); sb.Append(';'); } - } - - var manifestPath = Path.Combine(gameDataFolder, "manifest.txt"); - if (File.Exists(manifestPath)) - { - var info = new FileInfo(manifestPath); - sb.Append("manifest.txt|"); - sb.Append(info.Length); - sb.Append('|'); - sb.Append(info.LastWriteTimeUtc.Ticks); - } - - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString())); - return Convert.ToHexString(hash); - } - - public static DbContextOptions CreateDbOptions(string dbFilePath) - { - return new DbContextOptionsBuilder() - .UseSqlite($"Data Source={dbFilePath};Pooling=False") - .Options; - } - - public static DbContextOptions CreateDbOptionsFromConnectionString(string connectionString) - { - return new DbContextOptionsBuilder() - .UseSqlite(connectionString) - .Options; - } - - public static DbContextOptions CreateDbOptions(SqliteConnection connection) - { - if (connection.State != System.Data.ConnectionState.Open) - connection.Open(); - - return new DbContextOptionsBuilder() - .UseSqlite(connection) - .Options; - } - - private const int CurrentSchemaVersion = 3; - - public static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) - { - _logger.Here().Information($"Saving cache for '{container.Name}' with {container.GetFileCount()} files"); - - // Use EF only to create the schema, then dispose to release memory - using (var db = new CacheDbContext(dbOptions)) - { - db.Database.EnsureDeleted(); - db.Database.EnsureCreated(); - } - - // Use raw SQLite for bulk insert - avoids EF change tracker memory and perf overhead. - // If dbOptions carries an explicit SqliteConnection (for tests/in-memory), reuse it. - var (connection, shouldDisposeConnection) = GetSqliteConnection(dbOptions); - if (connection.State != System.Data.ConnectionState.Open) - connection.Open(); - - try - { - using var transaction = connection.BeginTransaction(); - - // Insert CacheInfo - using (var cmd = connection.CreateCommand()) + else { - cmd.Transaction = transaction; - cmd.CommandText = @"INSERT INTO CacheInfo (SchemaVersion, Fingerprint, ContainerName, SystemFilePath, SourcePackFilePaths) - VALUES ($schemaVersion, $fingerprint, $containerName, $systemFilePath, $sourcePackFilePaths)"; - cmd.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); - cmd.Parameters.AddWithValue("$fingerprint", fingerprint); - cmd.Parameters.AddWithValue("$containerName", container.Name); - cmd.Parameters.AddWithValue("$systemFilePath", container.SystemFilePath); - cmd.Parameters.AddWithValue("$sourcePackFilePaths", string.Join("|", container.SourcePackFilePaths)); - cmd.ExecuteNonQuery(); + _logger.Here().Warning($"Trying to compute CachecFingerPrint, but file {packFileName} is not found"); } - - // Bulk insert files using a prepared statement - using (var cmd = connection.CreateCommand()) - { - cmd.Transaction = transaction; - cmd.CommandText = @"INSERT INTO FileList (RelativePath, FileName, Extension, FolderPath, SourcePackFilePath, Offset, Size, IsEncrypted, IsCompressed, CompressionFormat, UncompressedSize) - VALUES ($relativePath, $fileName, $extension, $folderPath, $sourcePackFilePath, $offset, $size, $isEncrypted, $isCompressed, $compressionFormat, $uncompressedSize)"; - - var pRelativePath = cmd.Parameters.Add("$relativePath", SqliteType.Text); - var pFileName = cmd.Parameters.Add("$fileName", SqliteType.Text); - var pExtension = cmd.Parameters.Add("$extension", SqliteType.Text); - var pFolderPath = cmd.Parameters.Add("$folderPath", SqliteType.Text); - var pSourcePackFilePath = cmd.Parameters.Add("$sourcePackFilePath", SqliteType.Text); - var pOffset = cmd.Parameters.Add("$offset", SqliteType.Integer); - var pSize = cmd.Parameters.Add("$size", SqliteType.Integer); - var pIsEncrypted = cmd.Parameters.Add("$isEncrypted", SqliteType.Integer); - var pIsCompressed = cmd.Parameters.Add("$isCompressed", SqliteType.Integer); - var pCompressionFormat = cmd.Parameters.Add("$compressionFormat", SqliteType.Integer); - var pUncompressedSize = cmd.Parameters.Add("$uncompressedSize", SqliteType.Integer); - - cmd.Prepare(); - - foreach (var (relativePath, packFile) in container.GetAllFiles()) - { - if (packFile.DataSource is not PackedFileSource source) - continue; - - var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); - var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); - - pRelativePath.Value = relativePath; - pFileName.Value = packFile.Name; - pExtension.Value = Path.GetExtension(relativePath).ToLower(); - pFolderPath.Value = folderPath; - pSourcePackFilePath.Value = source.Parent.FilePath; - pOffset.Value = source.Offset; - pSize.Value = source.Size; - pIsEncrypted.Value = source.IsEncrypted ? 1 : 0; - pIsCompressed.Value = source.IsCompressed ? 1 : 0; - pCompressionFormat.Value = (int)source.CompressionFormat; - pUncompressedSize.Value = (long)source.UncompressedSize; - - cmd.ExecuteNonQuery(); - } - } - - transaction.Commit(); - _logger.Here().Information($"Cache saved successfully for '{container.Name}'"); - } - finally - { - if (shouldDisposeConnection) - connection.Dispose(); } - } - - private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) - { - var relationalOptions = dbOptions.Extensions - .OfType() - .FirstOrDefault(); - - if (relationalOptions?.Connection is SqliteConnection sqliteConnection) - return (sqliteConnection, false); - - var connectionString = relationalOptions?.ConnectionString; - if (string.IsNullOrWhiteSpace(connectionString)) - throw new InvalidOperationException("Unable to resolve SQLite connection from DbContextOptions."); - return (new SqliteConnection(connectionString), true); - } - - public static CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) - { - if (!File.Exists(dbFilePath)) - { - _logger.Here().Information($"No cache file found at '{dbFilePath}'"); - return null; - } + if (foundFiles == 0) + throw new Exception("Trying to compute CachecFingerPrint, but no files found. This will result in a default ID which can cause problems"); - var dbOptions = CreateDbOptions(dbFilePath); - return LoadContainerFromCache(dbOptions, expectedFingerprint); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString())); + return Convert.ToHexString(hash); } - public static CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint) + public CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath) { - using var db = new CacheDbContext(dbOptions); - - try + using (var temp = new CachedPackFileContainer(container.Name, cacheFilePath)) { - db.Database.EnsureCreated(); + temp.Save(fingerprint, container); } - catch (Exception ex) - { - _logger.Here().Warning($"Failed to open cache database: {ex.Message}"); - return null; - } - - CacheInfoEntity? cacheInfo; - try - { - cacheInfo = db.CacheInfo.FirstOrDefault(); - } - catch (Exception ex) - { - _logger.Here().Warning($"Failed to read cache info: {ex.Message}"); - return null; - } - - if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) - { - _logger.Here().Information($"Cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); - return null; - } - - var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) - { - SystemFilePath = cacheInfo.SystemFilePath, - }; - if (!string.IsNullOrEmpty(cacheInfo.SourcePackFilePaths)) - { - foreach (var path in cacheInfo.SourcePackFilePaths.Split('|')) - container.SourcePackFilePaths.Add(path); - } + var loaded = CachedPackFileContainer.CreateFromFingerPrint(cacheFilePath, fingerprint); + if (loaded == null) + throw new Exception($"Failed to load from cache after saving. CacheFile: {cacheFilePath}"); - _logger.Here().Information($"Loaded container '{container.Name}' from cache"); - return container; + return loaded; } - public static CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) + public CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) { if (!File.Exists(cacheFilePath)) { @@ -253,7 +84,7 @@ private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSq try { _logger.Here().Information($"Attempting to load cache from: {cacheFilePath} with fingerprint: {fingerprint}"); - var result = LoadContainerFromCache(cacheFilePath, fingerprint); + var result = CachedPackFileContainer.CreateFromFingerPrint(cacheFilePath, fingerprint); if (result == null) _logger.Here().Information($"Cache load returned null (fingerprint/schema mismatch) for: {cacheFilePath}"); return result; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs index d253b27a6..42f6c78bd 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileSerializerWriter.cs @@ -1,4 +1,6 @@ -using System.Text; +using System.Diagnostics; +using System.Text; +using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; @@ -19,8 +21,16 @@ class PackFileWriteInformation(PackFile pf, string fullFileName, long sizePositi static class PackFileSerializerWriter { + private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileSerializerWriter)); + public static void SaveToByteArray(string outputFileName, PackFileContainer container, BinaryWriter writer, GameInformation currentGameInformation) { + var stopWatch = Stopwatch.StartNew(); + + var numFiles = container.GetFileCount(); + var packFileName = container.Name; + _logger.Here().Information("Saving packfile {PackFileName} v={PackFileVersion} with {NumFiles} files to {OutputFileName}. Current game = {Game}", packFileName, container.Header.Version, numFiles, outputFileName, currentGameInformation.DisplayName); + if (container.Header.HasEncryptedData || container.Header.HasEncryptedIndex) throw new InvalidOperationException("Saving encrypted packs is not supported."); @@ -35,11 +45,16 @@ public static void SaveToByteArray(string outputFileName, PackFileContainer cont // Write the core of the file var fileMetaDataTable = BuildMetaDataTable(sortedFiles, container, currentGameInformation); SerializeFileTable(fileMetaDataTable, container, writer); - SerializeFileBlob(outputFileName, fileMetaDataTable, container, writer); + SerializeFileBlob(outputFileName, fileMetaDataTable, container, writer); + + stopWatch.Stop(); + _logger.Here().Information("Saving packfile {PackFileName} completed in {ElapsedMilliseconds} ms", packFileName, stopWatch.ElapsedMilliseconds); } - public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) + static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) { + _logger.Here().Information("Starting write Header"); + var packFileTypeStr = PackFileVersionConverter.ToString(header.Version); // 4 foreach (var c in packFileTypeStr) writer.Write(c); @@ -82,6 +97,8 @@ public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWrit writer.Write(fileNameBytes); writer.Write((byte)0); } + + _logger.Here().Information("Finished writing"); } static long ComputeFileHeaderSpecificByte(PackFileContainer container) @@ -151,6 +168,8 @@ public static FileCompressionInfo DetermineFileCompression(PackFileVersion outpu public static List BuildMetaDataTable(IList> sortedFiles, PackFileContainer container, GameInformation currentGameInformation) { + _logger.Here().Information("Starting to build FileTable"); + var filesToWrite = new List(); foreach (var file in sortedFiles) { @@ -159,11 +178,15 @@ public static List BuildMetaDataTable(IList fileMetaData, PackFileContainer container, BinaryWriter writer) + static void SerializeFileTable(List fileMetaData, PackFileContainer container, BinaryWriter writer) { + _logger.Here().Information("Starting SerializeFileTable"); + // Write file table // FileStartPosition // TimeStamp @@ -193,10 +216,14 @@ public static void SerializeFileTable(List fileMetaDat writer.Write(fileNameBytes); writer.Write((byte)0); // Zero terminator } + + _logger.Here().Information("Finished SerializeFileTable"); } - public static void SerializeFileBlob(string outputFileName, List fileMetaDataTabel, PackFileContainer container, BinaryWriter writer) + static void SerializeFileBlob(string outputFileName, List fileMetaDataTabel, PackFileContainer container, BinaryWriter writer) { + _logger.Here().Information("Starting SerializeFileBlob"); + foreach (var fileMetaData in fileMetaDataTabel) { var packFile = fileMetaData.PackFile; @@ -247,6 +274,8 @@ public static void SerializeFileBlob(string outputFileName, List GetPackFilesFromManifest(string gameDataFolder, out b { var items = line.Split('\t'); if (items[0].Contains(".pack")) - output.Add(items[0].Trim()); + output.Add(Path.Combine(gameDataFolder, items[0].Trim())); } manifestFileFound = true; return output; @@ -21,8 +21,7 @@ public static List GetPackFilesFromManifest(string gameDataFolder, out b else { var files = Directory.GetFiles(gameDataFolder) - .Where(x => Path.GetExtension(x) == ".pack") - .Select(x => Path.GetFileName(x)) + .Where(x => Path.GetExtension(x).ToLower() == ".pack") .ToList(); manifestFileFound = false; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs similarity index 50% rename from Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs rename to Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs index 0c8d58475..b0daa31e9 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader2.cs @@ -10,30 +10,41 @@ using Shared.Core.Settings; namespace Shared.Core.PackFiles.Utility -{ - public interface IPackFileContainerLoader +{ + public enum PackFileContainerType { - IPackFileContainer? Load(string packFileSystemPath); - IPackFileContainer? LoadAllCaFiles(GameTypeEnum gameEnum); - IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath); + Cached, + Normal, } + public interface IPackFileContainerLoader + { + IPackFileContainer CreateFromPackFile(PackFileContainerType type, string packFilePath, bool loadAsReadOnly); + // IPackFileContainer CreateFromCollection(PackFileContainerType type, string packFileSystemPath, List fullPackFilePaths, string createdPackFileName, bool loadAsReadOnly, IDuplicateFileResolver duplicateFileResolver); + IPackFileContainer? CreateFromGameEnum(PackFileContainerType type, GameTypeEnum game); + + IPackFileContainer CreateFromSystemFolder(string folderPath); + } - public class PackFileContainerLoader : IPackFileContainerLoader + class PackFileContainerLoader : IPackFileContainerLoader { static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); private readonly ApplicationSettingsService _settingsService; private readonly IStandardDialogs _standardDialogs; private readonly LocalizationManager _localizationManager; + private readonly IPackFileContainerCacheHelper _packFileContainerCacheHelper; - public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager) + public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager, IPackFileContainerCacheHelper packFileContainerCacheHelper) { + // TODO : Handle context for logger + _settingsService = settingsService; _standardDialogs = standardDialogs; _localizationManager = localizationManager; + _packFileContainerCacheHelper = packFileContainerCacheHelper; } - public IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath) + public IPackFileContainer CreateFromSystemFolder(string packFileSystemPath) { if (Directory.Exists(packFileSystemPath) == false) { @@ -68,35 +79,14 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri AddFolderContentToPackFile(container, folder, rootPath); } - public IPackFileContainer? Load(string packFileSystemPath) - { - try - { - if (!File.Exists(packFileSystemPath)) - { - _logger.Here().Error($"Trying to load file {packFileSystemPath}, which can not be located.", "Error"); - System.Windows.MessageBox.Show($"Unable to locate pack file \"{packFileSystemPath}\""); - return null; - } - - using var fileStream = File.OpenRead(packFileSystemPath); - using var reader = new BinaryReader(fileStream, Encoding.ASCII); - - var packFileSize = new FileInfo(packFileSystemPath).Length; - var pack = PackFileSerializerLoader.Load(packFileSystemPath, packFileSize, reader, new CustomPackDuplicateFileResolver()); - PackFileLog.LogPackCompression(pack); - return pack; - } - catch (Exception e) - { - var errorMessage = $"Failed to load file {packFileSystemPath}. Error : {e.Message}"; - _logger.Here().Error(errorMessage); - throw new Exception(errorMessage, e); - } + public IPackFileContainer CreateFromPackFile(PackFileContainerType type, string packFilePath, bool loadAsReadOnly) + { + var packfileName = Path.GetFileNameWithoutExtension(packFilePath); + return CreateFromCollection(type, packFilePath, [packFilePath], packfileName, loadAsReadOnly, new CustomPackDuplicateFileResolver()); } - public IPackFileContainer? LoadAllCaFiles(GameTypeEnum gameEnum) + public IPackFileContainer? CreateFromGameEnum(PackFileContainerType type, GameTypeEnum gameEnum) { var game = GameInformationDatabase.GetGameById(gameEnum); var gamePathInfo = _settingsService.CurrentSettings.GameDirectories.FirstOrDefault(x => x.Game == game.Type); @@ -110,101 +100,76 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri } var gameDataFolder = gamePathInfo.Path; + var fullPackFilePaths = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); - try + // When loading ca pack packs, we want to use the CA resolver as its faster. + // If there is no manifest file, we need to use the duplicate resolver as it loads all file in the folder. + // There might be custom mods in there that does not follow the rules! + IDuplicateFileResolver packfileResolver = new CaPackDuplicateFileResolver(); + if (manifestFileFound == false) { - _logger.Here().Information($"Loading pack files for {gameName} located in {gameDataFolder}"); - var allCaPackFiles = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); + _logger.Here().Warning($"Loading pack files for {gameName}, which does not uses manifest.txt. If there are MODs in the game folder, this might cause issues!"); + packfileResolver = new CustomPackDuplicateFileResolver(); + } + + return CreateFromCollection(PackFileContainerType.Cached, gameDataFolder, fullPackFilePaths, $"All Game Packs - {gameName}", true, packfileResolver); + } + - var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(gameDataFolder, allCaPackFiles); - _logger.Here().Information($"Computed fingerprint for {gameName}: {fingerprint}"); + public IPackFileContainer CreateFromCollection(PackFileContainerType type, string packFileSystemPath, List fullPackFilePaths, string createdPackFileName, bool loadAsReadOnly, IDuplicateFileResolver duplicateFileResolver) + { + if(type == PackFileContainerType.Cached && loadAsReadOnly == false) + throw new InvalidOperationException($"Cannot load as writable if loading from cache. Caching is only supported for read-only containers. PackFile {createdPackFileName}"); - var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName, fingerprint); - _logger.Here().Information($"Cache file path for {gameName}: {cacheFilePath}"); + var fingerprint = _packFileContainerCacheHelper.ComputeFingerprint(fullPackFilePaths); + var cachePrefix = createdPackFileName; + var cacheFilePath = _packFileContainerCacheHelper.GetCacheFilePath(cachePrefix, fingerprint); - var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); + if (type == PackFileContainerType.Cached) + { + var cached = _packFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); if (cached != null) - { - _logger.Here().Information($"Cache hit for {gameName} - loaded {cached.GetFileCount()} files from: {cacheFilePath}"); return cached; - } - _logger.Here().Information($"Cache miss for {gameName} at: {cacheFilePath}"); + //var cacheInvalidReason = GetCacheInvalidReason(cacheFilePath, fingerprint); + //_logger.Here().Information($"Cache invalid reason for {gameName}: {cacheInvalidReason}"); + //var reasonMessage = string.Format(_localizationManager.Get("PackFileCache.InvalidReason." + cacheInvalidReason), gameName); + //var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); + //var cacheDescription = _localizationManager.Get("PackFileCache.Description"); + _standardDialogs.ShowDialogBox("Failed to load from cache - make better error later"); + } - var cacheInvalidReason = GetCacheInvalidReason(cacheFilePath, fingerprint); - _logger.Here().Information($"Cache invalid reason for {gameName}: {cacheInvalidReason}"); - var reasonMessage = string.Format(_localizationManager.Get("PackFileCache.InvalidReason." + cacheInvalidReason), gameName); - var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); - var cacheDescription = _localizationManager.Get("PackFileCache.Description"); - _standardDialogs.ShowDialogBox( - reasonMessage + "\n\n" + buildingMessage + "\n\n" + cacheDescription, - _localizationManager.Get("PackFileCache.InvalidReason.Title")); + using (_standardDialogs.ShowWaitCursor()) + { + var container = LoadPackFilesFromDisk(createdPackFileName, fullPackFilePaths, duplicateFileResolver); + container.Name = createdPackFileName; + container.IsCaPackFile = loadAsReadOnly; + container.SystemFilePath = packFileSystemPath; - PackFileContainer container; - using (_standardDialogs.ShowWaitCursor()) + if (type == PackFileContainerType.Cached) { - container = LoadAllCaFilesFromDisk(gameDataFolder, gameName, allCaPackFiles, manifestFileFound); - - try - { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); - _logger.Here().Information($"Saved CA pack cache for {gameName} to {cacheFilePath}"); - } - catch (Exception cacheEx) - { - _logger.Here().Warning($"Failed to save CA pack cache for {gameName}: {cacheEx.Message}"); - } + return _packFileContainerCacheHelper.SaveAndLoadCache(fingerprint, container, cacheFilePath); } return container; } - catch (Exception e) - { - _logger.Here().Error($"Trying to get all CA packs in {gameDataFolder}. Error : {e.ToString()}"); - return null; - } } - private static string GetCacheInvalidReason(string cacheFilePath, string fingerprint) - { - if (!File.Exists(cacheFilePath)) - return "NotFound"; - try - { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, fingerprint); - if (result == null) - return "DataChanged"; - } - catch (Exception cacheReadException) - { - _logger.Here().Warning($"Failed to inspect CA pack cache '{cacheFilePath}': {cacheReadException.Message}"); - return "Corrupted"; - } - return "DataChanged"; - } - private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string gameName, List allCaPackFiles, bool manifestFileFound) - { - // When loading ca pack packs, we want to use the CA resolver as its faster. - // If there is no manifest file, we need to use the duplicate resolver as it loads all file in the folder. - // There might be custom mods in there that does not follow the rules! - IDuplicateFileResolver packfileResolver = new CaPackDuplicateFileResolver(); - if (manifestFileFound == false) - { - _logger.Here().Warning($"Loading pack files for {gameName}, which does not uses manifest.txt. If there are MODs in the game folder, this might cause issues!"); - packfileResolver = new CustomPackDuplicateFileResolver(); - } + + + + private static PackFileContainer LoadPackFilesFromDisk(string createdPackFileName, List fullPackFilePaths, IDuplicateFileResolver packfileResolver) + { var packList = new ConcurrentBag(); var packsCompressionStats = new ConcurrentDictionary(); - Parallel.ForEach(allCaPackFiles, packFilePath => + Parallel.ForEach(fullPackFilePaths, packFilePath => { - var path = gameDataFolder + "\\" + packFilePath; + var path = packFilePath; if (File.Exists(path)) { using var fileStream = File.OpenRead(path); @@ -227,15 +192,19 @@ private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string g } } else - _logger.Here().Warning($"{gameName} pack file '{path}' not found, loading skipped"); + _logger.Here().Warning($"{createdPackFileName} pack file '{path}' not found, loading skipped"); } ); PackFileLog.LogPacksCompression(packsCompressionStats); - var caPackFileContainer = new PackFileContainer($"All Game Packs - {gameName}"); - caPackFileContainer.IsCaPackFile = true; - caPackFileContainer.SystemFilePath = gameDataFolder; + // If there is only one packfile - we dont need to sort. Just return it. + // Be aware, that when we create a new PackFileContainer in the case of multiple packfiles, we will lose the original header information of the first packfile. + // This is because we need to create a new header for the new container. This should not be an issue, but its something to be aware of. + if (packList.Count == 1) + return packList.First(); + + var mergedPackFile = new PackFileContainer(createdPackFileName); var packFilesOrderedByGroup = packList.GroupBy(x => x.Header.LoadOrder).OrderBy(x => x.Key); foreach (var group in packFilesOrderedByGroup) @@ -244,12 +213,12 @@ private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string g foreach (var packfile in packFilesOrderedByName) { if (string.IsNullOrWhiteSpace(packfile.SystemFilePath) == false) - caPackFileContainer.SourcePackFilePaths.Add(packfile.SystemFilePath); - caPackFileContainer.MergePackFileContainer(packfile); + mergedPackFile.SourcePackFilePaths.Add(packfile.SystemFilePath); + mergedPackFile.MergePackFileContainer(packfile); } } - return caPackFileContainer; + return mergedPackFile; } } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs index 4b6c0c5a3..38fddfa7d 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileServiceUtility.cs @@ -2,34 +2,8 @@ namespace Shared.Core.PackFiles.Utility { - public sealed record DirectoryFileEntry(string FileName, PackFile File); - - public sealed record DirectoryEntriesSplitResult( - string DirectoryPath, - IReadOnlyList SubFolders, - IReadOnlyList Files); - public static class PackFileServiceUtility { - public static DirectoryEntriesSplitResult SplitDirectoryEntries(IPackFileContainer container, string directoryPath) - { - var packFileContainerInternal = PackFileService.CastContainer(container); - - var files = packFileContainerInternal.GetDirectoryContent(directoryPath) - .Select(entry => new DirectoryFileEntry(entry.File.Name, entry.File)) - .OrderBy(x => x.FileName, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - - var subFolders = packFileContainerInternal.GetSubDirectories(directoryPath) - .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) - .ToList(); - - return new DirectoryEntriesSplitResult( - directoryPath, - subFolders, - files); - } - public static List GetAllAnimPacks(IPackFileService pfs) { var animPacks = FindAllWithExtention(pfs, @".animpack"); diff --git a/Shared/SharedCore/Shared.Core/Shared.Core.csproj b/Shared/SharedCore/Shared.Core/Shared.Core.csproj index 13f699d45..5b41f4569 100644 --- a/Shared/SharedCore/Shared.Core/Shared.Core.csproj +++ b/Shared/SharedCore/Shared.Core/Shared.Core.csproj @@ -5,6 +5,7 @@ enable enable true + preview diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs index 69b345954..5c00a013a 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFiles.cs @@ -14,7 +14,7 @@ public PackFileContainerTests_GetAllFiles(Type containerType) : base(containerTy public void GetAllFiles_ReturnsExpectedCountAndKeys() { var files = _container.GetAllFiles(); - Assert.That(files.Count, Is.EqualTo(15)); + Assert.That(files.Count, Is.EqualTo(18)); Assert.That(files.ContainsKey("folder\\file.txt"), Is.True); Assert.That(files.ContainsKey("scripts\\campaign_script.lua"), Is.True); } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs new file mode 100644 index 000000000..2a6b0ca61 --- /dev/null +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetAllFilesByFolder.cs @@ -0,0 +1,131 @@ +using Shared.Core.PackFiles.Models.Containers; + +namespace Shared.CoreTest.PackFiles.Models.Containers +{ + [TestFixture(typeof(CachedPackFileContainer))] + [TestFixture(typeof(PackFileContainer))] + internal class PackFileContainerTests_GetAllFilesByFolder : PackFileContainerTests_TestBase + { + public PackFileContainerTests_GetAllFilesByFolder(Type containerType) : base(containerType) { } + + [Test] + public void GetAllFilesByFolder_ReturnsExpectedFolderCount() + { + var result = _container.GetAllFilesByFolder(); + + // Folders: folder, other, audio, (root), models, models\textures, models\textures\specular, scripts, folder_a, folder_b, compressed + Assert.That(result.Count, Is.GreaterThanOrEqualTo(10)); + } + + [Test] + public void GetAllFilesByFolder_KeysAreSorted() + { + var result = _container.GetAllFilesByFolder(); + var keys = result.Keys.ToList(); + + for (var i = 1; i < keys.Count; i++) + { + Assert.That(string.Compare(keys[i - 1], keys[i], StringComparison.InvariantCultureIgnoreCase), Is.LessThan(0), + $"Keys not sorted: '{keys[i - 1]}' should come before '{keys[i]}'"); + } + } + + [Test] + public void GetAllFilesByFolder_ContainsExpectedFolders() + { + var result = _container.GetAllFilesByFolder(); + + Assert.That(result.ContainsKey("folder"), Is.True); + Assert.That(result.ContainsKey("other"), Is.True); + Assert.That(result.ContainsKey("audio"), Is.True); + Assert.That(result.ContainsKey("models"), Is.True); + Assert.That(result.ContainsKey(@"models\textures"), Is.True); + Assert.That(result.ContainsKey(@"models\textures\specular"), Is.True); + Assert.That(result.ContainsKey("scripts"), Is.True); + Assert.That(result.ContainsKey("folder_a"), Is.True); + Assert.That(result.ContainsKey("folder_b"), Is.True); + Assert.That(result.ContainsKey("compressed"), Is.True); + } + + [Test] + public void GetAllFilesByFolder_FolderContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var audioFiles = result["audio"]; + Assert.That(audioFiles, Does.Contain("sound.wem")); + Assert.That(audioFiles, Does.Contain("music.wem")); + Assert.That(audioFiles, Does.Contain("battle_sound.wem")); + Assert.That(audioFiles, Does.Contain("voice.wem_temp")); + Assert.That(audioFiles, Does.Contain("voice.wem.{sdf}")); + Assert.That(audioFiles, Does.Contain("voice.txt")); + Assert.That(audioFiles.Count, Is.EqualTo(6)); + } + + [Test] + public void GetAllFilesByFolder_ModelsFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var modelsFiles = result["models"]; + Assert.That(modelsFiles, Does.Contain("unit.model")); + Assert.That(modelsFiles, Does.Contain("vehicle.model")); + Assert.That(modelsFiles.Count, Is.EqualTo(2)); + } + + [Test] + public void GetAllFilesByFolder_NestedFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var textureFiles = result[@"models\textures"]; + Assert.That(textureFiles, Does.Contain("diffuse.dds")); + Assert.That(textureFiles, Does.Contain("normal.dds")); + Assert.That(textureFiles.Count, Is.EqualTo(2)); + } + + [Test] + public void GetAllFilesByFolder_DeeplyNestedFolder_ContainsCorrectFiles() + { + var result = _container.GetAllFilesByFolder(); + + var specularFiles = result[@"models\textures\specular"]; + Assert.That(specularFiles, Does.Contain("gloss.dds")); + Assert.That(specularFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_SingleFileFolder_ContainsCorrectFile() + { + var result = _container.GetAllFilesByFolder(); + + var folderFiles = result["folder"]; + Assert.That(folderFiles, Does.Contain("file.txt")); + Assert.That(folderFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_FilesAreNotDuplicatedAcrossFolders() + { + var result = _container.GetAllFilesByFolder(); + + // shared.txt exists in both folder_a and folder_b + var folderAFiles = result["folder_a"]; + var folderBFiles = result["folder_b"]; + Assert.That(folderAFiles, Does.Contain("shared.txt")); + Assert.That(folderBFiles, Does.Contain("shared.txt")); + Assert.That(folderAFiles.Count, Is.EqualTo(1)); + Assert.That(folderBFiles.Count, Is.EqualTo(1)); + } + + [Test] + public void GetAllFilesByFolder_TotalFileCountMatchesGetAllFiles() + { + var result = _container.GetAllFilesByFolder(); + var totalFiles = result.Values.Sum(files => files.Count); + var allFiles = _container.GetAllFiles(); + + Assert.That(totalFiles, Is.EqualTo(allFiles.Count)); + } + } +} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs index 8a2ccb56a..b0e09de0c 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetDirectoryContent.cs @@ -1,5 +1,4 @@ using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Utility; namespace Shared.CoreTest.PackFiles.Models.Containers { @@ -35,11 +34,10 @@ public void GetDirectoryContent_UnknownFolder_ReturnsEmpty() } [Test] - public void GetDirectoryContent_ComposesWithUtility() + public void GetDirectoryContent_ModelsFolder_ReturnsExpectedFiles() { - var split = PackFileServiceUtility.SplitDirectoryEntries(_container, "models"); - Assert.That(split.Files.Any(x => x.FileName == "unit.model"), Is.True); - Assert.That(split.SubFolders, Does.Contain("textures")); + var files = _container.GetDirectoryContent("models"); + Assert.That(files.Any(x => x.File.Name == "unit.model"), Is.True); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs index 8b5e2355b..2c9714d35 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetFileCount.cs @@ -11,7 +11,7 @@ public PackFileContainerTests_GetFileCount(Type containerType) : base(containerT [Test] public void GetFileCount_ReturnsMasterDatasetCount() { - Assert.That(_container.GetFileCount(), Is.EqualTo(15)); + Assert.That(_container.GetFileCount(), Is.EqualTo(18)); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs deleted file mode 100644 index 2aa8aa55c..000000000 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_GetSubDirectories.cs +++ /dev/null @@ -1,35 +0,0 @@ -using Shared.Core.PackFiles.Models.Containers; - -namespace Shared.CoreTest.PackFiles.Models.Containers -{ - [TestFixture(typeof(CachedPackFileContainer))] - [TestFixture(typeof(PackFileContainer))] - internal class PackFileContainerTests_GetSubDirectories : PackFileContainerTests_TestBase - { - public PackFileContainerTests_GetSubDirectories(Type containerType) : base(containerType) { } - - [Test] - public void GetSubDirectories_Root_ReturnsImmediateSortedFolders() - { - var folders = _container.GetSubDirectories(""); - Assert.That(folders, Does.Contain("audio")); - Assert.That(folders, Does.Contain("models")); - - var sorted = folders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(); - Assert.That(folders, Is.EqualTo(sorted)); - } - - [Test] - public void GetSubDirectories_NestedFolder_ReturnsOnlyChildren() - { - var folders = _container.GetSubDirectories(@"models\textures"); - Assert.That(folders, Is.EqualTo(new[] { "specular" })); - } - - [Test] - public void GetSubDirectories_UnknownFolder_ReturnsEmpty() - { - Assert.That(_container.GetSubDirectories("missing\\folder"), Is.Empty); - } - } -} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs index b8278c865..ae091210b 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_SearchFiles.cs @@ -1,9 +1,4 @@ -using Shared.Core.PackFiles.Models; -using Shared.Core.PackFiles.Models.Containers; -using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization.CacheDatabase; -using Shared.Core.PackFiles.Utility; -using Microsoft.Data.Sqlite; +using Shared.Core.PackFiles.Models.Containers; namespace Shared.CoreTest.PackFiles.Models.Containers { @@ -17,7 +12,7 @@ public PackFileContainerTests_SearchFiles(Type containerType) : base(containerTy public void SearchFiles_NullFilters_ReturnsAll() { var results = _container.SearchFiles(null, null); - Assert.That(results.Count, Is.EqualTo(15)); + Assert.That(results.Count, Is.EqualTo(18)); } [Test] @@ -40,65 +35,19 @@ public void SearchFiles_ResultsArePathSorted() public void SearchFiles_EmptyExtensionList_TreatedAsNoFilter() { var results = _container.SearchFiles(null, []); - Assert.That(results.Count, Is.EqualTo(15)); + Assert.That(results.Count, Is.EqualTo(18)); } [Test] public void SearchFiles_ExtensionFilter_MatchesFilenameSubstrings_ForLegacyWemVariants() { - var sourceContainer = new PackFileContainer("SearchFilesVariants") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game\data" - }; - sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + var results = _container.SearchFiles(null, [".wem"]); + var paths = results.Select(x => x.Path).ToList(); - var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; - sourceContainer.AddOrUpdateFile("audio\\voice.wem", new PackFile("voice.wem", new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.wem_temp", new PackFile("voice.wem_temp", new PackedFileSource(parent, 10, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.wem.{sdf}", new PackFile("voice.wem.{sdf}", new PackedFileSource(parent, 20, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\voice.txt", new PackFile("voice.txt", new PackedFileSource(parent, 30, 10, false, false, CompressionFormat.None, 0))); - - SqliteConnection? keepAliveConnection = null; - IPackFileContainer container; - try - { - if (IsCachedContainer) - { - var dbName = "SearchFilesVariants_" + Guid.NewGuid().ToString("N"); - var connectionString = new SqliteConnectionStringBuilder - { - DataSource = dbName, - Mode = SqliteOpenMode.Memory, - Cache = SqliteCacheMode.Shared - }.ToString(); - - // Keep one connection alive so the shared in-memory DB remains available - // while SaveCache and LoadContainerFromCache use separate EF/raw connections. - keepAliveConnection = new SqliteConnection(connectionString); - keepAliveConnection.Open(); - - var dbOptions = PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); - PackFileContainerCacheHelper.SaveCache("variants_fp", sourceContainer, dbOptions); - container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "variants_fp")!; - } - else - { - container = sourceContainer; - } - - var results = container.SearchFiles(null, [".wem"]); - var paths = results.Select(x => x.Path).ToList(); - - Assert.That(paths, Does.Contain(@"audio\voice.wem")); - Assert.That(paths, Does.Contain(@"audio\voice.wem_temp")); - Assert.That(paths, Does.Contain(@"audio\voice.wem.{sdf}")); - Assert.That(paths, Does.Not.Contain(@"audio\voice.txt")); - } - finally - { - keepAliveConnection?.Dispose(); - } + Assert.That(paths, Does.Contain(@"audio\sound.wem")); + Assert.That(paths, Does.Contain(@"audio\voice.wem_temp")); + Assert.That(paths, Does.Contain(@"audio\voice.wem.{sdf}")); + Assert.That(paths, Does.Not.Contain(@"audio\voice.txt")); } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs index c85d3769d..29ab66839 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/PackFileContainerTests_TestBase.cs @@ -1,8 +1,6 @@ -using Microsoft.Data.Sqlite; -using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; namespace Shared.CoreTest.PackFiles.Models.Containers @@ -11,10 +9,31 @@ internal abstract class PackFileContainerTests_TestBase { private readonly bool _useCachedContainer; - private SqliteConnection? _cacheKeepAliveConnection; protected IPackFileContainerInternal _container = null!; protected bool IsCachedContainer => _useCachedContainer; + protected static readonly (string RelativePath, string FileName, long Offset, long Size, bool IsEncrypted, bool IsCompressed, CompressionFormat CompressionFormat, uint UncompressedSize)[] TestFiles = + [ + ("folder\\file.txt", "file.txt", 100, 200, false, false, CompressionFormat.None, 0), + ("other\\data.bin", "data.bin", 300, 400, false, true, CompressionFormat.Lz4, 800), + ("audio\\sound.wem", "sound.wem", 700, 500, false, false, CompressionFormat.None, 0), + ("root_file.txt", "root_file.txt", 0, 10, false, false, CompressionFormat.None, 0), + ("models\\unit.model", "unit.model", 10, 20, false, false, CompressionFormat.None, 0), + ("models\\vehicle.model", "vehicle.model", 30, 40, false, false, CompressionFormat.None, 0), + ("models\\textures\\diffuse.dds", "diffuse.dds", 70, 50, false, false, CompressionFormat.None, 0), + ("models\\textures\\normal.dds", "normal.dds", 120, 60, false, false, CompressionFormat.None, 0), + ("models\\textures\\specular\\gloss.dds", "gloss.dds", 180, 30, false, false, CompressionFormat.None, 0), + ("audio\\music.wem", "music.wem", 210, 100, false, false, CompressionFormat.None, 0), + ("audio\\battle_sound.wem", "battle_sound.wem", 400, 300, false, false, CompressionFormat.None, 0), + ("scripts\\campaign_script.lua", "campaign_script.lua", 850, 80, false, false, CompressionFormat.None, 0), + ("folder_a\\shared.txt", "shared.txt", 900, 10, false, false, CompressionFormat.None, 0), + ("folder_b\\shared.txt", "shared.txt", 910, 20, false, false, CompressionFormat.None, 0), + ("compressed\\data.bin", "data.bin", 1000, 500, true, true, CompressionFormat.Lz4, 2000), + ("audio\\voice.wem_temp", "voice.wem_temp", 1500, 10, false, false, CompressionFormat.None, 0), + ("audio\\voice.wem.{sdf}", "voice.wem.{sdf}", 1510, 10, false, false, CompressionFormat.None, 0), + ("audio\\voice.txt", "voice.txt", 1520, 10, false, false, CompressionFormat.None, 0), + ]; + protected PackFileContainerTests_TestBase(Type containerType) { _useCachedContainer = containerType == typeof(CachedPackFileContainer); @@ -23,64 +42,23 @@ protected PackFileContainerTests_TestBase(Type containerType) [SetUp] public void Setup() { - var sourceContainer = new PackFileContainer("TestCache") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game\data" - }; - sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); - - var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; - sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", - new PackedFileSource(parent, 100, 200, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", - new PackedFileSource(parent, 300, 400, false, true, CompressionFormat.Lz4, 800))); - sourceContainer.AddOrUpdateFile("audio\\sound.wem", new PackFile("sound.wem", - new PackedFileSource(parent, 700, 500, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt", - new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\unit.model", new PackFile("unit.model", - new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\vehicle.model", new PackFile("vehicle.model", - new PackedFileSource(parent, 30, 40, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\diffuse.dds", new PackFile("diffuse.dds", - new PackedFileSource(parent, 70, 50, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\normal.dds", new PackFile("normal.dds", - new PackedFileSource(parent, 120, 60, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("models\\textures\\specular\\gloss.dds", new PackFile("gloss.dds", - new PackedFileSource(parent, 180, 30, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\music.wem", new PackFile("music.wem", - new PackedFileSource(parent, 210, 100, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("audio\\battle_sound.wem", new PackFile("battle_sound.wem", - new PackedFileSource(parent, 400, 300, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("scripts\\campaign_script.lua", new PackFile("campaign_script.lua", - new PackedFileSource(parent, 850, 80, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_a\\shared.txt", new PackFile("shared.txt", - new PackedFileSource(parent, 900, 10, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("folder_b\\shared.txt", new PackFile("shared.txt", - new PackedFileSource(parent, 910, 20, false, false, CompressionFormat.None, 0))); - sourceContainer.AddOrUpdateFile("compressed\\data.bin", new PackFile("data.bin", - new PackedFileSource(parent, 1000, 500, true, true, CompressionFormat.Lz4, 2000))); - if (_useCachedContainer) { - var dbName = "CachedContainerTests_" + Guid.NewGuid().ToString("N"); - var connectionString = new SqliteConnectionStringBuilder - { - DataSource = dbName, - Mode = SqliteOpenMode.Memory, - Cache = SqliteCacheMode.Shared - }.ToString(); - - _cacheKeepAliveConnection = new SqliteConnection(connectionString); - _cacheKeepAliveConnection.Open(); - - var dbOptions = PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); - PackFileContainerCacheHelper.SaveCache("test_fp", sourceContainer, dbOptions); - _container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp")!; + _container = CachedPackFileContainer.CreateFromFileList("TestCache", TestFiles, useInMemoryDb: true, systemFilePath: @"c:\game\data", sourcePackFilePath: @"c:\game\data\pack1.pack"); } else { + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; + var sourceContainer = new PackFileContainer("TestCache") + { + IsCaPackFile = true, + SystemFilePath = @"c:\game\data" + }; + sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + + foreach (var file in TestFiles) + sourceContainer.AddOrUpdateFile(file.RelativePath, new PackFile(file.FileName, new PackedFileSource(parent, file.Offset, file.Size, file.IsEncrypted, file.IsCompressed, file.CompressionFormat, file.UncompressedSize))); + _container = sourceContainer; } } @@ -88,8 +66,7 @@ public void Setup() [TearDown] public void TearDown() { - _cacheKeepAliveConnection?.Dispose(); - _cacheKeepAliveConnection = null; + (_container as IDisposable)?.Dispose(); } protected void IgnoreIfNotCached(string scenario) diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index a8503b95c..9947d4eb4 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -1,4 +1,4 @@ -using Microsoft.Data.Sqlite; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; @@ -47,12 +47,22 @@ private DbContextOptions CreateTestDbOptions() keepAliveConnection.Open(); _inMemoryKeepAliveConnections.Add(keepAliveConnection); - return PackFileContainerCacheHelper.CreateDbOptionsFromConnectionString(connectionString); + return new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; } private DbContextOptions CreateFileDbOptions() { - return PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + return new DbContextOptionsBuilder() + .UseSqlite($"Data Source={_dbFilePath};Pooling=False") + .Options; + } + + private static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) + { + using var cached = new CachedPackFileContainer(container.Name, dbOptions); + cached.Save(fingerprint, container); } [Test] @@ -79,8 +89,8 @@ public void RoundTrip_PreservesMetadata() // Act var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fingerprint123", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fingerprint123"); + SaveCache("fingerprint123", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fingerprint123"); // Assert Assert.That(loaded, Is.Not.Null); @@ -109,10 +119,10 @@ public void LoadCache_ReturnsCorrectFileData() new PackedFileSource(parent, 2048, 4096, false, true, CompressionFormat.Lz4, 8192))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + SaveCache("fp", container, dbOptions); // Act - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); // Assert Assert.That(loaded, Is.Not.Null); @@ -153,8 +163,8 @@ public void LoadCache_PreservesSourcePackFilePath() new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); var sourceA = (PackedFileSource)loaded!.FindFile("a.txt")!.DataSource; var sourceB = (PackedFileSource)loaded!.FindFile("b.txt")!.DataSource; @@ -165,7 +175,7 @@ public void LoadCache_PreservesSourcePackFilePath() [Test] public void LoadCache_ReturnsNullForMissingFile() { - var result = PackFileContainerCacheHelper.LoadContainerFromCache( + var result = CachedPackFileContainer.CreateFromFingerPrint( Path.Combine(_tempDir, "nonexistent.db"), "fp"); Assert.That(result, Is.Null); } @@ -177,11 +187,10 @@ public void LoadCache_ReturnsNullForWrongFingerprint() { SystemFilePath = @"c:\game" }; - var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("correct_fp", container, dbOptions); + SaveCache("correct_fp", container, dbOptions); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "wrong_fp"); + var result = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "wrong_fp"); Assert.That(result, Is.Null); } @@ -192,10 +201,9 @@ public void ComputeFingerprint_DeterministicForSameInputs() Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); File.WriteAllText(Path.Combine(packDir, "b.pack"), "data_b"); - var packFiles = new List { "a.pack", "b.pack" }; - - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var packFiles = new List { Path.Combine(packDir, "a.pack"), Path.Combine(packDir, "b.pack") }; + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp1, Is.EqualTo(fp2)); } @@ -206,13 +214,13 @@ public void ComputeFingerprint_ChangesWhenFileChanges() var packDir = Path.Combine(_tempDir, "gamedata2"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); - var packFiles = new List { "a.pack" }; - - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + + var packFiles = new List { Path.Combine(packDir,"a.pack") }; + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a_modified_longer"); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp1, Is.Not.EqualTo(fp2)); } @@ -237,8 +245,8 @@ public void RoundTrip_FullCycle() // Act: save ? load var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("test_fp", container, dbOptions); - var restored = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp"); + SaveCache("test_fp", container, dbOptions); + var restored = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "test_fp"); // Assert Assert.That(restored, Is.Not.Null); @@ -271,9 +279,9 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() new PackedFileSource(parent, 0, 100, false, false, CompressionFormat.None, 0))); var dbOptions = CreateFileDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp_try", container, dbOptions); + SaveCache("fp_try", container, dbOptions); - var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp_try"); + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache(_dbFilePath, "fp_try"); Assert.That(result, Is.Not.Null); Assert.That(result.Name, Is.EqualTo("TryLoad Test")); } @@ -281,7 +289,7 @@ public void TryLoadFromCache_ReturnsContainerWhenValid() [Test] public void TryLoadFromCache_ReturnsNullForMissingFile() { - var result = PackFileContainerCacheHelper.TryLoadFromCache( + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache( Path.Combine(_tempDir, "does_not_exist.db"), "fp"); Assert.That(result, Is.Null); } @@ -290,7 +298,7 @@ public void TryLoadFromCache_ReturnsNullForMissingFile() public void TryLoadFromCache_ReturnsNullForCorruptFile() { File.WriteAllBytes(_dbFilePath, [0xFF, 0xFE, 0x00, 0x01]); - var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp"); + var result = (new PackFileContainerCacheHelper()).TryLoadFromCache(_dbFilePath, "fp"); Assert.That(result, Is.Null); } @@ -305,10 +313,9 @@ public void SaveCache_PreservesEncryptedFlag() var parent = new PackedFileSourceParent { FilePath = @"c:\game\encrypted.pack" }; container.AddOrUpdateFile("secret\\data.bin", new PackFile("data.bin", new PackedFileSource(parent, 0, 500, isEncrypted: true, isCompressed: false, CompressionFormat.None, 0))); - var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); var source = (PackedFileSource)loaded!.FindFile("secret\\data.bin")!.DataSource; Assert.That(source.IsEncrypted, Is.True); @@ -321,10 +328,9 @@ public void SaveCache_EmptyContainer_RoundTrips() { SystemFilePath = @"c:\game\empty" }; - var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp_empty", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp_empty"); + SaveCache("fp_empty", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp_empty"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.Name, Is.EqualTo("Empty Pack")); @@ -336,25 +342,24 @@ public void SaveCache_OverwritesExistingCache() { var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; var dbOptions = CreateFileDbOptions(); - // Save first version var container1 = new PackFileContainer("Version1") { SystemFilePath = @"c:\game" }; container1.AddOrUpdateFile("old.txt", new PackFile("old.txt", new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - PackFileContainerCacheHelper.SaveCache("fp1", container1, dbOptions); + SaveCache("fp1", container1, dbOptions); // Save second version (same db path) var container2 = new PackFileContainer("Version2") { SystemFilePath = @"c:\game" }; container2.AddOrUpdateFile("new.txt", new PackFile("new.txt", new PackedFileSource(parent, 0, 20, false, false, CompressionFormat.None, 0))); - PackFileContainerCacheHelper.SaveCache("fp2", container2, dbOptions); + SaveCache("fp2", container2, dbOptions); // Old fingerprint should fail - var oldResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp1"); + var oldResult = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp1"); Assert.That(oldResult, Is.Null); // New fingerprint should work - var newResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp2"); + var newResult = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp2"); Assert.That(newResult, Is.Not.Null); Assert.That(newResult.Name, Is.EqualTo("Version2")); Assert.That(newResult.GetFileCount(), Is.EqualTo(1)); @@ -378,18 +383,17 @@ public void SaveCache_StoresFolderPathCorrectly() new PackedFileSource(parent, 20, 20, false, false, CompressionFormat.None, 0))); var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + SaveCache("fp", container, dbOptions); // Verify via the CachedPackFileContainer's GetDirectoryContent - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); - var rootContent = PackFileServiceUtility.SplitDirectoryEntries(loaded, ""); - Assert.That(rootContent.Files.Any(f => f.FileName == "root_file.txt"), Is.True); - Assert.That(rootContent.SubFolders, Does.Contain("a")); + var rootFiles = loaded.GetDirectoryContent(""); + Assert.That(rootFiles.Any(f => f.File.Name == "root_file.txt"), Is.True); - var deepContent = PackFileServiceUtility.SplitDirectoryEntries(loaded, "a\\b\\c"); - Assert.That(deepContent.Files.Any(f => f.FileName == "file.txt"), Is.True); + var deepFiles = loaded.GetDirectoryContent("a\\b\\c"); + Assert.That(deepFiles.Any(f => f.File.Name == "file.txt"), Is.True); } [Test] @@ -405,10 +409,9 @@ public void SaveCache_SkipsNonPackedFileSources() new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); container.AddOrUpdateFile("memory.txt", new PackFile("memory.txt", new MemorySource([1, 2, 3]))); - var dbOptions = CreateTestDbOptions(); - PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + SaveCache("fp", container, dbOptions); + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, "fp"); Assert.That(loaded, Is.Not.Null); Assert.That(loaded.GetFileCount(), Is.EqualTo(1)); @@ -417,19 +420,19 @@ public void SaveCache_SkipsNonPackedFileSources() } [Test] + public void ComputeFingerprint_IgnoresMissingPackFiles() { var packDir = Path.Combine(_tempDir, "partial"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "exists.pack"), "data"); - - var packFiles = new List { "exists.pack", "missing.pack" }; - var fp = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + var packFiles = new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }; + var fp = (new PackFileContainerCacheHelper()).ComputeFingerprint(packFiles); Assert.That(fp, Is.Not.Null.And.Not.Empty); // Same result regardless of missing file in list - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, new List { "exists.pack", "missing.pack" }); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "exists.pack"), Path.Combine(packDir, "missing.pack") }); Assert.That(fp, Is.EqualTo(fp2)); } @@ -441,13 +444,9 @@ public void ComputeFingerprint_OrderIndependent() File.WriteAllText(Path.Combine(packDir, "alpha.pack"), "aaa"); File.WriteAllText(Path.Combine(packDir, "beta.pack"), "bbb"); File.WriteAllText(Path.Combine(packDir, "gamma.pack"), "ccc"); - - var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "gamma.pack", "alpha.pack", "beta.pack" }); - var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "alpha.pack", "beta.pack", "gamma.pack" }); - var fp3 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, - new List { "beta.pack", "gamma.pack", "alpha.pack" }); + var fp1 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack") }); + var fp2 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "alpha.pack"), Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack") }); + var fp3 = (new PackFileContainerCacheHelper()).ComputeFingerprint(new List { Path.Combine(packDir, "beta.pack"), Path.Combine(packDir, "gamma.pack"), Path.Combine(packDir, "alpha.pack") }); Assert.That(fp1, Is.EqualTo(fp2)); Assert.That(fp2, Is.EqualTo(fp3)); @@ -456,7 +455,7 @@ public void ComputeFingerprint_OrderIndependent() [Test] public void GetCacheFilePath_SanitizesInvalidChars() { - var path = PackFileContainerCacheHelper.GetCacheFilePath(@"c:\game", "Game:Name/WithChars", "abc123"); + var path = (new PackFileContainerCacheHelper()).GetCacheFilePath("Game:Name/WithChars", "abc123"); var fileName = Path.GetFileName(path); Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs new file mode 100644 index 000000000..7cb71feaf --- /dev/null +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/InMemoryPackFileContainerCacheHelper.cs @@ -0,0 +1,111 @@ +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles.Models.Containers; +using Shared.Core.PackFiles.Serialization.CacheDatabase; + +namespace Shared.CoreTest.PackFiles.Utility +{ + /// + /// An in-memory implementation of IPackFileContainerCacheHelper that avoids all filesystem I/O. + /// Uses shared in-memory SQLite databases keyed by a logical cache file path. + /// + internal class InMemoryPackFileContainerCacheHelper : IPackFileContainerCacheHelper + { + private readonly PackFileContainerCacheHelper _fingerprintHelper = new(); + + // Keeps shared in-memory SQLite connections alive (shared cache requires at least one open connection) + private readonly Dictionary _keepAliveConnections = new(StringComparer.OrdinalIgnoreCase); + + // Tracks which logical cache paths have been "saved" (i.e. exist) + private readonly HashSet _existingCaches = new(StringComparer.OrdinalIgnoreCase); + + // Tracks which caches have been "corrupted" (for simulating corruption scenarios) + private readonly HashSet _corruptedCaches = new(StringComparer.OrdinalIgnoreCase); + + public string ComputeFingerprint(List packFileNames) + => _fingerprintHelper.ComputeFingerprint(packFileNames); + + public string GetCacheFilePath(string gameName, string cacheId) + { + // Return a logical key, not a real filesystem path + return $"inmemory://{gameName}_{cacheId}"; + } + + public CachedPackFileContainer SaveAndLoadCache(string fingerprint, PackFileContainer container, string cacheFilePath) + { + var dbOptions = GetOrCreateDbOptions(cacheFilePath); + using (var temp = new CachedPackFileContainer(container.Name, dbOptions)) + { + temp.Save(fingerprint, container); + } + + _existingCaches.Add(cacheFilePath); + _corruptedCaches.Remove(cacheFilePath); + + var loaded = CachedPackFileContainer.CreateFromFingerPrint(dbOptions, fingerprint); + if (loaded == null) + throw new Exception($"Failed to load from cache after saving. CacheFile: {cacheFilePath}"); + + return loaded; + } + + public CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) + { + if (!_existingCaches.Contains(cacheFilePath)) + return null; + + if (_corruptedCaches.Contains(cacheFilePath)) + return null; + + try + { + var dbOptions = GetOrCreateDbOptions(cacheFilePath); + return CachedPackFileContainer.CreateFromFingerPrint(dbOptions, fingerprint); + } + catch + { + return null; + } + } + + /// + /// Simulates cache corruption for a given logical path. + /// + public void CorruptCache(string cacheFilePath) + { + _corruptedCaches.Add(cacheFilePath); + } + + public void Dispose() + { + foreach (var conn in _keepAliveConnections.Values) + conn.Dispose(); + + _keepAliveConnections.Clear(); + _existingCaches.Clear(); + _corruptedCaches.Clear(); + } + + private DbContextOptions GetOrCreateDbOptions(string logicalPath) + { + if (!_keepAliveConnections.TryGetValue(logicalPath, out var connection)) + { + var dbName = "InMemCache_" + Guid.NewGuid().ToString("N"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbName, + Mode = SqliteOpenMode.Memory, + Cache = SqliteCacheMode.Shared + }.ToString(); + + connection = new SqliteConnection(connectionString); + connection.Open(); + _keepAliveConnections[logicalPath] = connection; + } + + return new DbContextOptionsBuilder() + .UseSqlite(connection) + .Options; + } + } +} diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs index 31a1b0c3d..a0c86d2b5 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -14,12 +14,11 @@ internal class PackFileContainerLoaderTests private Mock _waitCursor; private LocalizationManager _localizationManager; private ApplicationSettingsService _settingsService; + private InMemoryPackFileContainerCacheHelper _cacheHelper; [SetUp] public void Setup() { - DirectoryHelper.EnsureCreated(); - _tempGameDir = Path.Combine(Path.GetTempPath(), "AE_LoaderTest_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_tempGameDir); @@ -37,19 +36,20 @@ public void Setup() _settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); _settingsService.CurrentSettings.GameDirectories.Add(new ApplicationSettings.GamePathPair(GameTypeEnum.Warhammer3, _tempGameDir)); + + _cacheHelper = new InMemoryPackFileContainerCacheHelper(); } [TearDown] public void TearDown() { + _cacheHelper.Dispose(); + if (Directory.Exists(_tempGameDir)) Directory.Delete(_tempGameDir, true); } - private PackFileContainerLoader CreateLoader() - { - return new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager); - } + private PackFileContainerLoader CreateLoader() => new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager, _cacheHelper); [Test] public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() @@ -57,7 +57,7 @@ public void LoadAllCaFiles_MissingGameDirectory_ShowsErrorAndSkipsBuild() _settingsService.CurrentSettings.GameDirectories.Clear(); var loader = CreateLoader(); - var result = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var result = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(result, Is.Null); } @@ -67,7 +67,7 @@ public void LoadAllCaFiles_NoCacheExists_ShowsNotFoundDialogAndWaitCursor() { var loader = CreateLoader(); - var result = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var result = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(result, Is.Not.Null); @@ -85,13 +85,13 @@ public void LoadAllCaFiles_CacheExists_NoDialogShown() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); _dialogs.Invocations.Clear(); // Second call should use cache - no dialogs - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Never); @@ -104,25 +104,19 @@ public void LoadAllCaFiles_CacheCorrupted_ShowsCorruptedDialog() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); - // Find and corrupt the cache file - var cacheDir = Shared.Core.Misc.DirectoryHelper.CacheDirectory; - var cacheFiles = Directory.GetFiles(cacheDir, "*.db", SearchOption.AllDirectories); - Assert.That(cacheFiles.Length, Is.GreaterThan(0), "Cache file should have been created"); - - // Corrupt all matching cache files - foreach (var cacheFile in cacheFiles) - { - if (cacheFile.Contains("Warhammer")) - File.WriteAllText(cacheFile, "CORRUPTED DATA"); - } + // Corrupt the cache via the in-memory helper + var packFiles = Directory.GetFiles(_tempGameDir, "*.pack").ToList(); + var fingerprint = _cacheHelper.ComputeFingerprint(packFiles); + var cacheFilePath = _cacheHelper.GetCacheFilePath("All Game Packs - Warhammer III", fingerprint); + _cacheHelper.CorruptCache(cacheFilePath); _dialogs.Invocations.Clear(); // Load again — should detect corruption - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); // Single combined dialog: reason + building message @@ -136,7 +130,7 @@ public void LoadAllCaFiles_DataChanged_ShowsDialogsAndRebuildsCache() var loader = CreateLoader(); // First call builds the cache - var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var firstResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(firstResult, Is.Not.Null); // Add a new pack file to the game dir to change the fingerprint @@ -146,7 +140,7 @@ public void LoadAllCaFiles_DataChanged_ShowsDialogsAndRebuildsCache() _dialogs.Invocations.Clear(); // Load again — fingerprint changed, new fingerprint has no cache file - var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + var secondResult = loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(secondResult, Is.Not.Null); // Single combined dialog: reason + building message @@ -180,7 +174,7 @@ public void LoadAllCaFiles_WaitCursorActive_DuringBuild() .Callback(() => waitCursorDisposed = true); var loader = CreateLoader(); - loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + loader.CreateFromGameEnum(PackFileContainerType.Cached, GameTypeEnum.Warhammer3); Assert.That(dialogShownBeforeWaitCursor, Is.True, "Dialog should be shown before wait cursor starts"); Assert.That(waitCursorCreated, Is.True, "Wait cursor should have been created"); diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs index 2de8df6a0..054f64e3e 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileServiceUtilityTests.cs @@ -1,48 +1,51 @@ -using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Utility; -using Shared.Core.Settings; namespace Shared.CoreTest.PackFiles.Utility { internal class PackFileServiceUtilityTests { [Test] - public void SplitDirectoryEntries_ComposesSortedFoldersAndFiles() + public void GetDirectoryContent_ReturnsSortedFiles() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, "texture"); + var files = container.GetDirectoryContent("texture") + .Select(x => x.File.Name) + .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) + .ToList(); - Assert.That(result.DirectoryPath, Is.EqualTo("texture")); - Assert.That(result.SubFolders, Is.EqualTo(new[] { "mesha", "meshb" })); - Assert.That(result.Files.Select(x => x.FileName), Is.EqualTo(new[] { "alpha.dds", "texture_file.dds" })); + Assert.That(files, Is.EqualTo(new[] { "alpha.dds", "texture_file.dds" })); } + + [Test] - public void SplitDirectoryEntries_RootDirectory_ReturnsTopLevelFoldersAndFiles() + public void GetDirectoryContent_RootDirectory_ReturnsTopLevelFiles() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, ""); + var files = container.GetDirectoryContent("") + .Select(x => x.File.Name) + .ToList(); - Assert.That(result.DirectoryPath, Is.EqualTo(string.Empty)); - Assert.That(result.SubFolders, Is.EqualTo(new[] { "audio", "texture" })); - Assert.That(result.Files.Select(x => x.FileName), Is.EqualTo(new[] { "root.txt" })); + Assert.That(files, Is.EqualTo(new[] { "root.txt" })); } + + [Test] - public void SplitDirectoryEntries_EmptyDirectory_ReturnsEmptyResult() + public void GetDirectoryContent_MissingDirectory_ReturnsEmpty() { var container = CreateContainer(); - var result = PackFileServiceUtility.SplitDirectoryEntries(container, "missing"); - Assert.That(result.DirectoryPath, Is.EqualTo("missing")); - Assert.That(result.SubFolders, Is.Empty); - Assert.That(result.Files, Is.Empty); + Assert.That(container.GetDirectoryContent("missing"), Is.Empty); } + + private static PackFileContainer CreateContainer() { var container = new PackFileContainer("Test"); diff --git a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj index 27966a8f4..f63e7efe9 100644 --- a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj +++ b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj @@ -8,6 +8,12 @@ false + + + + + + all @@ -32,8 +38,4 @@ - - - - diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs index e62bcf7ad..dc44b79c8 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.ComponentModel; using System.IO; using System.Linq; -using System.Windows.Input; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using Shared.Core.Events; diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs index 5363e8731..66f1efb61 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/Utility/PackFileTreeBuilder.cs @@ -1,163 +1,86 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; using Shared.Core.PackFiles.Models; namespace Shared.Ui.BaseDialogs.PackFileTree.Utility { public static class PackFileTreeBuilder { - private readonly record struct PathPrefixKey(string Path, int Length) - { - public static readonly PathPrefixKey Empty = new(string.Empty, 0); - - public ReadOnlySpan Span => Path.AsSpan(0, Length); - } - - private sealed class PathPrefixKeyComparer : IEqualityComparer - { - public static readonly PathPrefixKeyComparer Ordinal = new(); - - public bool Equals(PathPrefixKey x, PathPrefixKey y) - { - return x.Length == y.Length && x.Span.SequenceEqual(y.Span); - } - - public int GetHashCode(PathPrefixKey obj) - { - var hash = new HashCode(); - foreach (var ch in obj.Span) - hash.Add(ch); - - return hash.ToHashCode(); - } - } - - private static readonly Comparison TreeNodeComparison = (left, right) => + public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer container, bool skipWemFiles) { - var nodeTypeComparison = left.NodeType.CompareTo(right.NodeType); - if (nodeTypeComparison != 0) - return nodeTypeComparison; + // Get all files sorted by folders. The result is always sorted. + var fileByFolders = container.GetAllFilesByFolder(); + var addFiles = true; - return StringComparer.CurrentCultureIgnoreCase.Compare(left.Name, right.Name); - }; + var nodeLookUp = new Dictionary(); - public static void BuildTreeFromFiles(TreeNode root, IPackFileContainer container, bool skipWemFiles) - { - var allFiles = container.GetAllFiles(); - var filesByFolder = GroupFilesByFolder(allFiles, skipWemFiles); - var directoryMap = new Dictionary(filesByFolder.Count + 1, PathPrefixKeyComparer.Ordinal) + foreach (var folder in fileByFolders) { - [PathPrefixKey.Empty] = root - }; - var childrenByParent = new Dictionary>(filesByFolder.Count + 1); - var pendingDirectories = new List<(string FolderName, PathPrefixKey FullFolderPath)>(8); + var folderPath = folder.Key; - foreach (var folderPath in filesByFolder.Keys) - { if (folderPath.Length == 0) continue; - EnsureDirectoryPath(root, folderPath, directoryMap, pendingDirectories, childrenByParent); - } - - foreach (var folderEntry in filesByFolder) - { - var parentNode = directoryMap[folderEntry.Key]; - foreach (var file in folderEntry.Value) + // If this exact path already exists, skip folder creation + if (!nodeLookUp.ContainsKey(folderPath)) { - var fileNode = new TreeNode(file.Name, NodeType.File, parentNode); - AddChildForBuild(parentNode, fileNode, childrenByParent); + // Search from the right to find the deepest existing ancestor. + // Since keys are sorted, most parent paths will already be in the map. + var foldersToCreate = new List { folderPath }; + var parentNode = root; + var lastSep = folderPath.LastIndexOf(Path.DirectorySeparatorChar); + + while (lastSep != -1) + { + var parentPath = folderPath.Substring(0, lastSep); + if (nodeLookUp.TryGetValue(parentPath, out var foundNode)) + { + parentNode = foundNode; + break; + } + + foldersToCreate.Add(parentPath); + lastSep = parentPath.LastIndexOf(Path.DirectorySeparatorChar); + } + + // Create missing folders from shallowest to deepest + for (var i = foldersToCreate.Count - 1; i >= 0; i--) + { + var path = foldersToCreate[i]; + var sep = path.LastIndexOf(Path.DirectorySeparatorChar); + var folderName = sep == -1 ? path : path.Substring(sep + 1); + + var newNode = new TreeNode(folderName, NodeType.Directory, parentNode); + parentNode.AddChild(newNode); + nodeLookUp[path] = newNode; + parentNode = newNode; + } } } - FinalizeTree(root, childrenByParent); - } - - private static Dictionary> GroupFilesByFolder(Dictionary allFiles, bool skipWemFiles) - { - var filesByFolder = new Dictionary>(PathPrefixKeyComparer.Ordinal) - { - [PathPrefixKey.Empty] = [] - }; - - foreach (var item in allFiles) - { - var path = item.Key; - if (skipWemFiles && path.EndsWith(".wem", StringComparison.OrdinalIgnoreCase)) - continue; - - var separatorIndex = FindLastDirectorySeparatorIndex(path.AsSpan()); - var folderPath = separatorIndex == -1 - ? PathPrefixKey.Empty - : new PathPrefixKey(path, separatorIndex); - - ref var files = ref CollectionsMarshal.GetValueRefOrAddDefault(filesByFolder, folderPath, out _); - files ??= []; - files.Add(item.Value); - } - - return filesByFolder; - } - - private static TreeNode EnsureDirectoryPath(TreeNode root, PathPrefixKey folderPath, Dictionary directoryMap, List<(string FolderName, PathPrefixKey FullFolderPath)> pendingDirectories, Dictionary> childrenByParent) - { - if (directoryMap.TryGetValue(folderPath, out var existingDirectory)) - return existingDirectory; - - pendingDirectories.Clear(); - var currentFolderPath = folderPath; - while (currentFolderPath.Length > 0 && !directoryMap.TryGetValue(currentFolderPath, out existingDirectory)) - { - var currentPathSpan = currentFolderPath.Span; - var separatorIndex = FindLastDirectorySeparatorIndex(currentPathSpan); - var folderName = separatorIndex == -1 - ? currentFolderPath.Path[..currentFolderPath.Length] - : currentFolderPath.Path.Substring(separatorIndex + 1, currentFolderPath.Length - separatorIndex - 1); - pendingDirectories.Add((folderName, currentFolderPath)); - currentFolderPath = separatorIndex == -1 - ? PathPrefixKey.Empty - : new PathPrefixKey(currentFolderPath.Path, separatorIndex); - } - - var parentNode = currentFolderPath.Length == 0 ? root : directoryMap[currentFolderPath]; - for (var i = pendingDirectories.Count - 1; i >= 0; i--) + // Add files after all folders have been created + if (addFiles) { - var currentDirectory = pendingDirectories[i]; - var currentNode = new TreeNode(currentDirectory.FolderName, NodeType.Directory, parentNode); - AddChildForBuild(parentNode, currentNode, childrenByParent); - directoryMap[currentDirectory.FullFolderPath] = currentNode; - parentNode = currentNode; + foreach (var folder in fileByFolders) + { + if (folder.Key.Length == 0) + { + foreach (var fileName in folder.Value) + { + var fileNode = new TreeNode(fileName, NodeType.File, root); + root.AddChild(fileNode); + } + continue; + } + + var folderNode = nodeLookUp[folder.Key]; + foreach (var fileName in folder.Value) + { + var fileNode = new TreeNode(fileName, NodeType.File, folderNode); + folderNode.AddChild(fileNode); + } + } } - - return parentNode; - } - - private static void AddChildForBuild(TreeNode parent, TreeNode child, Dictionary> childrenByParent) - { - child.Parent = parent; - - ref var children = ref CollectionsMarshal.GetValueRefOrAddDefault(childrenByParent, parent, out _); - children ??= []; - children.Add(child); - } - - private static int FindLastDirectorySeparatorIndex(ReadOnlySpan path) - { - return path.LastIndexOfAny(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - } - - private static void FinalizeTree(TreeNode node, Dictionary> childrenByParent) - { - if (!childrenByParent.TryGetValue(node, out var children) || children.Count == 0) - return; - - children.Sort(TreeNodeComparison); - node.SetChildren(children); - - foreach (var child in children) - FinalizeTree(child, childrenByParent); } } } diff --git a/Testing/Shared/Shared/AssetEditorTestRunner.cs b/Testing/Shared/Shared/AssetEditorTestRunner.cs index a3362318f..126e8d829 100644 --- a/Testing/Shared/Shared/AssetEditorTestRunner.cs +++ b/Testing/Shared/Shared/AssetEditorTestRunner.cs @@ -49,7 +49,7 @@ public AssetEditorTestRunner(GameTypeEnum gameEnum = GameTypeEnum.Warhammer3, bo public IPackFileContainer? LoadPackFile(string path, bool createOutputPackFile = true) { var loader = ServiceProvider.GetRequiredService(); - var container = loader.Load(path); + var container = loader.CreateFromPackFile(PackFileContainerType.Normal, path, false); PackFileService.AddContainer(container); if (createOutputPackFile) @@ -77,7 +77,7 @@ public T GetRequiredServiceInCurrentEditorScope() public IPackFileContainer LoadFolderPackFile(string path) { var loader = ServiceProvider.GetRequiredService(); - var container = loader.LoadSystemFolderAsPackFileContainer(path); + var container = loader.CreateFromSystemFolder(path); PackFileService.AddContainer(container); return container; diff --git a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs index cf77e6172..bca42ecdb 100644 --- a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs +++ b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs @@ -1,5 +1,6 @@ using Moq; using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -12,8 +13,8 @@ public static class PackFileSerivceTestHelper public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = GameTypeEnum.Warhammer3) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager()); - var container = loader.LoadSystemFolderAsPackFileContainer(path); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); + var container = loader.CreateFromSystemFolder(path); container.IsCaPackFile = true; pfs.AddContainer(container); @@ -23,9 +24,9 @@ public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = G public static IPackFileService CreateFromFolder(GameTypeEnum selectedGame, string path ) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager()); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager(), new PackFileContainerCacheHelper()); - var container = loader.LoadSystemFolderAsPackFileContainer(PathHelper.GetDataFolder(path)); + var container = loader.CreateFromSystemFolder(PathHelper.GetDataFolder(path)); container.IsCaPackFile = true; pfs.AddContainer(container); return pfs;