From 8b7ebdc2722b248e83aad4085591c294ddf08aa5 Mon Sep 17 00:00:00 2001 From: jvyden Date: Thu, 2 May 2024 04:16:03 -0400 Subject: [PATCH 1/6] zip path getting --- Refresh.GameServer/RefreshGameServer.cs | 8 +- .../Storage/DryArchiveConfig.cs | 18 +++++ Refresh.GameServer/Storage/DryDataStore.cs | 78 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 Refresh.GameServer/Storage/DryArchiveConfig.cs create mode 100644 Refresh.GameServer/Storage/DryDataStore.cs diff --git a/Refresh.GameServer/RefreshGameServer.cs b/Refresh.GameServer/RefreshGameServer.cs index ebb5abb0a..43be94d24 100644 --- a/Refresh.GameServer/RefreshGameServer.cs +++ b/Refresh.GameServer/RefreshGameServer.cs @@ -19,6 +19,7 @@ using Refresh.GameServer.Importing; using Refresh.GameServer.Middlewares; using Refresh.GameServer.Services; +using Refresh.GameServer.Storage; using Refresh.GameServer.Time; using Refresh.GameServer.Types.Levels.Categories; using Refresh.GameServer.Types.Roles; @@ -46,7 +47,12 @@ public RefreshGameServer( ) : base(listener) { databaseProvider ??= () => new GameDatabaseProvider(); - dataStore ??= new FileSystemDataStore(); + dataStore ??= new DryDataStore(new DryArchiveConfig() + { + Enabled = true, + Location = "/home/jvyden/Documents/dry/", + UseFolderNames = true, + }); this._databaseProvider = databaseProvider.Invoke(); this._databaseProvider.Initialize(); diff --git a/Refresh.GameServer/Storage/DryArchiveConfig.cs b/Refresh.GameServer/Storage/DryArchiveConfig.cs new file mode 100644 index 000000000..c71bd74ae --- /dev/null +++ b/Refresh.GameServer/Storage/DryArchiveConfig.cs @@ -0,0 +1,18 @@ +using Bunkum.Core.Configuration; + +namespace Refresh.GameServer.Storage; + +public class DryArchiveConfig : Config +{ + public override int CurrentConfigVersion => 1; + public override int Version { get; set; } + + protected override void Migrate(int oldVer, dynamic oldConfig) + { + + } + + public bool Enabled { get; set; } + public string Location { get; set; } = "/var/dry"; + public bool UseFolderNames { get; set; } = true; +} \ No newline at end of file diff --git a/Refresh.GameServer/Storage/DryDataStore.cs b/Refresh.GameServer/Storage/DryDataStore.cs new file mode 100644 index 000000000..5289b10c6 --- /dev/null +++ b/Refresh.GameServer/Storage/DryDataStore.cs @@ -0,0 +1,78 @@ +using System.Text; +using Bunkum.Core.Storage; +using Refresh.GameServer.Verification; + +namespace Refresh.GameServer.Storage; + +public class DryDataStore : IDataStore +{ + private DryArchiveConfig _config; + + public DryDataStore(DryArchiveConfig config) + { + this._config = config; + } + + private string? GetZipPath(string hash) + { + if (!CommonPatterns.Sha1Regex().IsMatch(hash)) + return null; + + StringBuilder builder = new(); + builder.Append(this._config.Location); + + if (this._config.UseFolderNames) + { + builder.Append("dry23r"); + builder.Append(hash[0]); + builder.Append('/'); + } + + builder.Append("dry"); + builder.Append(hash.AsSpan(0, 2)); + builder.Append(".zip"); + + return builder.ToString(); + } + + public bool ExistsInStore(string key) + { + Console.WriteLine(this.GetZipPath(key)); + throw new NotImplementedException(); + } + + public bool WriteToStore(string key, byte[] data) + { + throw new NotImplementedException(); + } + + public byte[] GetDataFromStore(string key) + { + throw new NotImplementedException(); + } + + public bool RemoveFromStore(string key) + { + throw new NotImplementedException(); + } + + public string[] GetKeysFromStore() + { + throw new NotImplementedException(); + } + + public bool WriteToStoreFromStream(string key, Stream data) + { + throw new NotImplementedException(); + } + + public Stream GetStreamFromStore(string key) + { + throw new NotImplementedException(); + } + + public Stream OpenWriteStream(string key) + { + throw new NotImplementedException(); + } +} \ No newline at end of file From c03240174a9d04f7228f5d11c31dd39a9167c926 Mon Sep 17 00:00:00 2001 From: jvyden Date: Thu, 2 May 2024 04:40:50 -0400 Subject: [PATCH 2/6] Path retrieval --- Refresh.GameServer/Storage/DryDataStore.cs | 46 +++++++++++++++++----- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/Refresh.GameServer/Storage/DryDataStore.cs b/Refresh.GameServer/Storage/DryDataStore.cs index 5289b10c6..9406e82d0 100644 --- a/Refresh.GameServer/Storage/DryDataStore.cs +++ b/Refresh.GameServer/Storage/DryDataStore.cs @@ -1,19 +1,23 @@ +using System.IO.Compression; using System.Text; using Bunkum.Core.Storage; +using JetBrains.Annotations; using Refresh.GameServer.Verification; namespace Refresh.GameServer.Storage; public class DryDataStore : IDataStore { - private DryArchiveConfig _config; + private readonly DryArchiveConfig _config; + + private const string WriteError = $"{nameof(DryDataStore)} is an archive, and cannot be written to."; public DryDataStore(DryArchiveConfig config) { this._config = config; } - private string? GetZipPath(string hash) + private string? GetZipPath(ReadOnlySpan hash) { if (!CommonPatterns.Sha1Regex().IsMatch(hash)) return null; @@ -29,21 +33,45 @@ public DryDataStore(DryArchiveConfig config) } builder.Append("dry"); - builder.Append(hash.AsSpan(0, 2)); + builder.Append(hash.Slice(0, 2)); builder.Append(".zip"); return builder.ToString(); } + private static string GetEntryPath(ReadOnlySpan hash) + { + StringBuilder builder = new(); + builder.Append(hash.Slice(0, 2)); + builder.Append('/'); + builder.Append(hash.Slice(2, 2)); + builder.Append('/'); + builder.Append(hash); + + return builder.ToString(); + } + + [Pure] + private ZipArchive? OpenZipForHash(string hash) + { + string? zipPath = this.GetZipPath(hash); + if (zipPath == null) return null; + + FileStream stream = File.OpenRead(zipPath); + ZipArchive zipArchive = new(stream, ZipArchiveMode.Read, false); + + return zipArchive; + } + public bool ExistsInStore(string key) { - Console.WriteLine(this.GetZipPath(key)); - throw new NotImplementedException(); + using ZipArchive? zip = this.OpenZipForHash(key); + return zip?.GetEntry(GetEntryPath(key)) != null; } public bool WriteToStore(string key, byte[] data) { - throw new NotImplementedException(); + throw new InvalidOperationException(WriteError); } public byte[] GetDataFromStore(string key) @@ -53,7 +81,7 @@ public byte[] GetDataFromStore(string key) public bool RemoveFromStore(string key) { - throw new NotImplementedException(); + throw new InvalidOperationException(WriteError); } public string[] GetKeysFromStore() @@ -63,7 +91,7 @@ public string[] GetKeysFromStore() public bool WriteToStoreFromStream(string key, Stream data) { - throw new NotImplementedException(); + throw new InvalidOperationException(WriteError); } public Stream GetStreamFromStore(string key) @@ -73,6 +101,6 @@ public Stream GetStreamFromStore(string key) public Stream OpenWriteStream(string key) { - throw new NotImplementedException(); + throw new InvalidOperationException(WriteError); } } \ No newline at end of file From b81f94f6e6c7b5c6966ff8874517fa316d577136 Mon Sep 17 00:00:00 2001 From: jvyden Date: Thu, 2 May 2024 05:10:10 -0400 Subject: [PATCH 3/6] it works but rip ram --- Refresh.GameServer/Storage/DryDataStore.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Refresh.GameServer/Storage/DryDataStore.cs b/Refresh.GameServer/Storage/DryDataStore.cs index 9406e82d0..9e281d614 100644 --- a/Refresh.GameServer/Storage/DryDataStore.cs +++ b/Refresh.GameServer/Storage/DryDataStore.cs @@ -76,7 +76,11 @@ public bool WriteToStore(string key, byte[] data) public byte[] GetDataFromStore(string key) { - throw new NotImplementedException(); + using MemoryStream ms = new(); + Stream stream = this.GetStreamFromStore(key); + stream.CopyTo(ms); + + return ms.ToArray(); } public bool RemoveFromStore(string key) @@ -96,7 +100,12 @@ public bool WriteToStoreFromStream(string key, Stream data) public Stream GetStreamFromStore(string key) { - throw new NotImplementedException(); + ZipArchive? zip = this.OpenZipForHash(key); + Stream? stream = zip?.GetEntry(GetEntryPath(key))?.Open(); + if (stream == null) + throw new InvalidOperationException("The entry could not be found, or the zip could not be opened."); + + return stream; } public Stream OpenWriteStream(string key) From b2d2cd3dc717f7aaba30d341aa284f7c01359188 Mon Sep 17 00:00:00 2001 From: jvyden Date: Sun, 5 May 2024 00:51:52 -0400 Subject: [PATCH 4/6] Extracted format --- Refresh.GameServer/RefreshGameServer.cs | 6 +- .../Storage/DryArchiveConfig.cs | 2 +- Refresh.GameServer/Storage/DryDataStore.cs | 68 +++++++------------ 3 files changed, 30 insertions(+), 46 deletions(-) diff --git a/Refresh.GameServer/RefreshGameServer.cs b/Refresh.GameServer/RefreshGameServer.cs index 43be94d24..0ecc97bd9 100644 --- a/Refresh.GameServer/RefreshGameServer.cs +++ b/Refresh.GameServer/RefreshGameServer.cs @@ -47,11 +47,11 @@ public RefreshGameServer( ) : base(listener) { databaseProvider ??= () => new GameDatabaseProvider(); - dataStore ??= new DryDataStore(new DryArchiveConfig() + dataStore ??= new DryDataStore(new DryArchiveConfig { Enabled = true, - Location = "/home/jvyden/Documents/dry/", - UseFolderNames = true, + Location = "/home/jvyden/Documents/dry/extracted", + UseFolderNames = false, }); this._databaseProvider = databaseProvider.Invoke(); diff --git a/Refresh.GameServer/Storage/DryArchiveConfig.cs b/Refresh.GameServer/Storage/DryArchiveConfig.cs index c71bd74ae..690d2c760 100644 --- a/Refresh.GameServer/Storage/DryArchiveConfig.cs +++ b/Refresh.GameServer/Storage/DryArchiveConfig.cs @@ -13,6 +13,6 @@ protected override void Migrate(int oldVer, dynamic oldConfig) } public bool Enabled { get; set; } - public string Location { get; set; } = "/var/dry"; + public string Location { get; set; } = "/var/dry/"; public bool UseFolderNames { get; set; } = true; } \ No newline at end of file diff --git a/Refresh.GameServer/Storage/DryDataStore.cs b/Refresh.GameServer/Storage/DryDataStore.cs index 9e281d614..73d9bf084 100644 --- a/Refresh.GameServer/Storage/DryDataStore.cs +++ b/Refresh.GameServer/Storage/DryDataStore.cs @@ -1,4 +1,3 @@ -using System.IO.Compression; using System.Text; using Bunkum.Core.Storage; using JetBrains.Annotations; @@ -17,14 +16,22 @@ public DryDataStore(DryArchiveConfig config) this._config = config; } - private string? GetZipPath(ReadOnlySpan hash) + [Pure] + private string? GetPath(ReadOnlySpan hash) { if (!CommonPatterns.Sha1Regex().IsMatch(hash)) return null; StringBuilder builder = new(); + + // /var/dry builder.Append(this._config.Location); + // /var/dry/ + if (!this._config.Location.EndsWith('/')) + builder.Append('/'); + + // /var/dry/dry23r0/ if (this._config.UseFolderNames) { builder.Append("dry23r"); @@ -32,42 +39,21 @@ public DryDataStore(DryArchiveConfig config) builder.Append('/'); } - builder.Append("dry"); - builder.Append(hash.Slice(0, 2)); - builder.Append(".zip"); - - return builder.ToString(); - } - - private static string GetEntryPath(ReadOnlySpan hash) - { - StringBuilder builder = new(); + // /var/dry/dry23r0/01/ builder.Append(hash.Slice(0, 2)); builder.Append('/'); + + // /var/dry/dry23r0/01/02/ builder.Append(hash.Slice(2, 2)); builder.Append('/'); + + // /var/dry/dry2er0/01/02/010220123644c8e53d4054bf0d30d0e2bd0786ff8 builder.Append(hash); return builder.ToString(); } - [Pure] - private ZipArchive? OpenZipForHash(string hash) - { - string? zipPath = this.GetZipPath(hash); - if (zipPath == null) return null; - - FileStream stream = File.OpenRead(zipPath); - ZipArchive zipArchive = new(stream, ZipArchiveMode.Read, false); - - return zipArchive; - } - - public bool ExistsInStore(string key) - { - using ZipArchive? zip = this.OpenZipForHash(key); - return zip?.GetEntry(GetEntryPath(key)) != null; - } + public bool ExistsInStore(string key) => File.Exists(this.GetPath(key)); public bool WriteToStore(string key, byte[] data) { @@ -76,11 +62,12 @@ public bool WriteToStore(string key, byte[] data) public byte[] GetDataFromStore(string key) { - using MemoryStream ms = new(); - Stream stream = this.GetStreamFromStore(key); - stream.CopyTo(ms); + string? path = this.GetPath(key); + + if (path == null) + throw new FormatException("The key was invalid."); - return ms.ToArray(); + return File.ReadAllBytes(path); } public bool RemoveFromStore(string key) @@ -88,10 +75,7 @@ public bool RemoveFromStore(string key) throw new InvalidOperationException(WriteError); } - public string[] GetKeysFromStore() - { - throw new NotImplementedException(); - } + public string[] GetKeysFromStore() => []; public bool WriteToStoreFromStream(string key, Stream data) { @@ -100,12 +84,12 @@ public bool WriteToStoreFromStream(string key, Stream data) public Stream GetStreamFromStore(string key) { - ZipArchive? zip = this.OpenZipForHash(key); - Stream? stream = zip?.GetEntry(GetEntryPath(key))?.Open(); - if (stream == null) - throw new InvalidOperationException("The entry could not be found, or the zip could not be opened."); + string? path = this.GetPath(key); + + if (path == null) + throw new FormatException("The key was invalid."); - return stream; + return File.OpenRead(path); } public Stream OpenWriteStream(string key) From 6aba443e2749377bda86fdcd1c2ee5e088915118 Mon Sep 17 00:00:00 2001 From: jvyden Date: Sun, 5 May 2024 01:50:27 -0400 Subject: [PATCH 5/6] Aggregate data stores --- Refresh.GameServer/RefreshGameServer.cs | 15 ++++--- .../Storage/AggregateDataStore.cs | 42 +++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 Refresh.GameServer/Storage/AggregateDataStore.cs diff --git a/Refresh.GameServer/RefreshGameServer.cs b/Refresh.GameServer/RefreshGameServer.cs index 0ecc97bd9..47ae11995 100644 --- a/Refresh.GameServer/RefreshGameServer.cs +++ b/Refresh.GameServer/RefreshGameServer.cs @@ -47,27 +47,26 @@ public RefreshGameServer( ) : base(listener) { databaseProvider ??= () => new GameDatabaseProvider(); - dataStore ??= new DryDataStore(new DryArchiveConfig - { - Enabled = true, - Location = "/home/jvyden/Documents/dry/extracted", - UseFolderNames = false, - }); + dataStore ??= new FileSystemDataStore(); this._databaseProvider = databaseProvider.Invoke(); this._databaseProvider.Initialize(); this._dataStore = dataStore; + DryArchiveConfig dryConfig = Config.LoadFromJsonFile("dry.json", this.Logger); + if (dryConfig.Enabled) + this._dataStore = new AggregateDataStore(dataStore, new DryDataStore(dryConfig)); + this.SetupInitializer(() => { GameDatabaseProvider provider = databaseProvider.Invoke(); this.WorkerManager?.Stop(); - this.WorkerManager = new WorkerManager(this.Logger, this._dataStore!, provider); + this.WorkerManager = new WorkerManager(this.Logger, this._dataStore, provider); authProvider ??= new GameAuthenticationProvider(this._config!); - this.InjectBaseServices(provider, authProvider, dataStore); + this.InjectBaseServices(provider, authProvider, this._dataStore); }); } diff --git a/Refresh.GameServer/Storage/AggregateDataStore.cs b/Refresh.GameServer/Storage/AggregateDataStore.cs new file mode 100644 index 000000000..b3dc645af --- /dev/null +++ b/Refresh.GameServer/Storage/AggregateDataStore.cs @@ -0,0 +1,42 @@ +using System.Collections.Frozen; +using Bunkum.Core.Storage; + +namespace Refresh.GameServer.Storage; + +public class AggregateDataStore : IDataStore +{ + private IDataStore PrimaryStore { get; } + private FrozenSet AllStores { get; } + + public AggregateDataStore(IDataStore primary, params IDataStore[] others) + { + this.PrimaryStore = primary; + + List stores = new(others.Length + 1) { primary }; + stores.AddRange(others); + + this.AllStores = stores.ToFrozenSet(); + } + + private IDataStore GetStoreContainingKey(string key) + { + IDataStore? store = this.AllStores.FirstOrDefault(s => s.ExistsInStore(key)); + + if (store == null) + throw new InvalidOperationException($"No data stores contained the key '{key}'."); + + return store; + } + + public bool ExistsInStore(string key) => this.AllStores.Any(s => s.ExistsInStore(key)); + public byte[] GetDataFromStore(string key) => this.GetStoreContainingKey(key).GetDataFromStore(key); + public Stream GetStreamFromStore(string key) => this.GetStoreContainingKey(key).GetStreamFromStore(key); + + public string[] GetKeysFromStore() => this.AllStores.SelectMany(r => r.GetKeysFromStore()).ToArray(); + + public bool WriteToStore(string key, byte[] data) => this.PrimaryStore.WriteToStore(key, data); + public bool RemoveFromStore(string key) => this.PrimaryStore.RemoveFromStore(key); + + public Stream OpenWriteStream(string key) => this.PrimaryStore.OpenWriteStream(key); + public bool WriteToStoreFromStream(string key, Stream data) => this.PrimaryStore.WriteToStoreFromStream(key, data); +} \ No newline at end of file From 66f719271b56a44c7cf0a759694f1baf7bf49371 Mon Sep 17 00:00:00 2001 From: jvyden Date: Sun, 5 May 2024 01:54:24 -0400 Subject: [PATCH 6/6] TODO for GetKeysFromDataStore --- Refresh.GameServer/Storage/DryDataStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Refresh.GameServer/Storage/DryDataStore.cs b/Refresh.GameServer/Storage/DryDataStore.cs index 73d9bf084..885c29b4e 100644 --- a/Refresh.GameServer/Storage/DryDataStore.cs +++ b/Refresh.GameServer/Storage/DryDataStore.cs @@ -75,7 +75,7 @@ public bool RemoveFromStore(string key) throw new InvalidOperationException(WriteError); } - public string[] GetKeysFromStore() => []; + public string[] GetKeysFromStore() => []; // TODO: Implement when we want to store these as GameAssets public bool WriteToStoreFromStream(string key, Stream data) {