Add support for simple background migration jobs

Refresh.Database/GameDatabaseContext.Workers.cs

@@ -40,4 +40,30 @@ public bool MarkWorkerContacted(int id)
 
         return true;
     }
+
+    public object? GetJobState(string jobId, Type type)
+    {
+        PersistentJobState? state = this.JobStates.FirstOrDefault(s => s.JobId == jobId);
+        if (state == null)
+            return null;
+
+        return JsonConvert.DeserializeObject(state.State, type);
+    }
+
+    public void UpdateOrCreateJobState(string jobId, object state)
+    {
+        PersistentJobState? jobState = this.JobStates.FirstOrDefault(s => s.JobId == jobId);
+        if (jobState == null)
+        {
+            jobState = new PersistentJobState
+            {
+                JobId = jobId,
+            };
+
+            this.JobStates.Add(jobState);
+        }
+
+        jobState.State = JsonConvert.SerializeObject(state, Formatting.None);
+        this.SaveChanges();
+    }
 }

Refresh.Database/GameDatabaseContext.cs

@@ -73,6 +73,7 @@ public partial class GameDatabaseContext : DbContext, IDatabaseContext
     internal DbSet<ProfilePinRelation> ProfilePinRelations { get; set; }
     internal DbSet<GameSkillReward> GameSkillRewards { get; set; }
     internal DbSet<WorkerInfo> Workers { get; set; }
+    internal DbSet<PersistentJobState> JobStates { get; set; }
 
 #pragma warning disable CS8618 // Non-nullable variable must contain a non-null value when exiting constructor. Consider declaring it as nullable.
     internal GameDatabaseContext(Logger logger, IDateTimeProvider time, IDatabaseConfig dbConfig)

Refresh.Database/Migrations/20250722045529_AddJobStateTable.cs

@@ -0,0 +1,36 @@
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Refresh.Database.Migrations
+{
+    [DbContext(typeof(GameDatabaseContext))]
+    [Migration("20250722045529_AddJobStateTable")]
+    /// <inheritdoc />
+    public partial class AddJobStateTable : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "JobStates",
+                columns: table => new
+                {
+                    JobId = table.Column<string>(type: "text", nullable: false),
+                    State = table.Column<string>(type: "jsonb", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_JobStates", x => x.JobId);
+                });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "JobStates");
+        }
+    }
+}

Refresh.Database/Migrations/GameDatabaseContextModelSnapshot.cs

@@ -1437,6 +1437,20 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("QueuedRegistrations");
                 });
 
+            modelBuilder.Entity("Refresh.Database.Models.Workers.PersistentJobState", b =>
+                {
+                    b.Property<string>("JobId")
+                        .HasColumnType("text");
+
+                    b.Property<string>("State")
+                        .IsRequired()
+                        .HasColumnType("jsonb");
+
+                    b.HasKey("JobId");
+
+                    b.ToTable("JobStates");
+                });
+
             modelBuilder.Entity("Refresh.Database.Models.Workers.WorkerInfo", b =>
                 {
                     b.Property<int>("WorkerId")

Refresh.Database/Models/Workers/PersistentJobState.cs

@@ -0,0 +1,7 @@
+namespace Refresh.Database.Models.Workers;
+
+public class PersistentJobState
+{
+    [Key, Required] public string JobId { get; set; } = null!;
+    [Column(TypeName = "jsonb"), Required] public string State { get; set; } = null!;
+}

Refresh.Workers/MigrationJob.cs

@@ -0,0 +1,49 @@
+using Microsoft.EntityFrameworkCore.Storage;
+using Refresh.Workers.State;
+
+namespace Refresh.Workers;
+
+public abstract class MigrationJob<TEntity> : WorkerJob, IJobStoresState where TEntity : class
+{
+    public virtual string JobId => this.GetType().Name;
+    public object JobState { get; set; } = null!;
+    public Type JobStateType => typeof(MigrationJobState);
+
+    public MigrationJobState? MigrationJobState => JobState as MigrationJobState;
+
+    protected virtual int BatchCount => 1_000;
+    protected virtual IQueryable<TEntity> SortAndFilter(IQueryable<TEntity> query) => query;
+
+    public override bool CanExecute()
+    {
+        return this.MigrationJobState == null || !this.MigrationJobState.StateInitialized || !this.MigrationJobState.Complete;
+    }
+
+    public override void ExecuteJob(WorkContext context)
+    {
+        IQueryable<TEntity> query = context.Database.Set<TEntity>();
+        query = this.SortAndFilter(query);
+
+        MigrationJobState state = this.MigrationJobState!;
+
+        if (!state.StateInitialized)
+        {
+            state.Total = query.Count();
+            state.StateInitialized = true;
+        }
+
+        query = query.Skip(state.Processed).Take(this.BatchCount);
+
+        using IDbContextTransaction transaction = context.Database.Database.BeginTransaction();
+
+        TEntity[] batch = query.ToArray();
+
+        Migrate(context, batch);
+        context.Database.SaveChanges();
+        transaction.Commit();
+
+        state.Processed += batch.Length;
+    }
+
+    protected abstract void Migrate(WorkContext context, TEntity[] batch);
+}

Refresh.Workers/Refresh.Workers.csproj

@@ -11,6 +11,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <InternalsVisibleTo Include="RefreshTests.GameServer" />
         <ProjectReference Include="..\Refresh.Core\Refresh.Core.csproj" />
     </ItemGroup>
 

Refresh.Workers/State/IJobStoresState.cs

@@ -0,0 +1,8 @@
+namespace Refresh.Workers.State;
+
+public interface IJobStoresState
+{
+    public string JobId { get; }
+    public object JobState { get; set; }
+    public Type JobStateType { get; }
+}

Refresh.Workers/State/MigrationJobState.cs

@@ -0,0 +1,11 @@
+namespace Refresh.Workers.State;
+
+public class MigrationJobState
+{
+    public bool StateInitialized;
+    public int Total;
+    public int Processed;
+
+    public bool Complete => this.Remaining <= 0;
+    public int Remaining => this.Total - this.Processed;
+}

Refresh.Workers/WorkerManager.cs

@@ -2,6 +2,7 @@
 using NotEnoughLogs;
 using Refresh.Core;
 using Refresh.Database;
+using Refresh.Workers.State;
 
 namespace Refresh.Workers;
 
@@ -54,7 +55,22 @@ private void RunWorkCycle()
             if (!job.CanExecute())
                 continue;
 
+            IJobStoresState? jobWithState = job as IJobStoresState;
+            if (jobWithState != null)
+            {
+                object? jobState = context.Database.GetJobState(jobWithState.JobId, jobWithState.JobStateType);
+                jobState ??= Activator.CreateInstance(jobWithState.JobStateType);
+
+                jobWithState.JobState = jobState!;
+
+                // jobs that consume state may have different execution requirements when state is updated
+                // check again to handle this case. the check above is still retained to avoid unnecessary db lookups
+                if (!job.CanExecute())
+                    continue;
+            }
+
             this._logger.LogTrace(RefreshContext.Worker, $"Running work cycle for {job.GetType().Name}");
+
             try
             {
                 job.ExecuteJob(context);
@@ -64,6 +80,9 @@ private void RunWorkCycle()
             {
                 this._logger.LogError(RefreshContext.Worker, $"Unhandled exception while running work cycle for {job.GetType().Name}: {e}");
             }
+
+            if (jobWithState != null)
+                context.Database.UpdateOrCreateJobState(jobWithState.JobId, jobWithState.JobState);
         }
 
         long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();

RefreshTests.GameServer/GameServer/TestMigrationJob.cs

@@ -0,0 +1,20 @@
+using Refresh.Database.Models.Levels;
+using Refresh.Workers;
+
+namespace RefreshTests.GameServer.GameServer;
+
+public class TestMigrationJob : MigrationJob<GameLevel>
+{
+    protected override void Migrate(WorkContext context, GameLevel[] batch)
+    {
+        foreach (GameLevel level in batch)
+        {
+            level.Title += " test";
+        }
+    }
+
+    protected override IQueryable<GameLevel> SortAndFilter(IQueryable<GameLevel> query)
+    {
+        return query.OrderBy(l => l.LevelId);
+    }
+}

RefreshTests.GameServer/Tests/Workers/MigrationTests.cs

@@ -0,0 +1,32 @@
+using Refresh.Database.Models.Authentication;
+using Refresh.Database.Models.Levels;
+using Refresh.Database.Models.Users;
+using Refresh.Database.Query;
+using Refresh.Workers.State;
+
+namespace RefreshTests.GameServer.Tests.Workers;
+
+public class MigrationTests : GameServerTest
+{
+    [Test]
+    public void MigrationJobWorks()
+    {
+        using TestContext context = this.GetServer();
+        TestMigrationJob job = new();
+        GameUser user = context.CreateUser();
+
+        for (int i = 0; i < 100; i++)
+        {
+            context.CreateLevel(user);
+        }
+
+        IEnumerable<GameLevel> allLevels = context.Database.GetNewestLevels(100, 0, null, new LevelFilterSettings(TokenGame.Website)).Items;
+        Assert.That(allLevels.All(l => l.Title == "Level"), Is.True);
+
+        job.JobState = new MigrationJobState();
+        job.ExecuteJob(context.GetWorkContext());
+
+        allLevels = context.Database.GetNewestLevels(100, 0, null, new LevelFilterSettings(TokenGame.Website)).Items;
+        Assert.That(allLevels.All(l => l.Title == "Level test"), Is.True);
+    }
+}