feat: Introduce more temporary bulk importer (skin URLs and hostnames)

Author: SpraxDev

src/cli/commands/ImportCommand.ts

@@ -5,14 +5,14 @@ import BulkQueueImporter, { type BulkQueueImportResult } from '../../import_queu
 import CliCommand from './CliCommand.js';
 
 type ImportCommandArgs = {
-  type: 'uuid' | 'username' | 'profile-texture-value' | 'dir-with-skin-files',
+  type: 'uuid' | 'username' | 'profile-texture-value' | 'dir-with-skin-files' | 'skin-urls' | 'domains',
   filePath: string,
   apiKeyId: bigint
 };
 
 @injectable({ token: ContainerTokens.CLI_COMMAND })
 export default class ImportCommand implements CliCommand {
-  private readonly VALID_IMPORT_TYPES: string[] = ['uuid', 'username', 'profile-texture-value', 'dir-with-skin-files'] satisfies ImportCommandArgs['type'][];
+  private readonly VALID_IMPORT_TYPES: string[] = ['uuid', 'username', 'profile-texture-value', 'dir-with-skin-files', 'skin-urls', 'domains'] satisfies ImportCommandArgs['type'][];
 
   constructor(
     private readonly bulkQueueImporter: BulkQueueImporter,

src/import_queue/bulk/BulkQueueImporter.ts

@@ -3,9 +3,13 @@ import Fs from 'node:fs';
 import Path from 'node:path';
 import { singleton } from 'tsyringe';
 import DatabaseClient from '../../database/DatabaseClient.js';
+import AutoProxiedHttpClient from '../../http/clients/AutoProxiedHttpClient.js';
+import ServerBlocklistService from '../../minecraft/server/blocklist/ServerBlocklistService.js';
 import BulkImporter from './importer/BulkImporter.js';
+import DomainBulkImporter from './importer/DomainBulkImporter.js';
 import ProfileTextureValueBulkImporter from './importer/ProfileTextureValueBulkImporter.js';
 import SkinFileBulkImporter from './importer/SkinFileBulkImporter.js';
+import SkinUrlBulkImporter from './importer/SkinUrlBulkImporter.js';
 import UsernameBulkImporter from './importer/UsernameBulkImporter.js';
 import UuidBulkImporter from './importer/UuidBulkImporter.js';
 
@@ -22,10 +26,12 @@ export type BulkQueueImportResult = {
 export default class BulkQueueImporter {
   constructor(
     private readonly databaseClient: DatabaseClient,
+    private readonly httpClient: AutoProxiedHttpClient,
+    private readonly serverBlocklistService: ServerBlocklistService,
   ) {
   }
 
-  async importEachLine(filePath: string, type: 'uuid' | 'username' | 'profile-texture-value', importingApiKeyId: bigint): Promise<BulkQueueImportResult> {
+  async importEachLine(filePath: string, type: 'uuid' | 'username' | 'profile-texture-value' | 'skin-urls' | 'domains', importingApiKeyId: bigint): Promise<BulkQueueImportResult> {
     const fileHandle = await Fs.promises.open(filePath, 'r');
     const totalFileBytes = (await fileHandle.stat()).size;
 
@@ -40,6 +46,12 @@ export default class BulkQueueImporter {
       case 'profile-texture-value':
         payloadImporter = new ProfileTextureValueBulkImporter(new UuidBulkImporter());
         break;
+      case 'skin-urls':
+        payloadImporter = new SkinUrlBulkImporter(this.httpClient);
+        break;
+      case 'domains':
+        payloadImporter = new DomainBulkImporter(this.serverBlocklistService);
+        break;
 
       default:
         throw new Error(`Unsupported bulk queue import type: ${type}`);
@@ -94,7 +106,7 @@ export default class BulkQueueImporter {
             continue;
           }
 
-          insertBatch.push(...payloadImporter.createTasks(payload, importGroup.id));
+          insertBatch.push(...(await payloadImporter.createTasks(payload, importGroup.id)));
           if (insertBatch.length >= 250) {
             const newlyQueued = await this.writeBatch(transaction, insertBatch);
             result.queued += newlyQueued;

src/import_queue/bulk/importer/BulkImporter.ts

@@ -3,5 +3,5 @@ import type * as PrismaClient from '@prisma/client';
 export default interface BulkImporter {
   isValidPayload(payload: string): true | string | Promise<true | string>;
 
-  createTasks(payload: string, importGroupId: bigint): PrismaClient.Prisma.ImportTaskCreateManyInput[];
+  createTasks(payload: string, importGroupId: bigint): PrismaClient.Prisma.ImportTaskCreateManyInput[]|Promise<PrismaClient.Prisma.ImportTaskCreateManyInput[]>;
 }

src/import_queue/bulk/importer/DomainBulkImporter.ts

@@ -0,0 +1,24 @@
+import * as PrismaClient from '@prisma/client';
+import type ServerBlocklistService from '../../../minecraft/server/blocklist/ServerBlocklistService.js';
+import type BulkImporter from './BulkImporter.js';
+
+export default class DomainBulkImporter implements BulkImporter {
+  constructor(
+    private readonly serverBlocklistService: ServerBlocklistService,
+  ) {
+  }
+
+  isValidPayload(payload: string): true | string {
+    return true;
+  }
+
+  async createTasks(payload: string, importGroupId: bigint): Promise<PrismaClient.Prisma.ImportTaskCreateManyInput[]> {
+    try {
+      await this.serverBlocklistService.checkBlocklist(payload);
+    } catch (err) {
+      console.error(`Error checking blocklist for Hostname ${payload}: ` + (err as Error).message);
+    }
+
+    return [];
+  }
+}

src/import_queue/bulk/importer/SkinUrlBulkImporter.ts

@@ -0,0 +1,43 @@
+import * as PrismaClient from '@prisma/client';
+import type AutoProxiedHttpClient from '../../../http/clients/AutoProxiedHttpClient.js';
+import type BulkImporter from './BulkImporter.js';
+
+export default class SkinUrlBulkImporter implements BulkImporter {
+  constructor(
+    private readonly httpClient: AutoProxiedHttpClient,
+  ) {
+  }
+
+  isValidPayload(payload: string): true | string {
+    try {
+      new URL(payload);
+      return true;
+    } catch (err: any) {
+      return `Invalid URL (${err.message}): ${JSON.stringify(payload)}`;
+    }
+  }
+
+  async createTasks(payload: string, importGroupId: bigint): Promise<PrismaClient.Prisma.ImportTaskCreateManyInput[]> {
+    const skinImage = await this.httpClient.get(payload);
+    if (skinImage.statusCode === 404) {
+      console.warn('Skin URL returned 404 Not Found: ' + payload);
+      return [];
+    }
+    if (skinImage.statusCode !== 200) {
+      console.error(`Skin URL returned unexpected status code ${skinImage.statusCode}: ` + payload);
+      return [];
+    }
+
+    const contentType = skinImage.headers.get('content-type');
+    if (typeof contentType === 'string' && !contentType.includes('image/') && contentType !== 'binary/octet-stream') {
+      console.error(`Skin URL returned unexpected content type ${contentType}: ` + payload);
+      return [];
+    }
+
+    return [{
+      payload: skinImage.body,
+      payloadType: 'SKIN_IMAGE',
+      importGroupId,
+    }];
+  }
+}

src/import_queue/worker/ContinuousQueueWorker.ts

@@ -78,7 +78,7 @@ export default class ContinuousQueueWorker {
   private async tick(): Promise<void> {
     const task = await this.fetchNextTask();
     if (task == null) {
-      console.debug('No tasks in the queue, waiting for new tasks...');
+//      console.debug('No tasks in the queue, waiting for new tasks...');
       await this.tickForEmptyQueue();
       return;
     }