Support R2 Raid Images via backend

.github/workflows/coverage.yml

@@ -49,6 +49,11 @@ jobs:
                   BUNGIE_API_KEY: ${{ secrets.BUNGIE_API_KEY }}
                   RABBIT_API_USER: ${{ secrets.RABBIT_API_USER }}
                   RABBIT_API_PASSWORD: ${{ secrets.RABBIT_API_PASSWORD }}
+                  R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+                  R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+                  R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+                  R2_BUCKET: ${{ secrets.R2_BUCKET }}
+
               run: bun test --timeout=30000 --coverage
               continue-on-error: true
 

.github/workflows/test.yml

@@ -58,6 +58,11 @@ jobs:
                   BUNGIE_API_KEY: ${{ secrets.BUNGIE_API_KEY }}
                   RABBIT_API_USER: ${{ secrets.RABBIT_API_USER }}
                   RABBIT_API_PASSWORD: ${{ secrets.RABBIT_API_PASSWORD }}
+                  R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+                  R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+                  R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+                  R2_BUCKET: ${{ secrets.R2_BUCKET }}
+
               run: bun test --coverage --bail=5 --timeout=20000
 
             #  Compares two code coverage files and generates report as a comment

example.env

@@ -12,4 +12,9 @@ ADMIN_CLIENT_SECRET=abc123
 JWT_SECRET=secretkey
 
 RABBIT_API_USER=guest
-RABBIT_API_PASSWORD=guest
+RABBIT_API_PASSWORD=guest
+
+R2_ACCESS_KEY_ID=""
+R2_SECRET_ACCESS_KEY=""
+R2_ENDPOINT=""
+R2_BUCKET=""

open-api/openapi.json

@@ -1400,6 +1400,9 @@
             "nullable": true,
             "minimum": 0,
             "format": "uint32"
+          },
+          "splashSlug": {
+            "type": "string"
           }
         },
         "required": [
@@ -1412,7 +1415,8 @@
           "dayOneEnd",
           "contestEnd",
           "weekOneEnd",
-          "milestoneHash"
+          "milestoneHash",
+          "splashSlug"
         ],
         "additionalProperties": false,
         "description": "The definition of an activity in the RaidHub database.",
@@ -1426,7 +1430,8 @@
           "dayOneEnd": "2021-05-23T00:00:00.000Z",
           "contestEnd": "2021-05-23T00:00:00.000Z",
           "weekOneEnd": "2021-05-25T00:00:00.000Z",
-          "milestoneHash": 1888320892
+          "milestoneHash": 1888320892,
+          "splashSlug": "vog"
         }
       },
       "FeatDefinition": {
@@ -1474,6 +1479,46 @@
         ],
         "additionalProperties": false
       },
+      "ImageSize": {
+        "type": "string",
+        "enum": ["tiny", "small", "medium", "large", "xlarge", "full"],
+        "description": "The size of a RaidHub CDN hosted image.",
+        "example": "medium"
+      },
+      "ImageContentData": {
+        "type": "object",
+        "properties": {
+          "slug": {
+            "type": "string"
+          },
+          "size": {
+            "$ref": "#/components/schemas/ImageSize"
+          },
+          "fileName": {
+            "type": "string"
+          },
+          "fileFormat": {
+            "type": "string"
+          },
+          "path": {
+            "type": "string"
+          },
+          "url": {
+            "type": "string"
+          }
+        },
+        "required": ["slug", "size", "fileName", "fileFormat", "path", "url"],
+        "additionalProperties": false,
+        "description": "A URL to a piece of content hosted on the RaidHub CDN.",
+        "example": {
+          "slug": "vog",
+          "size": "medium",
+          "fileName": "medium.jpg",
+          "fileFormat": "jpg",
+          "path": "content/splash/vog/medium.jpg",
+          "url": "https://cdn.raidhub.io/content/splash/vog/medium.jpg"
+        }
+      },
       "VersionDefinition": {
         "type": "object",
         "properties": {
@@ -2294,6 +2339,16 @@
             "items": {
               "$ref": "#/components/schemas/FeatDefinition"
             }
+          },
+          "splashUrls": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/ImageContentData"
+              }
+            },
+            "description": "The mapping of each RaidHub activityId to its splash image URLs"
           }
         },
         "required": [
@@ -2310,7 +2365,8 @@
           "pantheonIds",
           "versionsForActivity",
           "rankingTiers",
-          "feats"
+          "feats",
+          "splashUrls"
         ],
         "additionalProperties": false
       },

package.json

@@ -50,6 +50,7 @@
     "typescript": "5.4"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.908.0",
     "@clickhouse/client": "^1.5.0",
     "amqplib": "^0.10.3",
     "bungie-net-core": "2.1.3",

src/integrations/r2/index.ts

@@ -0,0 +1,85 @@
+import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3"
+
+const R2_ENDPOINT = process.env.R2_ENDPOINT!
+const R2_ACCESS_KEY_ID = process.env.R2_ACCESS_KEY_ID!
+const R2_SECRET_ACCESS_KEY = process.env.R2_SECRET_ACCESS_KEY!
+const R2_BUCKET_NAME = process.env.R2_BUCKET!
+
+class BucketCache {
+    readonly ttl: number
+    private cache = new Map<string, string[]>()
+    private cacheTimer: ReturnType<typeof setTimeout> | null = null
+
+    constructor(ttl = 1000 * 60 * 5) {
+        this.ttl = ttl
+    }
+
+    has(cacheKey: string) {
+        return this.cache.has(cacheKey)
+    }
+
+    get(cacheKey: string) {
+        return this.cache.get(cacheKey)
+    }
+
+    set(cacheKey: string, content: string[]) {
+        this.cache.set(cacheKey, content)
+        this.queueCacheClear(cacheKey)
+    }
+
+    private queueCacheClear(cacheKey: string) {
+        if (this.cacheTimer) {
+            clearTimeout(this.cacheTimer)
+        }
+        this.cacheTimer = setTimeout(() => {
+            this.cache.delete(cacheKey)
+        }, this.ttl)
+    }
+}
+
+const cache = new BucketCache()
+
+export async function* streamR2BucketContents({ prefix = "", useCache = false } = {}) {
+    const s3 = new S3Client({
+        region: "auto", // Required for R2
+        endpoint: R2_ENDPOINT,
+        credentials: {
+            accessKeyId: R2_ACCESS_KEY_ID,
+            secretAccessKey: R2_SECRET_ACCESS_KEY
+        }
+    })
+
+    let continuationToken: string | undefined = undefined
+
+    const cacheKey = R2_BUCKET_NAME + "|" + prefix
+    if (useCache && cache.has(cacheKey)) {
+        const cached = cache.get(cacheKey)!
+        for (const item of cached) {
+            yield item
+        }
+        return
+    }
+
+    const allContents: string[] = []
+    do {
+        const command: ListObjectsV2Command = new ListObjectsV2Command({
+            Bucket: R2_BUCKET_NAME,
+            Prefix: prefix,
+            ContinuationToken: continuationToken
+        })
+
+        const response = await s3.send(command)
+        const objects = response.Contents || []
+
+        for (const obj of objects) {
+            if (obj.Key) {
+                allContents.push(obj.Key)
+                yield obj.Key
+            }
+        }
+
+        continuationToken = response.NextContinuationToken
+    } while (continuationToken)
+
+    cache.set(cacheKey, allContents)
+}

src/routes/clanStats.test.ts

@@ -1,4 +1,5 @@
-import { BungieApiError, bungiePlatformHttp } from "@/integrations/bungie"
+import * as bungie from "@/integrations/bungie"
+import { BungieApiError } from "@/integrations/bungie"
 import { clanQueue, playersQueue } from "@/integrations/rabbitmq/queues"
 import { expectErr, expectOk } from "@/lib/test-utils"
 import { ErrorCode } from "@/schema/errors/ErrorCode"
@@ -56,15 +57,15 @@ describe("clan 404", () => {
     })
 })
 
-test("clan 503", async () => {
-    const spyBungieFetch = spyOn(bungiePlatformHttp({ ttl: 30_000 }), "fetch")
+describe("clan 503", async () => {
+    const spyGetClan = spyOn(bungie, "getClan")
 
     afterAll(() => {
-        spyBungieFetch.mockRestore()
+        spyGetClan.mockRestore()
     })
 
     test("system disabled", async () => {
-        spyBungieFetch.mockRejectedValueOnce(
+        spyGetClan.mockRejectedValue(
             new BungieApiError({
                 cause: {
                     ErrorCode: PlatformErrorCodes.SystemDisabled,

src/routes/manifest.ts

@@ -2,6 +2,7 @@ import { RaidHubRoute } from "@/core/RaidHubRoute"
 import { cacheControl } from "@/middleware/cache-control"
 import { zActivityDefinition } from "@/schema/components/ActivityDefinition"
 import { zFeatDefinition } from "@/schema/components/FeatDefinition"
+import { zImageContentData } from "@/schema/components/ImageContentData"
 import { zVersionDefinition } from "@/schema/components/VersionDefinition"
 import { zNaturalNumber, zNumericalRecordKey } from "@/schema/util"
 import {
@@ -11,6 +12,7 @@ import {
     listVersionDefinitions
 } from "@/services/manifest/definitions"
 import { TierBreaks } from "@/services/manifest/tiers"
+import { generateSplashUrls } from "@/services/manifest/urls"
 import { z } from "zod"
 
 export const manifestRoute = new RaidHubRoute({
@@ -97,17 +99,25 @@ export const manifestRoute = new RaidHubRoute({
                             })
                         })
                     ),
-                    feats: z.array(zFeatDefinition)
+                    feats: z.array(zFeatDefinition),
+                    splashUrls: z
+                        .record(zNumericalRecordKey(), z.array(zImageContentData))
+                        .openapi({
+                            description:
+                                "The mapping of each RaidHub activityId to its splash image URLs"
+                        })
                 })
                 .strict()
         }
     },
     handler: async () => {
-        const [activities, versions, hashes, feats] = await Promise.all([
-            listActivityDefinitions(),
+        const activitiesPromise = listActivityDefinitions()
+        const [activities, versions, hashes, feats, splashUrls] = await Promise.all([
+            activitiesPromise,
             listVersionDefinitions(),
             listHashes(),
-            listFeatDefinitions()
+            listFeatDefinitions(),
+            activitiesPromise.then(generateSplashUrls)
         ])
         const raids = activities.filter(a => a.isRaid)
         const pantheonId = 101
@@ -159,7 +169,8 @@ export const manifestRoute = new RaidHubRoute({
             pantheonIds: [pantheonId],
             versionsForActivity: versionsForActivity,
             rankingTiers: TierBreaks,
-            feats: feats
+            feats: feats,
+            splashUrls: splashUrls
         })
     }
 })

src/schema/components/ActivityDefinition.ts

@@ -16,7 +16,8 @@ export const zActivityDefinition = registry.register(
             dayOneEnd: zISODateString({ nullable: true }),
             contestEnd: zISODateString({ nullable: true }),
             weekOneEnd: zISODateString({ nullable: true }),
-            milestoneHash: zUInt32().nullable()
+            milestoneHash: zUInt32().nullable(),
+            splashSlug: z.string()
         })
         .strict()
         .openapi({
@@ -31,7 +32,8 @@ export const zActivityDefinition = registry.register(
                 dayOneEnd: new Date("2021-05-23T00:00:00Z"),
                 contestEnd: new Date("2021-05-23T00:00:00Z"),
                 weekOneEnd: new Date("2021-05-25T00:00:00Z"),
-                milestoneHash: 1888320892
+                milestoneHash: 1888320892,
+                splashSlug: "vog"
             }
         })
 )

src/schema/components/ImageContentData.ts

@@ -0,0 +1,37 @@
+import { registry } from "@/schema/registry"
+import { z } from "zod"
+
+export type ImageSize = z.input<typeof zImageSize>
+export const zImageSize = registry.register(
+    "ImageSize",
+    z.enum(["tiny", "small", "medium", "large", "xlarge", "full"]).openapi({
+        description: "The size of a RaidHub CDN hosted image.",
+        example: "medium"
+    })
+)
+
+export type ImageContentData = z.input<typeof zImageContentData>
+export const zImageContentData = registry.register(
+    "ImageContentData",
+    z
+        .object({
+            slug: z.string(),
+            size: zImageSize,
+            fileName: z.string(),
+            fileFormat: z.string(),
+            path: z.string(),
+            url: z.string()
+        })
+        .strict()
+        .openapi({
+            description: "A URL to a piece of content hosted on the RaidHub CDN.",
+            example: {
+                slug: "vog",
+                size: "medium",
+                fileName: "medium.jpg",
+                fileFormat: "jpg",
+                path: "content/splash/vog/medium.jpg",
+                url: "https://cdn.raidhub.io/content/splash/vog/medium.jpg"
+            }
+        })
+)

src/services/manifest/definitions.ts

@@ -50,7 +50,8 @@ export const listActivityDefinitions = async () => {
             day_one_end AS "dayOneEnd",
             week_one_end AS "weekOneEnd",
             contest_end AS "contestEnd",
-            milestone_hash AS "milestoneHash"
+            milestone_hash AS "milestoneHash",
+            splash_path AS "splashSlug"
         FROM activity_definition`
     )
 }