From 2bbe61a183ef98c3336c9be200549dd1c1f25fe7 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Tue, 30 Jun 2026 21:27:37 +0100 Subject: [PATCH] feat: add S3-compatible storage provider --- .env.example | 9 + .../src/api/routes/media.controller.ts | 4 +- apps/frontend/src/app/(app)/layout.tsx | 10 +- apps/frontend/src/app/(extension)/layout.tsx | 10 +- apps/frontend/src/app/(provider)/layout.tsx | 10 +- .../src/components/media/new.uploader.tsx | 11 +- .../src/upload/r2.uploader.ts | 1 - .../nestjs-libraries/src/upload/s3.storage.ts | 130 +++++++++ .../src/upload/s3.uploader.ts | 260 ++++++++++++++++++ .../src/upload/upload.factory.ts | 10 + .../src/helpers/uppy.upload.ts | 1 + .../src/helpers/use.media.directory.ts | 36 ++- .../src/helpers/variable.context.tsx | 2 +- .../react-shared-libraries/tsconfig.json | 1 + 14 files changed, 482 insertions(+), 13 deletions(-) create mode 100644 libraries/nestjs-libraries/src/upload/s3.storage.ts create mode 100644 libraries/nestjs-libraries/src/upload/s3.uploader.ts diff --git a/.env.example b/.env.example index 4eba944b01..74b17da50b 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,15 @@ CLOUDFLARE_BUCKETNAME="your-bucket-name" CLOUDFLARE_BUCKET_URL="https://your-bucket-url.r2.cloudflarestorage.com/" CLOUDFLARE_REGION="auto" +## S3 / S3-compatible storage (AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2) +## Set STORAGE_PROVIDER=s3 to use this instead of Cloudflare R2. +# S3_ACCESS_KEY="" +# S3_SECRET_KEY="" +# S3_BUCKET="" +# S3_REGION="us-east-1" +# S3_BUCKET_URL="https://your-bucket.s3.amazonaws.com" +# S3_ENDPOINT= + # === Common optional Settings ## This is a dummy key, you must create your own from Resend. diff --git a/apps/backend/src/api/routes/media.controller.ts b/apps/backend/src/api/routes/media.controller.ts index 6d5b557a1f..aaa928f246 100644 --- a/apps/backend/src/api/routes/media.controller.ts +++ b/apps/backend/src/api/routes/media.controller.ts @@ -18,6 +18,7 @@ import { Organization } from '@prisma/client'; import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service'; import { ApiTags } from '@nestjs/swagger'; import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader'; +import handleS3Upload from '@gitroom/nestjs-libraries/upload/s3.uploader'; import { FileInterceptor } from '@nestjs/platform-express'; import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation'; import { SubscriptionService } from '@gitroom/nestjs-libraries/database/prisma/subscriptions/subscription.service'; @@ -158,7 +159,8 @@ export class MediaController { @Res() res: Response, @Param('endpoint') endpoint: string ) { - const upload = await handleR2Upload(endpoint, req, res); + const storageProvider = process.env.STORAGE_PROVIDER || 'local'; + const upload = await (storageProvider === 's3' ? handleS3Upload : handleR2Upload)(endpoint, req, res); if (endpoint !== 'complete-multipart-upload') { return upload; } diff --git a/apps/frontend/src/app/(app)/layout.tsx b/apps/frontend/src/app/(app)/layout.tsx index 3a46e8d6f9..b7db51121a 100644 --- a/apps/frontend/src/app/(app)/layout.tsx +++ b/apps/frontend/src/app/(app)/layout.tsx @@ -56,7 +56,7 @@ export default async function AppLayout({ children }: { children: ReactNode }) { > { setLocked(true); uppy2.setFileMeta(file.id, { - useCloudflare: storageProvider === 'cloudflare' ? 'true' : 'false', // Example of adding a custom field + useCloudflare: + storageProvider === 'cloudflare' || storageProvider === 's3' + ? 'true' + : 'false', // Example of adding a custom field addedOrder: fileOrderIndex++, // Track original order for sorting after upload // Add more fields as needed }); @@ -225,7 +228,11 @@ export function useUppyUploader(props: { if (transloadit.length > 0) { // @ts-ignore const allRes = result.transloadit[0].results; - const toSave = uniqBy<{ name: string; originalName: string; order: number }>( + const toSave = uniqBy<{ + name: string; + originalName: string; + order: number; + }>( // @ts-ignore Object.values(allRes).flatMap((p: any[]) => { return p.flatMap((item) => ({ diff --git a/libraries/nestjs-libraries/src/upload/r2.uploader.ts b/libraries/nestjs-libraries/src/upload/r2.uploader.ts index a0ba225001..d5c07860d1 100644 --- a/libraries/nestjs-libraries/src/upload/r2.uploader.ts +++ b/libraries/nestjs-libraries/src/upload/r2.uploader.ts @@ -270,7 +270,6 @@ export async function signPart(req: Request, res: Response) { Key: key, PartNumber: partNumber, UploadId: uploadId, - Expires: 3600, }; const command = new UploadPartCommand({ ...params }); diff --git a/libraries/nestjs-libraries/src/upload/s3.storage.ts b/libraries/nestjs-libraries/src/upload/s3.storage.ts new file mode 100644 index 0000000000..150772869c --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/s3.storage.ts @@ -0,0 +1,130 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import 'multer'; +import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; +import { IUploadProvider } from './upload.interface'; +import { isSafePublicHttpsUrl } from '@gitroom/nestjs-libraries/dtos/webhooks/webhook.url.validator'; +import { ssrfSafeDispatcher } from '@gitroom/nestjs-libraries/dtos/webhooks/ssrf.safe.dispatcher'; +import { parseDataUrl } from '@gitroom/nestjs-libraries/upload/data.url'; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { fromBuffer } = require('file-type'); + +const ALLOWED_MIME_TYPES = new Set([ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + 'image/avif', + 'image/bmp', + 'image/tiff', + 'video/mp4', + 'audio/mpeg', + 'audio/mp4', + 'audio/wav', + 'audio/ogg', +]); + +class S3Storage implements IUploadProvider { + private _client: S3Client; + + constructor( + accessKey: string, + secretKey: string, + private region: string, + private _bucketName: string, + private _uploadUrl: string, + endpoint?: string + ) { + this._client = new S3Client({ + region, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + ...(endpoint ? { endpoint, forcePathStyle: true } : {}), + }); + } + + async uploadSimple(path: string) { + const dataUrl = path.startsWith('data:') ? parseDataUrl(path) : null; + + let body: Buffer; + if (dataUrl) { + body = dataUrl.buffer; + } else { + if (!(await isSafePublicHttpsUrl(path))) { + throw new Error('Unsafe URL'); + } + const loadImage = await fetch(path, { + // @ts-ignore — undici option, not in lib.dom fetch types + dispatcher: ssrfSafeDispatcher, + }); + body = Buffer.from(await loadImage.arrayBuffer()); + } + const detected = await fromBuffer(body); + if (!detected || !ALLOWED_MIME_TYPES.has(detected.mime)) { + throw new Error('Unsupported file type.'); + } + const extension = detected.ext; + const safeContentType = detected.mime; + const id = makeId(10); + + const command = new PutObjectCommand({ + Bucket: this._bucketName, + Key: `${id}.${extension}`, + Body: body, + ContentType: safeContentType, + }); + await this._client.send(command); + + return `${this._uploadUrl}/${id}.${extension}`; + } + + async uploadFile(file: Express.Multer.File): Promise { + try { + const detected = await fromBuffer(file.buffer); + if (!detected || !ALLOWED_MIME_TYPES.has(detected.mime)) { + throw new Error('Unsupported file type.'); + } + const id = makeId(10); + const extension = detected.ext; + const safeContentType = detected.mime; + + const command = new PutObjectCommand({ + Bucket: this._bucketName, + ACL: 'public-read', + Key: `${id}.${extension}`, + Body: file.buffer, + ContentType: safeContentType, + }); + await this._client.send(command); + + return { + filename: `${id}.${extension}`, + mimetype: file.mimetype, + size: file.size, + buffer: file.buffer, + originalname: `${id}.${extension}`, + fieldname: 'file', + path: `${this._uploadUrl}/${id}.${extension}`, + destination: `${this._uploadUrl}/${id}.${extension}`, + encoding: '7bit', + stream: file.buffer as any, + }; + } catch (err) { + console.error('Error uploading file to S3:', err); + throw err; + } + } + + async removeFile(filePath: string): Promise { + const key = filePath.replace(`${this._uploadUrl}/`, ''); + const command = new DeleteObjectCommand({ + Bucket: this._bucketName, + Key: key, + }); + await this._client.send(command); + } +} + +export { S3Storage }; +export default S3Storage; diff --git a/libraries/nestjs-libraries/src/upload/s3.uploader.ts b/libraries/nestjs-libraries/src/upload/s3.uploader.ts new file mode 100644 index 0000000000..f4e4b56540 --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/s3.uploader.ts @@ -0,0 +1,260 @@ +import { + UploadPartCommand, + S3Client, + ListPartsCommand, + CreateMultipartUploadCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Request, Response } from 'express'; +import path from 'path'; +import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { fromBuffer } = require('file-type'); + +const ALLOWED_EXT_TO_MIME: Record = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.mp4': 'video/mp4', +}; + +function normalizeExtension(filename: string): string | null { + const ext = path.extname(filename || '').toLowerCase(); + return ALLOWED_EXT_TO_MIME[ext] ? ext : null; +} + +let _s3Client: S3Client | null = null; + +function getS3Client(): S3Client { + if (!_s3Client) { + _s3Client = new S3Client({ + region: process.env.S3_REGION || 'us-east-1', + credentials: { + accessKeyId: process.env.S3_ACCESS_KEY!, + secretAccessKey: process.env.S3_SECRET_KEY!, + }, + ...(process.env.S3_ENDPOINT + ? { endpoint: process.env.S3_ENDPOINT, forcePathStyle: true } + : {}), + }); + } + return _s3Client; +} + +function generateRandomString() { + return makeId(20); +} + +function getPublicUrl(key: string) { + return `${process.env.S3_BUCKET_URL?.replace(/\/+$/, '')}/${key}`; +} + +export default async function handleS3Upload( + endpoint: string, + req: Request, + res: Response +) { + switch (endpoint) { + case 'create-multipart-upload': + return createMultipartUpload(req, res); + case 'prepare-upload-parts': + return prepareUploadParts(req, res); + case 'complete-multipart-upload': + return completeMultipartUpload(req, res); + case 'list-parts': + return listParts(req, res); + case 'abort-multipart-upload': + return abortMultipartUpload(req, res); + case 'sign-part': + return signPart(req, res); + } + return res.status(404).end(); +} + +export async function simpleUpload( + data: Buffer, + originalFilename: string, + _contentType: string +) { + const detected = await fromBuffer(data); + if (!detected || !Object.values(ALLOWED_EXT_TO_MIME).includes(detected.mime)) { + throw new Error('Unsupported file type.'); + } + const fileExtension = `.${detected.ext}`; + const safeContentType = detected.mime; + const randomFilename = generateRandomString() + fileExtension; + + const command = new PutObjectCommand({ + Bucket: process.env.S3_BUCKET, + Key: randomFilename, + Body: data, + ContentType: safeContentType, + }); + + await getS3Client().send(command); + + return getPublicUrl(randomFilename); +} + +export async function createMultipartUpload(req: Request, res: Response) { + const { file, fileHash } = req.body; + const safeExt = normalizeExtension(file?.name || ''); + if (!safeExt) { + return res.status(400).json({ message: 'Unsupported file type.' }); + } + const safeContentType = ALLOWED_EXT_TO_MIME[safeExt]; + const randomFilename = generateRandomString() + safeExt; + + try { + const command = new CreateMultipartUploadCommand({ + Bucket: process.env.S3_BUCKET, + Key: randomFilename, + ContentType: safeContentType, + Metadata: { + 'x-amz-meta-file-hash': fileHash, + }, + }); + const response = await getS3Client().send(command); + return res.status(200).json({ + uploadId: response.UploadId, + key: response.Key, + }); + } catch (err) { + console.log('Error', err); + return res.status(500).json({ source: { status: 500 } }); + } +} + +export async function prepareUploadParts(req: Request, res: Response) { + const { partData } = req.body; + const parts = partData.parts; + const response = { presignedUrls: {} }; + + for (const part of parts) { + try { + const command = new UploadPartCommand({ + Bucket: process.env.S3_BUCKET, + Key: partData.key, + PartNumber: part.number, + UploadId: partData.uploadId, + }); + const url = await getSignedUrl(getS3Client(), command, { expiresIn: 3600 }); + // @ts-ignore + response.presignedUrls[part.number] = url; + } catch (err) { + console.log('Error', err); + return res.status(500).json(err); + } + } + + return res.status(200).json(response); +} + +export async function listParts(req: Request, res: Response) { + const { key, uploadId } = req.body; + + try { + const command = new ListPartsCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + UploadId: uploadId, + }); + const response = await getS3Client().send(command); + return res.status(200).json(response['Parts']); + } catch (err) { + console.log('Error', err); + return res.status(500).json(err); + } +} + +export async function completeMultipartUpload(req: Request, res: Response) { + const { key, uploadId, parts } = req.body; + + try { + const command = new CompleteMultipartUploadCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: parts }, + }); + const response = await getS3Client().send(command); + + const safeExt = normalizeExtension(key || ''); + if (!safeExt) { + await getS3Client().send(new DeleteObjectCommand({ Bucket: process.env.S3_BUCKET, Key: key })); + return res.status(400).json({ message: 'Unsupported file type.' }); + } + const expectedMime = ALLOWED_EXT_TO_MIME[safeExt]; + + const head = await getS3Client().send( + new GetObjectCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + Range: 'bytes=0-4100', + }) + ); + const chunks: Buffer[] = []; + // @ts-ignore + for await (const chunk of head.Body as AsyncIterable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const prefix = Buffer.concat(chunks); + const detected = await fromBuffer(prefix); + + if (!detected || detected.mime !== expectedMime) { + await getS3Client().send(new DeleteObjectCommand({ Bucket: process.env.S3_BUCKET, Key: key })); + return res + .status(400) + .json({ message: 'File contents do not match declared type.' }); + } + + response.Location = getPublicUrl(key); + return response; + } catch (err) { + console.log('Error', err); + return res.status(500).json(err); + } +} + +export async function abortMultipartUpload(req: Request, res: Response) { + const { key, uploadId } = req.body; + + try { + const command = new AbortMultipartUploadCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + UploadId: uploadId, + }); + const response = await getS3Client().send(command); + return res.status(200).json(response); + } catch (err) { + console.log('Error', err); + return res.status(500).json(err); + } +} + +export async function signPart(req: Request, res: Response) { + const { key, uploadId } = req.body; + const partNumber = parseInt(req.body.partNumber); + + const command = new UploadPartCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + PartNumber: partNumber, + UploadId: uploadId, + }); + const url = await getSignedUrl(getS3Client(), command, { expiresIn: 3600 }); + + return res.status(200).json({ url }); +} diff --git a/libraries/nestjs-libraries/src/upload/upload.factory.ts b/libraries/nestjs-libraries/src/upload/upload.factory.ts index f89d310a8c..e689746677 100644 --- a/libraries/nestjs-libraries/src/upload/upload.factory.ts +++ b/libraries/nestjs-libraries/src/upload/upload.factory.ts @@ -1,6 +1,7 @@ import { CloudflareStorage } from './cloudflare.storage'; import { IUploadProvider } from './upload.interface'; import { LocalStorage } from './local.storage'; +import { S3Storage } from './s3.storage'; export class UploadFactory { static createStorage(): IUploadProvider { @@ -18,6 +19,15 @@ export class UploadFactory { process.env.CLOUDFLARE_BUCKETNAME!, process.env.CLOUDFLARE_BUCKET_URL! ); + case 's3': + return new S3Storage( + process.env.S3_ACCESS_KEY!, + process.env.S3_SECRET_KEY!, + process.env.S3_REGION!, + process.env.S3_BUCKET!, + process.env.S3_BUCKET_URL!, + process.env.S3_ENDPOINT + ); default: throw new Error(`Invalid storage type ${storageProvider}`); } diff --git a/libraries/react-shared-libraries/src/helpers/uppy.upload.ts b/libraries/react-shared-libraries/src/helpers/uppy.upload.ts index 12af1cd579..7271c959c6 100644 --- a/libraries/react-shared-libraries/src/helpers/uppy.upload.ts +++ b/libraries/react-shared-libraries/src/helpers/uppy.upload.ts @@ -40,6 +40,7 @@ export const getUppyUploadPlugin = ( }, }, }; + case 's3': case 'cloudflare': return { plugin: AwsS3Multipart, diff --git a/libraries/react-shared-libraries/src/helpers/use.media.directory.ts b/libraries/react-shared-libraries/src/helpers/use.media.directory.ts index fdddb62026..f8b3a867f2 100644 --- a/libraries/react-shared-libraries/src/helpers/use.media.directory.ts +++ b/libraries/react-shared-libraries/src/helpers/use.media.directory.ts @@ -1,8 +1,40 @@ import { useCallback } from 'react'; +import { useVariables } from '@gitroom/react/helpers/variable.context'; + +const isKnownStorageHost = (hostname: string, storageHostname: string) => { + return ( + hostname === storageHostname || + hostname.endsWith('.r2.cloudflarestorage.com') || + /(^|\.)s3([.-][a-z0-9-]+)?\.amazonaws\.com$/.test(hostname) + ); +}; + export const useMediaDirectory = () => { + const { storageProvider, cloudflareUrl } = useVariables(); + const set = useCallback((path: string) => { - return path; - }, []); + if ( + !path || + (storageProvider !== 'cloudflare' && storageProvider !== 's3') || + !cloudflareUrl + ) { + return path; + } + + try { + const currentUrl = new URL(path); + const storageUrl = new URL(cloudflareUrl); + if (!isKnownStorageHost(currentUrl.hostname, storageUrl.hostname)) { + return path; + } + currentUrl.protocol = storageUrl.protocol; + currentUrl.host = storageUrl.host; + return currentUrl.toString(); + } catch { + return path; + } + }, [cloudflareUrl, storageProvider]); + return { set, }; diff --git a/libraries/react-shared-libraries/src/helpers/variable.context.tsx b/libraries/react-shared-libraries/src/helpers/variable.context.tsx index 3243cf245c..b1d4df6ae2 100644 --- a/libraries/react-shared-libraries/src/helpers/variable.context.tsx +++ b/libraries/react-shared-libraries/src/helpers/variable.context.tsx @@ -14,7 +14,7 @@ interface VariableContextInterface { mainUrl: string; frontEndUrl: string; plontoKey: string; - storageProvider: 'local' | 'cloudflare'; + storageProvider: 'local' | 'cloudflare' | 's3'; backendUrl: string; environment: string; discordUrl: string; diff --git a/libraries/react-shared-libraries/tsconfig.json b/libraries/react-shared-libraries/tsconfig.json index ef0c1df9d0..9f17c5fde8 100644 --- a/libraries/react-shared-libraries/tsconfig.json +++ b/libraries/react-shared-libraries/tsconfig.json @@ -3,6 +3,7 @@ "compilerOptions": { "forceConsistentCasingInFileNames": true, "esModuleInterop": true, + "jsx": "react-jsx", "allowSyntheticDefaultImports": true, "strict": true, "noImplicitOverride": true,