diff --git a/api/main_endpoints/routes/Printer.js b/api/main_endpoints/routes/Printer.js index 5e9eca40e..24267d02a 100644 --- a/api/main_endpoints/routes/Printer.js +++ b/api/main_endpoints/routes/Printer.js @@ -5,6 +5,8 @@ const FormData = require('form-data'); const logger = require('../../util/logger'); const fs = require('fs'); const path = require('path'); +const { MetricsHandler, register } = require('../../util/metrics.js'); +const { cleanUpChunks, cleanUpExpiredChunks, recordPrintingFolderSize } = require('../util/Printer.js'); const { decodeToken, @@ -19,7 +21,6 @@ const { const { PRINTING = {} } = require('../../config/config.json'); -const { MetricsHandler } = require('../../util/metrics'); // see https://github.com/SCE-Development/Quasar/tree/dev/docker-compose.dev.yml#L11 let PRINTER_URL = process.env.PRINTER_URL @@ -40,6 +41,16 @@ const storage = multer.diskStorage({ const upload = multer({ storage }); +const FIVE_MINUTES_MS = 300000; + +if (PRINTING.ENABLED) { + setInterval(() => { + const dir = path.join(__dirname, 'printing'); + cleanUpExpiredChunks(dir, FIVE_MINUTES_MS); + recordPrintingFolderSize(dir); + }, FIVE_MINUTES_MS); +} + router.get('/healthCheck', async (req, res) => { /* * How these work with Quasar: @@ -61,7 +72,7 @@ router.get('/healthCheck', async (req, res) => { }); }); -router.post('/sendPrintRequest', upload.single('file'), async (req, res) => { +router.post('/sendPrintRequest', upload.single('chunk'), async (req, res) => { if (!checkIfTokenSent(req)) { logger.warn('/sendPrintRequest was requested without a token'); return res.sendStatus(UNAUTHORIZED); @@ -74,31 +85,58 @@ router.post('/sendPrintRequest', upload.single('file'), async (req, res) => { logger.warn('Printing is disabled, returning 200 to mock the printing server'); return res.sendStatus(OK); } - const { copies, sides } = req.body; - const file = req.file; + + const dir = path.join(__dirname, 'printing'); + const { totalChunks, chunkIdx } = req.body; + + // reassemble pdf on last chunk received + if (Number(chunkIdx) < totalChunks - 1) { + return res.sendStatus(OK); + } + + const { copies, sides, id } = req.body; + + const chunks = await fs.promises.readdir(dir); + const assembledPdfFromChunks = path.join(dir, id + '.pdf'); + + for (let chunk of chunks) { + if (path.extname(chunk) !== '.CHUNK') continue; + if (!path.basename(chunk).includes(id)) continue; + + try { + const chunkData = await fs.promises.readFile(path.join(dir, chunk)); + fs.appendFileSync(assembledPdfFromChunks, chunkData); + } catch (err) { + logger.error('/sendPrintRequest encountered an error while assembling pdf: ' + err); + await cleanUpChunks(dir, id); + return res.sendStatus(SERVER_ERROR); + } + } + + const stream = await fs.createReadStream(assembledPdfFromChunks); const data = new FormData(); - data.append('file', fs.createReadStream(file.path), { filename: file.originalname }); + data.append('file', stream, {filename: id, type: 'application/pdf'}); data.append('copies', copies); data.append('sides', sides); - axios.post(PRINTER_URL + '/print', - data, - { + + try { + // full pdf can be sent to quasar no problem + await axios.post(PRINTER_URL + '/print', data, { headers: { ...data.getHeaders(), - } - }) - .then(() => { - // delete file from temp folder after printing - fs.unlink(file.path, (err) => { - if (err) { - logger.error(`Unable to delete file at path ${file.path}:`, err); - } - }); - res.sendStatus(OK); - }).catch((err) => { - logger.error('/sendPrintRequest had an error: ', err); - res.sendStatus(SERVER_ERROR); + }, + maxContentLength: 1024 * 1024 * 150, // 150 mb + maxBodyLength: Infinity }); + + await cleanUpChunks(dir, id); + res.sendStatus(OK); + } catch (err) { + logger.error('/sendPrintRequest had an error: ', err); + + await cleanUpChunks(dir, id); + res.sendStatus(SERVER_ERROR); + } }); module.exports = router; diff --git a/api/main_endpoints/util/Printer.js b/api/main_endpoints/util/Printer.js new file mode 100644 index 000000000..0da5e2219 --- /dev/null +++ b/api/main_endpoints/util/Printer.js @@ -0,0 +1,77 @@ +const fs = require('fs'); +const path = require('path'); +const logger = require('../../util/logger'); +const { MetricsHandler } = require('../../util/metrics.js'); + +/** + * Deletes all chunks with the specified id from a directory + * @param {String} dir The directory the chunks are stored in + * @param {String} id The UUID (36 char) assigned to the chunks + */ +async function cleanUpChunks(dir, id) { + try { + const chunks = await fs.promises.readdir(dir); + + for (let chunk of chunks) { + if (!path.basename(chunk).includes(id)) continue; + + await fs.promises.unlink(path.join(dir, chunk), err => { + logger.error(`Failed to delete chunk with id ${id} in ${dir}: ` + err); + }); + } + } catch (err) { + logger.error('Encountered error while trying to delete chunks: ' + err); + return false; + } + + return true; +} + +/** + * Deletes all chunks of specified age or older, 'expired', from a directory + * @param {String} dir The directory the chunks are stored in + * @param {Number} expiry Minimum age (ms) to be considered expired + */ +async function cleanUpExpiredChunks(dir, expiry) { + try { + const chunks = await fs.promises.readdir(dir); + + for (let chunk of chunks) { + if (!['.pdf', '.CHUNK'].includes(path.extname(chunk))) continue; + + const stats = await fs.promises.stat(path.join(dir, chunk)); + const age = Date.now() - stats.mtimeMs; + + if (age >= expiry) { + await fs.promises.unlink(path.join(dir, chunk), err => { + logger.warn(`Failed to delete expired chunk in ${dir}: ` + err); + }); + + MetricsHandler.totalExpiredBytesDeleted.inc(stats.size); + MetricsHandler.totalExpiredChunksDeleted.inc(1); + } + } + } catch (err) { + logger.error(`Encountered error while trying to delete expired chunks in ${dir}: ` + err); + return false; + } + + return true; +} + +/** + * Gets size of printing folder and records it in the prometheus metric + * @param {String} dir The directory of the printing folder + */ +async function recordPrintingFolderSize(dir) { + const files = await fs.promises.readdir(dir); + let sizeOfDir = 0; + + for (file of files) { + sizeOfDir += await fs.promises.stat(path.join(dir, file)).then(stat => stat.size); + } + + MetricsHandler.currentSizeOfPrintingFolderBytes.set(sizeOfDir); +} + +module.exports = { cleanUpChunks, cleanUpExpiredChunks, recordPrintingFolderSize }; diff --git a/api/util/metrics.js b/api/util/metrics.js index f992efa70..039d1d9aa 100644 --- a/api/util/metrics.js +++ b/api/util/metrics.js @@ -25,33 +25,48 @@ class MetricsHandler { labelNames: ['type'], }); - totalMessagesSent = new client.Counter({ - name: 'total_messages_sent', - help: 'Total number of messages sent' - }); - - currentConnectionsOpen = new client.Gauge({ - name: 'current_connections_open', - help: 'Total number of connections open', - labelNames: ['id'] - }); - - totalChatMessagesPerChatRoom = new client.Counter({ - name: 'total_chat_messages_per_chatroom', - help: 'Total number of messages sent per chatroom', - labelNames: ['id'] - }); - - constructor() { - register.setDefaultLabels({ - app: 'sce-core', - }); - client.collectDefaultMetrics({ register }); - - Object.keys(this).forEach(metric => { - register.registerMetric(this[metric]); - }); - } + totalMessagesSent = new client.Counter({ + name: 'total_messages_sent', + help: 'Total number of messages sent' + }); + + currentConnectionsOpen = new client.Gauge({ + name: 'current_connections_open', + help: 'Total number of connections open', + labelNames: ['id'] + }); + + totalChatMessagesPerChatRoom = new client.Counter({ + name: 'total_chat_messages_per_chatroom', + help: 'Total number of messages sent per chatroom', + labelNames: ['id'] + }); + + currentSizeOfPrintingFolderBytes = new client.Gauge({ + name: 'current_size_of_printing_folder_bytes', + help: 'Current size of printing folder in bytes' + }); + + totalExpiredChunksDeleted = new client.Counter({ + name: 'total_expired_chunks_deleted', + help: 'Total number of expired chunks that have been deleted' + }); + + totalExpiredBytesDeleted = new client.Counter({ + name: 'total_expired_bytes_deleted', + help: 'Total number of bytes from expired chunks that have been deleted' + }); + + constructor() { + register.setDefaultLabels({ + app: 'sce-core', + }); + client.collectDefaultMetrics({ register }); + + Object.keys(this).forEach(metric => { + register.registerMetric(this[metric]); + }); + } } module.exports = { diff --git a/src/APIFunctions/2DPrinting.js b/src/APIFunctions/2DPrinting.js index ace81e073..05c0b2d8f 100644 --- a/src/APIFunctions/2DPrinting.js +++ b/src/APIFunctions/2DPrinting.js @@ -23,7 +23,11 @@ export async function healthCheck() { const url = new URL('/api/Printer/healthCheck', BASE_API_URL); try { const res = await fetch(url.href); - status.error = !res.ok; + if (res.ok) { + status.responseData = await res.json(); + } else { + status.error = true; + } } catch (err) { status.responseData = err; status.error = true; @@ -76,21 +80,45 @@ export function parseRange(pages, maxPages) { export async function printPage(data, token) { let status = new ApiResponse(); const url = new URL('/api/Printer/sendPrintRequest', BASE_API_URL); - try { - const res = await fetch(url.href, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${token}` - }, - body: data - }); - if (res.ok) { - status.responseData = true; + + const pdf = data.get('file'); + const sides = data.get('sides'); + const copies = data.get('copies'); + const id = crypto.randomUUID(); + const CHUNK_SIZE = 1024 * 1024 * 0.5; // 0.5 MB ------- SENT DATA **CANNOT** EXCEED 1 MB + const totalChunks = Math.ceil(pdf.size / CHUNK_SIZE); + + for (let i = 0; i < totalChunks; i++) { + let chunkData = new FormData(); + let chunkStart = i * CHUNK_SIZE; + let chunk = pdf.slice(chunkStart, chunkStart + CHUNK_SIZE); + chunkData.append('chunk', chunk, id + '_' + i + '.CHUNK'); + chunkData.append('totalChunks', totalChunks); + chunkData.append('chunkIdx', i); + + if (i === totalChunks - 1) { + chunkData.append('id', id); + chunkData.append('sides', sides); + chunkData.append('copies', copies); + } + + try { + const res = await fetch(url.href, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}` + }, + body: chunkData + }); + + status.responseData = !!res.ok; + } catch (err) { + status.responseData = err; + status.error = true; + return status; } - } catch (err) { - status.responseData = err; - status.error = true; } + return status; } diff --git a/src/Pages/2DPrinting/2DPrinting.js b/src/Pages/2DPrinting/2DPrinting.js index f1d7b1e64..7133774b3 100644 --- a/src/Pages/2DPrinting/2DPrinting.js +++ b/src/Pages/2DPrinting/2DPrinting.js @@ -6,15 +6,13 @@ import { getPagesPrinted, } from '../../APIFunctions/2DPrinting'; import { editUser } from '../../APIFunctions/User'; -import { useUser } from '../../Components/context/UserContext'; import { PDFDocument } from 'pdf-lib'; import { healthCheck } from '../../APIFunctions/2DPrinting'; import ConfirmationModal from '../../Components/DecisionModal/ConfirmationModal.js'; -export default function Printing() { - const { user, setUser } = useUser(); +export default function Printing(props) { const [dragActive, setDragActive] = useState(false); const [confirmModal, setConfirmModal] = useState(false); const [numberOfPagesInPdfPreview, setNumberOfPagesInPdfPreview] = useState(0); @@ -42,8 +40,8 @@ export default function Printing() { async function getNumberOfPagesPrintedSoFar() { const result = await getPagesPrinted( - user.email, - user.token, + props.user.email, + props.user.token, ); setPrinterHealthy(!result.error); if (!result.error) { @@ -52,11 +50,9 @@ export default function Printing() { } useEffect(() => { - if (user && user.email && user.token) { - checkPrinterHealth(); - getNumberOfPagesPrintedSoFar(); - } - }, [user]); + checkPrinterHealth(); + getNumberOfPagesPrintedSoFar(); + }, []); const INPUT_CLASS_NAME = 'indent-2 block rounded-md border-0 py-1.5 shadow-sm ring-1 ring-inset ring-gray-300 placeholder: focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6'; @@ -189,18 +185,27 @@ export default function Printing() { } async function handlePrinting() { + if (PdfFile.size > 1024 * 1024 * 150) { + setPrintStatus('File exceeds 150 MB size limit'); + setPrintStatusColor('error'); + return; + } + // send print request with files and configuratiosn in formData const data = new FormData(); + data.append('file', PdfFile); data.append('sides', sides); data.append('copies', copies); - let status = await printPage(data, user.token); + let status = await printPage(data, props.user.token); if (!status.error) { + editUser( + { ...props.user, pagesPrinted: pagesPrinted + pagesToBeUsedInPrintRequest }, + props.user.token, + ); setPrintStatus('Printing succeeded!'); setPrintStatusColor('success'); - const updatedUser = { ...user, pagesPrinted: pagesPrinted + pagesToBeUsedInPrintRequest }; - editUser(updatedUser, user.token); } else { setPrintStatus('Printing failed. Please try again or reach out to SCE Dev team if the issue persists.'); setPrintStatusColor('error'); @@ -391,7 +396,7 @@ export default function Printing() {

Click to upload or drag and drop

-

PDF only, max 10MB

+

PDF only, max 150MB

{ + before(done => { + initializeTokenMock(); + + app = tools.initializeServer([ + __dirname + '/../../api/main_endpoints/routes/Printer.js', + ]); + test = new SceApiTester(app); + done(); + }); + + after(done => { + restoreTokenMock(); + sandbox.restore(); + tools.terminateServer(done); + }); + + beforeEach(() => { + setTokenStatus(false); + }); + + afterEach(() => { + resetTokenMock(); + }); + + describe('cleanUpExpiredChunks', () => { + const CHUNK_DIRECTORY = __dirname + '/../../api/main_endpoints/routes/printing'; + const MY_BIRTH_DATE = new Date('December 4, 2005 07:53:00'); + + it('Should delete expired chunks (5 minutes or older)', async () => { + const dirBefore = await fs.promises.readdir(CHUNK_DIRECTORY); + const numFilesBefore = dirBefore.length; + + // 5 minutes old + const firstExpiredChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'expired1.CHUNK', new Date(Date.now() - 300000)); + + // 10 minutes old + const secondExpiredChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'expired2.CHUNK', new Date(Date.now() - 600000)); + + // nearly two decades old + const thirdExpiredChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'expired3.CHUNK', MY_BIRTH_DATE); + + await printerUtil.cleanUpExpiredChunks(CHUNK_DIRECTORY, 300000); + const dirAfter = await fs.promises.readdir(CHUNK_DIRECTORY); + expect(dirAfter.length).to.equal(numFilesBefore); + }); + + it('Should not delete fresh chunks (less than 5 minutes old)', async () => { + const dirBefore = await fs.promises.readdir(CHUNK_DIRECTORY); + const numFilesBefore = dirBefore.length; + + // 4.90 minutes old + const firstFreshChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'fresh1.CHUNK', new Date(Date.now() - 294000)); + + // 1 minute old + const secondFreshChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'fresh2.CHUNK', new Date(Date.now() - 60000)); + + // 20 minutes from the future + const thirdFreshChunk = tools.createFakeChunk(CHUNK_DIRECTORY, 'fresh3.CHUNK', new Date(Date.now() + 1200000)); + + await printerUtil.cleanUpExpiredChunks(CHUNK_DIRECTORY, 300000); + const dirAfter = await fs.promises.readdir(CHUNK_DIRECTORY); + expect(dirAfter.length).to.equal(numFilesBefore + 3); + + await fs.promises.unlink(firstFreshChunk); + await fs.promises.unlink(secondFreshChunk); + await fs.promises.unlink(thirdFreshChunk); + }); + + it ('Should return false if a bad directory is given', async () => { + const response = await printerUtil.cleanUpExpiredChunks('haha this directory doesnt do not exist', 1); + expect(response).to.equal(false); + }); + }); + + describe('/POST sendPrintRequest', () => { + const id = crypto.randomUUID(); + const CHUNK_SIZE = 1024 * 1024 * 0.5; // 0.5 MB + const FAKE_PDF = new File([new Uint32Array(1024 * 1024)], 'real_pdf'); // 16 MB + const TOTAL_CHUNKS = Math.ceil(FAKE_PDF.size / CHUNK_SIZE); + + const DUMMY_CHUNK = new FormData(); + + it('Should return 400 when token is not sent', async () => { + const result = await test.sendPostRequest('/api/Printer/sendPrintRequest', { DUMMY_CHUNK }); + expect(result).to.have.status(UNAUTHORIZED); + }); + + it('Should return 400 when invalid token is sent', async () => { + const result = await test.sendPostRequestWithToken(token, '/api/Printer/sendPrintRequest', { DUMMY_CHUNK }); + expect(result).to.have.status(UNAUTHORIZED); + }); + + it(`Should successfully process all ${TOTAL_CHUNKS} chunks sent (with valid token)`, async () => { + let chunksProcessed = 0; + setTokenStatus(true); + + for (let i = 0; i < TOTAL_CHUNKS; i++) { + let chunkStart = i * CHUNK_SIZE; + let chunk = FAKE_PDF.slice(chunkStart, chunkStart + CHUNK_SIZE); + const arrayBuffer = await chunk.arrayBuffer(); + + const result = await chai + .request(app) + .post('/api/Printer/sendPrintRequest') + .set('Authorization', `Bearer ${token}`) + .type('form') + .field('totalChunks', TOTAL_CHUNKS) + .field('chunkIdx', i) + .field('sides', 'one-sided') + .field('copies', 1) + .field('id', id) + .attach('chunk', Buffer.from(arrayBuffer), id + '_' + i + '.CHUNK'); + + if (result.status === OK) { + chunksProcessed++; + } + } + + expect(chunksProcessed).to.equal(TOTAL_CHUNKS); + }); + }); +}); diff --git a/test/util/tools/tools.js b/test/util/tools/tools.js index ae51c2231..92deeab11 100644 --- a/test/util/tools/tools.js +++ b/test/util/tools/tools.js @@ -1,4 +1,6 @@ const { SceHttpServer } = require('../../../api/util/SceHttpServer'); +const fs = require('fs'); +const path = require('path'); let serverInstance = null; @@ -29,10 +31,18 @@ function terminateServer(done) { serverInstance.closeConnection(done); } +function createFakeChunk(dir, name, mtime) { + const filePath = path.join(dir, name); + fs.writeFileSync(filePath, ''); + fs.utimesSync(filePath, 0, mtime, () => {}); + return filePath; +} + // Exporting functions module.exports = { emptySchema, insertItem, initializeServer, - terminateServer + terminateServer, + createFakeChunk };