send PDFs to backend in chunks

api/main_endpoints/routes/Printer.js

@@ -5,6 +5,8 @@ const FormData = require('form-data');
 const logger = require('../../util/logger');
 const fs = require('fs');
 const path = require('path');
+const { MetricsHandler, register } = require('../../util/metrics.js');
+const { cleanUpChunks, cleanUpExpiredChunks, recordPrintingFolderSize } = require('../util/Printer.js');
 
 const {
   decodeToken,
@@ -19,7 +21,6 @@ const {
 const {
   PRINTING = {}
 } = require('../../config/config.json');
-const { MetricsHandler } = require('../../util/metrics');
 
 // see https://github.com/SCE-Development/Quasar/tree/dev/docker-compose.dev.yml#L11
 let PRINTER_URL = process.env.PRINTER_URL
@@ -40,6 +41,16 @@ const storage = multer.diskStorage({
 
 const upload = multer({ storage });
 
+const FIVE_MINUTES_MS = 300000;
+
+if (PRINTING.ENABLED) {
+  setInterval(() => {
+    const dir = path.join(__dirname, 'printing');
+    cleanUpExpiredChunks(dir, FIVE_MINUTES_MS);
+    recordPrintingFolderSize(dir);
+  }, FIVE_MINUTES_MS);
+}
+
 router.get('/healthCheck', async (req, res) => {
   /*
    * How these work with Quasar:
@@ -61,7 +72,7 @@ router.get('/healthCheck', async (req, res) => {
     });
 });
 
-router.post('/sendPrintRequest', upload.single('file'), async (req, res) => {
+router.post('/sendPrintRequest', upload.single('chunk'), async (req, res) => {
   if (!checkIfTokenSent(req)) {
     logger.warn('/sendPrintRequest was requested without a token');
     return res.sendStatus(UNAUTHORIZED);
@@ -74,31 +85,58 @@ router.post('/sendPrintRequest', upload.single('file'), async (req, res) => {
     logger.warn('Printing is disabled, returning 200 to mock the printing server');
     return res.sendStatus(OK);
   }
-  const { copies, sides } = req.body;
-  const file = req.file;
+
+  const dir = path.join(__dirname, 'printing');
+  const { totalChunks, chunkIdx } = req.body;
+
+  // reassemble pdf on last chunk received
+  if (Number(chunkIdx) < totalChunks - 1) {
+    return res.sendStatus(OK);
+  }
+
+  const { copies, sides, id } = req.body;
+
+  const chunks = await fs.promises.readdir(dir);
+  const assembledPdfFromChunks = path.join(dir, id + '.pdf');
+
+  for (let chunk of chunks) {
+    if (path.extname(chunk) !== '.CHUNK') continue;
+    if (!path.basename(chunk).includes(id)) continue;
+
+    try {
+      const chunkData = await fs.promises.readFile(path.join(dir, chunk));
+      fs.appendFileSync(assembledPdfFromChunks, chunkData);
+    } catch (err) {
+      logger.error('/sendPrintRequest encountered an error while assembling pdf: ' + err);
+      await cleanUpChunks(dir, id);
+      return res.sendStatus(SERVER_ERROR);
+    }
+  }
+
+  const stream = await fs.createReadStream(assembledPdfFromChunks);
   const data = new FormData();
-  data.append('file', fs.createReadStream(file.path), { filename: file.originalname });
+  data.append('file', stream, {filename: id, type: 'application/pdf'});
   data.append('copies', copies);
   data.append('sides', sides);
-  axios.post(PRINTER_URL + '/print',
-    data,
-    {
+
+  try {
+    // full pdf can be sent to quasar no problem
+    await axios.post(PRINTER_URL + '/print', data, {
       headers: {
         ...data.getHeaders(),
-      }
-    })
-    .then(() => {
-      // delete file from temp folder after printing
-      fs.unlink(file.path, (err) => {
-        if (err) {
-          logger.error(`Unable to delete file at path ${file.path}:`, err);
-        }
-      });
-      res.sendStatus(OK);
-    }).catch((err) => {
-      logger.error('/sendPrintRequest had an error: ', err);
-      res.sendStatus(SERVER_ERROR);
+      },
+      maxContentLength: 1024 * 1024 * 150, // 150 mb
+      maxBodyLength: Infinity
     });
+
+    await cleanUpChunks(dir, id);
+    res.sendStatus(OK);
+  } catch (err) {
+    logger.error('/sendPrintRequest had an error: ', err);
+
+    await cleanUpChunks(dir, id);
+    res.sendStatus(SERVER_ERROR);
+  }
 });
 
 module.exports = router;

api/main_endpoints/util/Printer.js

@@ -0,0 +1,77 @@
+const fs = require('fs');
+const path = require('path');
+const logger = require('../../util/logger');
+const { MetricsHandler } = require('../../util/metrics.js');
+
+/**
+ * Deletes all chunks with the specified id from a directory
+ * @param {String} dir The directory the chunks are stored in
+ * @param {String} id  The UUID (36 char) assigned to the chunks
+ */
+async function cleanUpChunks(dir, id) {
+  try {
+    const chunks = await fs.promises.readdir(dir);
+
+    for (let chunk of chunks) {
+      if (!path.basename(chunk).includes(id)) continue;
+
+      await fs.promises.unlink(path.join(dir, chunk), err => {
+        logger.error(`Failed to delete chunk with id ${id} in ${dir}: ` + err);
+      });
+    }
+  } catch (err) {
+    logger.error('Encountered error while trying to delete chunks: ' + err);
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Deletes all chunks of specified age or older, 'expired', from a directory
+ * @param {String} dir The directory the chunks are stored in
+ * @param {Number} expiry Minimum age (ms) to be considered expired
+ */
+async function cleanUpExpiredChunks(dir, expiry) {
+  try {
+    const chunks = await fs.promises.readdir(dir);
+
+    for (let chunk of chunks) {
+      if (!['.pdf', '.CHUNK'].includes(path.extname(chunk))) continue;
+
+      const stats = await fs.promises.stat(path.join(dir, chunk));
+      const age = Date.now() - stats.mtimeMs;
+
+      if (age >= expiry) {
+        await fs.promises.unlink(path.join(dir, chunk), err => {
+          logger.warn(`Failed to delete expired chunk in ${dir}: ` + err);
+        });
+
+        MetricsHandler.totalExpiredBytesDeleted.inc(stats.size);
+        MetricsHandler.totalExpiredChunksDeleted.inc(1);
+      }
+    }
+  } catch (err) {
+    logger.error(`Encountered error while trying to delete expired chunks in ${dir}: `  + err);
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Gets size of printing folder and records it in the prometheus metric
+ * @param {String} dir The directory of the printing folder
+ */
+async function recordPrintingFolderSize(dir) {
+  const files = await fs.promises.readdir(dir);
+  let sizeOfDir = 0;
+
+  for (file of files) {
+    sizeOfDir += await fs.promises.stat(path.join(dir, file)).then(stat => stat.size);
+  }
+
+  MetricsHandler.currentSizeOfPrintingFolderBytes.set(sizeOfDir);
+}
+
+module.exports = { cleanUpChunks, cleanUpExpiredChunks, recordPrintingFolderSize };

api/util/metrics.js

@@ -25,33 +25,48 @@ class MetricsHandler {
 	  labelNames: ['type'],
 	});
 
-	  totalMessagesSent = new client.Counter({
-	    name: 'total_messages_sent',
-	    help: 'Total number of messages sent'
-	  });
-
-	  currentConnectionsOpen = new client.Gauge({
-	    name: 'current_connections_open',
-	    help: 'Total number of connections open',
-	    labelNames: ['id']
-	  });
-
-	  totalChatMessagesPerChatRoom = new client.Counter({
-	    name: 'total_chat_messages_per_chatroom',
-	    help: 'Total number of messages sent per chatroom',
-	    labelNames: ['id']
-	  });
-
-	  constructor() {
-	  register.setDefaultLabels({
-	    app: 'sce-core',
-	  });
-	  client.collectDefaultMetrics({ register });
-
-	  Object.keys(this).forEach(metric => {
-	    register.registerMetric(this[metric]);
-	  });
-	  }
+  totalMessagesSent = new client.Counter({
+    name: 'total_messages_sent',
+    help: 'Total number of messages sent'
+  });
+
+  currentConnectionsOpen = new client.Gauge({
+    name: 'current_connections_open',
+    help: 'Total number of connections open',
+    labelNames: ['id']
+  });
+
+  totalChatMessagesPerChatRoom = new client.Counter({
+    name: 'total_chat_messages_per_chatroom',
+    help: 'Total number of messages sent per chatroom',
+    labelNames: ['id']
+  });
+
+  currentSizeOfPrintingFolderBytes = new client.Gauge({
+    name: 'current_size_of_printing_folder_bytes',
+    help: 'Current size of printing folder in bytes'
+  });
+
+  totalExpiredChunksDeleted = new client.Counter({
+    name: 'total_expired_chunks_deleted',
+    help: 'Total number of expired chunks that have been deleted'
+  });
+
+  totalExpiredBytesDeleted = new client.Counter({
+    name: 'total_expired_bytes_deleted',
+    help: 'Total number of bytes from expired chunks that have been deleted'
+  });
+
+  constructor() {
+    register.setDefaultLabels({
+      app: 'sce-core',
+    });
+    client.collectDefaultMetrics({ register });
+
+    Object.keys(this).forEach(metric => {
+      register.registerMetric(this[metric]);
+    });
+  }
 }
 
 module.exports = {

src/APIFunctions/2DPrinting.js

@@ -23,7 +23,11 @@ export async function healthCheck() {
   const url = new URL('/api/Printer/healthCheck', BASE_API_URL);
   try {
     const res = await fetch(url.href);
-    status.error = !res.ok;
+    if (res.ok) {
+      status.responseData = await res.json();
+    } else {
+      status.error = true;
+    }
   } catch (err) {
     status.responseData = err;
     status.error = true;
@@ -76,21 +80,45 @@ export function parseRange(pages, maxPages) {
 export async function printPage(data, token) {
   let status = new ApiResponse();
   const url = new URL('/api/Printer/sendPrintRequest', BASE_API_URL);
-  try {
-    const res = await fetch(url.href, {
-      method: 'POST',
-      headers: {
-        'Authorization': `Bearer ${token}`
-      },
-      body: data
-    });
-    if (res.ok) {
-      status.responseData = true;
+
+  const pdf = data.get('file');
+  const sides = data.get('sides');
+  const copies = data.get('copies');
+  const id = crypto.randomUUID();
+  const CHUNK_SIZE = 1024 * 1024 * 0.5; // 0.5 MB ------- SENT DATA **CANNOT** EXCEED 1 MB
+  const totalChunks = Math.ceil(pdf.size / CHUNK_SIZE);
+
+  for (let i = 0; i < totalChunks; i++) {
+    let chunkData = new FormData();
+    let chunkStart = i * CHUNK_SIZE;
+    let chunk = pdf.slice(chunkStart, chunkStart + CHUNK_SIZE);
+    chunkData.append('chunk', chunk, id + '_' + i + '.CHUNK');
+    chunkData.append('totalChunks', totalChunks);
+    chunkData.append('chunkIdx', i);
+
+    if (i === totalChunks - 1) {
+      chunkData.append('id', id);
+      chunkData.append('sides', sides);
+      chunkData.append('copies', copies);
+    }
+
+    try {
+      const res = await fetch(url.href, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${token}`
+        },
+        body: chunkData
+      });
+
+      status.responseData = !!res.ok;
+    } catch (err) {
+      status.responseData = err;
+      status.error = true;
+      return status;
     }
-  } catch (err) {
-    status.responseData = err;
-    status.error = true;
   }
+
   return status;
 }