From 8a6bc0d5f4264a5f940bcefa3cce96e30c44413b Mon Sep 17 00:00:00 2001 From: Zachary Brown Date: Mon, 13 Apr 2026 13:19:59 -0700 Subject: [PATCH] fix(V2): convert CSV batch upload staged data to snake_case The V2 CSV batch upload endpoints for projects and units were staging data with camelCase field names (from CSV headers), but the V2 commit pipeline expects snake_case. This caused commits to fail because the primary key field (e.g. cad_trust_project_id) could not be found in the staged data headers. - Reuse toDbFieldNames() from v2-xls.js to convert CSV records to snake_case before staging, matching the XLSX upload path - Inject org_uid into staged records (consistent with XLSX path) - Hoist getHomeOrg() call before the CSV stream loop (once per batch instead of once per row) - Reuse UnitV2.prepareXlsRow() for serial ID derivation instead of duplicating the logic inline - Add integration tests verifying staged data format (snake_case fields, org_uid presence, primary key correctness) - Add tests for array field parsing (pipe-separated, JSON), error cases (non-existent ID, empty CSV), and serial ID derivation - Update V2 API docs to clarify CSV batch create/update behavior and document that NEW- placeholders are XLSX-only --- docs/cadt_rpc_api_v2.md | 29 ++++- src/models/v2/project-v2.model.js | 34 +++--- src/models/v2/unit-v2.model.js | 38 +++--- src/utils/v2-xls.js | 2 +- tests/v2/integration/project-v2.spec.js | 156 +++++++++++++++++++++--- tests/v2/integration/unit-v2.spec.js | 133 +++++++++++++++++--- 6 files changed, 318 insertions(+), 74 deletions(-) diff --git a/docs/cadt_rpc_api_v2.md b/docs/cadt_rpc_api_v2.md index 3b4c23dc9..709d65a45 100644 --- a/docs/cadt_rpc_api_v2.md +++ b/docs/cadt_rpc_api_v2.md @@ -2416,12 +2416,20 @@ Response #### Batch upload projects from CSV -**Array Field Formatting**: For array fields like `projectType` and `projectSector`, the CSV can use any of these formats: +Stage one or more projects from a CSV file. Each row becomes a separate staging record. + +**Creating vs updating:** +- To **create** a new project, omit the `cadTrustProjectId` column (or leave it blank). The server auto-generates a UUID. +- To **update** an existing project, include `cadTrustProjectId` with the existing project's UUID. The project must already exist. + +> **Note**: `NEW-` placeholder IDs (e.g., `NEW-1`, `NEW-2`) are only supported in the [XLSX import](#xlsx-importexport) workflow, not in CSV batch upload. For new projects via CSV, simply omit `cadTrustProjectId`. + +**Array field formatting**: For array fields like `projectType` and `projectSector`, the CSV can use any of these formats: - **Single value**: `Solar` → becomes `["Solar"]` - **JSON array**: `["Solar","Wind"]` → becomes `["Solar","Wind"]` - **Pipe-separated**: `Solar|Wind` → becomes `["Solar","Wind"]` -Example CSV content: +Example CSV content (creating new projects — `cadTrustProjectId` omitted): ```csv projectRegistryName,projectId,projectName,projectType,projectSector,projectStatus VCS,PROJ-001,Solar Farm Project,Solar,Energy,Registered @@ -3689,6 +3697,23 @@ Response #### Batch upload units from CSV +Stage one or more units from a CSV file. Each row becomes a separate staging record. + +**Creating vs updating:** +- To **create** a new unit, omit the `cadTrustUnitId` column (or leave it blank). The server auto-generates a UUID. +- To **update** an existing unit, include `cadTrustUnitId` with the existing unit's UUID. The unit must already exist. + +> **Note**: `NEW-` placeholder IDs (e.g., `NEW-1`, `NEW-2`) are only supported in the [XLSX import](#xlsx-importexport) workflow, not in CSV batch upload. For new units via CSV, simply omit `cadTrustUnitId`. + +**Serial ID derivation**: If `unitSerialId` is omitted but `unitStartBlock` and `unitEndBlock` are provided, the server derives `unitSerialId` automatically (e.g., `BLOCK001-BLOCK050`). + +Example CSV content (creating new units — `cadTrustUnitId` omitted): +```csv +unitSerialId,unitStartBlock,unitEndBlock,unitVintageYear,unitCount,unitType,unitStatus,unitLink,unitMetric,cadTrustProjectId +UNIT-001,BLOCK001,BLOCK050,2024,100,Removal - nature,Held,http://example.com/unit1,tCO2e,51ca9638-22b0-4e14-ae7a-c09d23b37b58 +UNIT-002,BLOCK051,BLOCK100,2024,200,Avoidance,Buffer,http://example.com/unit2,tCO2e,51ca9638-22b0-4e14-ae7a-c09d23b37b58 +``` + Request ```shell curl --location --request POST 'http://localhost:31310/v2/unit/batch' --form 'csv=@"./createUnit.csv"' diff --git a/src/models/v2/project-v2.model.js b/src/models/v2/project-v2.model.js index 52b2b0776..4c4529045 100644 --- a/src/models/v2/project-v2.model.js +++ b/src/models/v2/project-v2.model.js @@ -10,7 +10,7 @@ import { createXlsFromSequelizeResults, transformFullXslsToChangeList, } from '../../utils/xls.js'; -import { parseV2Xlsx, stageV2XlsRecords } from '../../utils/v2-xls.js'; +import { parseV2Xlsx, stageV2XlsRecords, toDbFieldNames } from '../../utils/v2-xls.js'; import { getDeletedItems } from '../../utils/model-utils.js'; import { keyValueToChangeList } from '../../utils/datalayer-utils.js'; import { LocationV2 } from './location-v2.model.js'; @@ -365,17 +365,21 @@ class ProjectV2 extends Model { const recordsToCreate = []; + const homeOrg = await OrganizationsV2.getHomeOrg(false); + if (!homeOrg) { + throw new Error('No home organization found'); + } + const orgUid = homeOrg.org_uid; + return new Promise((resolve, reject) => { csv() .fromStream(stream) .subscribe(async (newRecord) => { let action = 'UPDATE'; - // Convert camelCase to snake_case for V2 const projectId = newRecord.cadTrustProjectId || newRecord.cad_trust_project_id; if (projectId) { - // Check if project exists const possibleExistingRecord = await ProjectV2.findByPk(projectId); if (!possibleExistingRecord) { @@ -386,32 +390,22 @@ class ProjectV2 extends Model { ); return; } - - // Verify it belongs to home org (for updates) - const homeOrg = await OrganizationsV2.getHomeOrg(); - if (!homeOrg) { - reject(new Error('No home organization found')); - return; - } } else { - // New project - generate UUID newRecord.cadTrustProjectId = uuidv4(); - const homeOrg = await OrganizationsV2.getHomeOrg(); - if (!homeOrg) { - reject(new Error('No home organization found')); - return; - } action = 'INSERT'; } - // Update project properties (handle child records) ProjectV2.updateProjectPropertiesV2(newRecord); + const uuid = newRecord.cadTrustProjectId || newRecord.cad_trust_project_id; + const dbRecord = toDbFieldNames(newRecord, ProjectV2); + dbRecord.org_uid = orgUid; + const stagedData = { - uuid: newRecord.cadTrustProjectId, - action: action, + uuid, + action, table: 'project', - data: JSON.stringify([newRecord]), + data: JSON.stringify([dbRecord]), }; recordsToCreate.push(stagedData); diff --git a/src/models/v2/unit-v2.model.js b/src/models/v2/unit-v2.model.js index ebee45df6..7a2e5eb8d 100644 --- a/src/models/v2/unit-v2.model.js +++ b/src/models/v2/unit-v2.model.js @@ -14,7 +14,7 @@ import { createXlsFromSequelizeResults, transformFullXslsToChangeList, } from '../../utils/xls.js'; -import { parseV2Xlsx, stageV2XlsRecords } from '../../utils/v2-xls.js'; +import { parseV2Xlsx, stageV2XlsRecords, toDbFieldNames } from '../../utils/v2-xls.js'; import { getDeletedItems } from '../../utils/model-utils.js'; import { UnitLabelV2 } from './unit-label-v2.model.js'; import { loggerV2 } from '../../config/logger.js'; @@ -426,17 +426,21 @@ class UnitV2 extends Model { const recordsToCreate = []; + const homeOrg = await OrganizationsV2.getHomeOrg(false); + if (!homeOrg) { + throw new Error('No home organization found'); + } + const orgUid = homeOrg.org_uid; + return new Promise((resolve, reject) => { csv() .fromStream(stream) .subscribe(async (newRecord) => { let action = 'UPDATE'; - // Convert camelCase to snake_case for V2 const unitId = newRecord.cadTrustUnitId || newRecord.cad_trust_unit_id; if (unitId) { - // Check if unit exists const possibleExistingRecord = await UnitV2.findByPk(unitId); if (!possibleExistingRecord) { @@ -447,34 +451,22 @@ class UnitV2 extends Model { ); return; } - - // Verify it belongs to home org (for updates) - const homeOrg = await OrganizationsV2.getHomeOrg(); - if (!homeOrg) { - reject(new Error('No home organization found')); - return; - } } else { - // New unit - generate UUID newRecord.cadTrustUnitId = uuidv4(); - const homeOrg = await OrganizationsV2.getHomeOrg(); - if (!homeOrg) { - reject(new Error('No home organization found')); - return; - } action = 'INSERT'; } - // Update unit properties (handle serial ID from blocks) - if (newRecord.unitStartBlock && newRecord.unitEndBlock) { - newRecord.unitSerialId = `${newRecord.unitStartBlock}-${newRecord.unitEndBlock}`; - } + UnitV2.prepareXlsRow(newRecord); + + const uuid = newRecord.cadTrustUnitId || newRecord.cad_trust_unit_id; + const dbRecord = toDbFieldNames(newRecord, UnitV2); + dbRecord.org_uid = orgUid; const stagedData = { - uuid: newRecord.cadTrustUnitId, - action: action, + uuid, + action, table: 'unit', - data: JSON.stringify([newRecord]), + data: JSON.stringify([dbRecord]), }; recordsToCreate.push(stagedData); diff --git a/src/utils/v2-xls.js b/src/utils/v2-xls.js index 5a370c03a..987b080b4 100644 --- a/src/utils/v2-xls.js +++ b/src/utils/v2-xls.js @@ -178,7 +178,7 @@ function parseArrayFields(row) { * staging data to use snake_case field names, matching what the normal API * controllers produce. */ -function toDbFieldNames(row, modelClass) { +export function toDbFieldNames(row, modelClass) { const attrs = modelClass.rawAttributes; const result = {}; for (const [key, value] of Object.entries(row)) { diff --git a/tests/v2/integration/project-v2.spec.js b/tests/v2/integration/project-v2.spec.js index 1755bf5b7..bde502cdb 100644 --- a/tests/v2/integration/project-v2.spec.js +++ b/tests/v2/integration/project-v2.spec.js @@ -1631,7 +1631,6 @@ describe('V2 Project API - Basic CRUD Tests', function () { describe('POST /v2/project/batch', function () { it('should batch upload new projects from CSV file (INSERT)', async function () { - // Create a CSV file buffer without cadTrustProjectId to trigger INSERT const csvContent = `projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId Test Registry,CSV-001,CSV Test Project 1,Agriculture,Landfill gas,Listed,tCO2e,${testProgram.cadTrustProgramId} Test Registry,CSV-002,CSV Test Project 2,Energy,Energy efficiency,Registered,tCO2e,${testProgram.cadTrustProgramId}`; @@ -1646,19 +1645,88 @@ Test Registry,CSV-002,CSV Test Project 2,Energy,Energy efficiency,Registered,tCO expect(response.body.success).to.be.true; expect(response.body.message).to.include('CSV processing complete'); - // Verify records were staged const stagingRecords = await StagingV2.findAll({ - where: { - table: 'project', - action: 'INSERT', - }, + where: { table: 'project', action: 'INSERT' }, }); expect(stagingRecords.length).to.be.at.least(2); }); + it('should stage INSERT data in snake_case with org_uid and primary key', async function () { + const csvContent = `projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId +Test Registry,CSV-FMT-001,Format Test,Agriculture,Landfill gas,Listed,tCO2e,${testProgram.cadTrustProgramId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'project', action: 'INSERT' }, + }); + expect(stagingRecords.length).to.equal(1); + + const staged = JSON.parse(stagingRecords[0].data); + expect(staged).to.be.an('array').with.lengthOf(1); + const record = staged[0]; + + // Must use snake_case field names (commit pipeline expects this) + expect(record).to.have.property('cad_trust_project_id'); + expect(record).to.have.property('project_registry_name', 'Test Registry'); + expect(record).to.have.property('project_name', 'Format Test'); + + // Must NOT have camelCase duplicates + expect(record).to.not.have.property('cadTrustProjectId'); + expect(record).to.not.have.property('projectRegistryName'); + expect(record).to.not.have.property('projectName'); + + // Must include org_uid from home org + const homeOrgId = await getV2HomeOrgId(); + expect(record).to.have.property('org_uid', homeOrgId); + + // Staging uuid must match the generated primary key + expect(stagingRecords[0].uuid).to.equal(record.cad_trust_project_id); + }); + + it('should stage UPDATE data in snake_case with org_uid', async function () { + const homeOrgId = await getV2HomeOrgId(); + const project = await ProjectV2.create(addUuidIfNeeded('ProjectV2', { + projectRegistryName: 'Test Registry', + projectId: 'CSV-FMT-UPD-001', + projectName: 'Original', + projectSector: ['Agriculture'], + projectType: ['Landfill gas'], + projectStatus: 'Listed', + projectUnitMetric: 'tCO2e', + cadTrustProgramId: testProgram.cadTrustProgramId, + orgUid: homeOrgId, + })); + + const csvContent = `cadTrustProjectId,projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId +${project.cadTrustProjectId},Test Registry,CSV-FMT-UPD-001,Updated,Agriculture,Landfill gas,Listed,tCO2e,${testProgram.cadTrustProgramId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'project', action: 'UPDATE' }, + }); + expect(stagingRecords.length).to.equal(1); + + const record = JSON.parse(stagingRecords[0].data)[0]; + expect(record).to.have.property('cad_trust_project_id', project.cadTrustProjectId); + expect(record).to.have.property('project_name', 'Updated'); + expect(record).to.have.property('org_uid', homeOrgId); + expect(record).to.not.have.property('cadTrustProjectId'); + }); + it('should batch update existing projects from CSV file (UPDATE)', async function () { - // Create projects first const homeOrgId = await getV2HomeOrgId(); const project1 = await ProjectV2.create(addUuidIfNeeded('ProjectV2', { projectRegistryName: 'Test Registry', @@ -1684,7 +1752,6 @@ Test Registry,CSV-002,CSV Test Project 2,Energy,Energy efficiency,Registered,tCO orgUid: homeOrgId, })); - // Create a CSV file buffer with cadTrustProjectId to trigger UPDATE const csvContent = `cadTrustProjectId,projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId ${project1.cadTrustProjectId},Test Registry,CSV-UPDATE-001,Updated Name 1,Agriculture,Landfill gas,Listed,tCO2e,${testProgram.cadTrustProgramId} ${project2.cadTrustProjectId},Test Registry,CSV-UPDATE-002,Updated Name 2,Energy,Energy efficiency,Registered,tCO2e,${testProgram.cadTrustProgramId}`; @@ -1697,19 +1764,80 @@ ${project2.cadTrustProjectId},Test Registry,CSV-UPDATE-002,Updated Name 2,Energy .expect(200); expect(response.body.success).to.be.true; - expect(response.body.message).to.include('CSV processing complete'); - // Verify records were staged as UPDATE const stagingRecords = await StagingV2.findAll({ - where: { - table: 'project', - action: 'UPDATE', - }, + where: { table: 'project', action: 'UPDATE' }, }); expect(stagingRecords.length).to.be.at.least(2); }); + it('should parse pipe-separated array fields', async function () { + const csvContent = `projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId +Test Registry,CSV-PIPE-001,Pipe Test,Agriculture|Energy,Landfill gas|Energy efficiency,Listed,tCO2e,${testProgram.cadTrustProgramId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'project', action: 'INSERT' }, + }); + const record = JSON.parse(stagingRecords[0].data)[0]; + + expect(record.project_sector).to.deep.equal(['Agriculture', 'Energy']); + expect(record.project_type).to.deep.equal(['Landfill gas', 'Energy efficiency']); + }); + + it('should parse JSON array fields', async function () { + const csvContent = `projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric,cadTrustProgramId +Test Registry,CSV-JSON-001,JSON Test,"[""Agriculture"",""Energy""]","[""Landfill gas""]",Listed,tCO2e,${testProgram.cadTrustProgramId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'project', action: 'INSERT' }, + }); + const record = JSON.parse(stagingRecords[0].data)[0]; + + expect(record.project_sector).to.deep.equal(['Agriculture', 'Energy']); + expect(record.project_type).to.deep.equal(['Landfill gas']); + }); + + it('should reject CSV with non-existent cadTrustProjectId', async function () { + const csvContent = `cadTrustProjectId,projectRegistryName,projectId,projectName,projectSector,projectType,projectStatus,projectUnitMetric +${uuidv4()},Test Registry,CSV-NOEXIST-001,No Exist,Agriculture,Landfill gas,Listed,tCO2e`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + const response = await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(400); + + expect(response.body.success).to.be.false; + }); + + it('should reject empty CSV with no data rows', async function () { + const csvContent = `projectRegistryName,projectId,projectName`; + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + const response = await supertest(app) + .post('/v2/project/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(400); + + expect(response.body.success).to.be.false; + }); + it('should return error if no CSV file is provided', async function () { const response = await supertest(app) .post('/v2/project/batch') diff --git a/tests/v2/integration/unit-v2.spec.js b/tests/v2/integration/unit-v2.spec.js index 502246baa..5e8b60db6 100644 --- a/tests/v2/integration/unit-v2.spec.js +++ b/tests/v2/integration/unit-v2.spec.js @@ -1577,7 +1577,6 @@ describe('V2 Unit API - Basic CRUD Tests', function () { describe('POST /v2/unit/batch', function () { it('should batch upload new units from CSV file (INSERT)', async function () { - // Create a CSV file buffer without cadTrustUnitId to trigger INSERT const csvContent = `unitSerialId,unitStartBlock,unitEndBlock,unitCount,unitType,unitVintageYear,unitStatus,cadTrustIssuanceId CSV-UNIT-001,1000,2000,50,Avoidance - nature,2024,Issued,${testIssuanceForAdvanced.cadTrustIssuanceId} CSV-UNIT-002,2000,3000,75,Reduction - technical,2024,Held,${testIssuanceForAdvanced.cadTrustIssuanceId}`; @@ -1592,19 +1591,105 @@ CSV-UNIT-002,2000,3000,75,Reduction - technical,2024,Held,${testIssuanceForAdvan expect(response.body.success).to.be.true; expect(response.body.message).to.include('CSV processing complete'); - // Verify records were staged const stagingRecords = await StagingV2.findAll({ - where: { - table: 'unit', - action: 'INSERT', - }, + where: { table: 'unit', action: 'INSERT' }, }); expect(stagingRecords.length).to.be.at.least(2); }); + it('should stage INSERT data in snake_case with org_uid and primary key', async function () { + const csvContent = `unitSerialId,unitStartBlock,unitEndBlock,unitCount,unitType,unitVintageYear,unitStatus,cadTrustIssuanceId +CSV-FMT-001,1000,2000,50,Avoidance - nature,2024,Issued,${testIssuanceForAdvanced.cadTrustIssuanceId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/unit/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'unit', action: 'INSERT' }, + }); + expect(stagingRecords.length).to.equal(1); + + const staged = JSON.parse(stagingRecords[0].data); + expect(staged).to.be.an('array').with.lengthOf(1); + const record = staged[0]; + + // Must use snake_case field names (commit pipeline expects this) + expect(record).to.have.property('cad_trust_unit_id'); + expect(record).to.have.property('unit_serial_id', 'CSV-FMT-001'); + expect(record).to.have.property('unit_count', '50'); + + // Must NOT have camelCase duplicates + expect(record).to.not.have.property('cadTrustUnitId'); + expect(record).to.not.have.property('unitSerialId'); + expect(record).to.not.have.property('unitCount'); + + // Must include org_uid from home org + const homeOrgId = await getV2HomeOrgId(); + expect(record).to.have.property('org_uid', homeOrgId); + + // Staging uuid must match the generated primary key + expect(stagingRecords[0].uuid).to.equal(record.cad_trust_unit_id); + }); + + it('should stage UPDATE data in snake_case with org_uid', async function () { + const homeOrgId = await getV2HomeOrgId(); + const unit = await UnitV2.create(addUuidIfNeeded('UnitV2', { + unitSerialId: 'CSV-FMT-UPD-001', + unitStartBlock: '1000', + unitEndBlock: '2000', + unitCount: 50, + unitVintageYear: 2024, + cadTrustIssuanceId: testIssuanceForAdvanced.cadTrustIssuanceId, + orgUid: homeOrgId, + })); + + const csvContent = `cadTrustUnitId,unitSerialId,unitStartBlock,unitEndBlock,unitCount,unitType,unitVintageYear,unitStatus,cadTrustIssuanceId +${unit.cadTrustUnitId},CSV-FMT-UPD-001,1000,2000,60,Avoidance - nature,2024,Issued,${testIssuanceForAdvanced.cadTrustIssuanceId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/unit/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'unit', action: 'UPDATE' }, + }); + expect(stagingRecords.length).to.equal(1); + + const record = JSON.parse(stagingRecords[0].data)[0]; + expect(record).to.have.property('cad_trust_unit_id', unit.cadTrustUnitId); + expect(record).to.have.property('unit_count', '60'); + expect(record).to.have.property('org_uid', homeOrgId); + expect(record).to.not.have.property('cadTrustUnitId'); + }); + + it('should derive unitSerialId from start/end blocks', async function () { + const csvContent = `unitStartBlock,unitEndBlock,unitCount,unitVintageYear,cadTrustIssuanceId +BLOCK-A,BLOCK-Z,100,2024,${testIssuanceForAdvanced.cadTrustIssuanceId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + await supertest(app) + .post('/v2/unit/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(200); + + const stagingRecords = await StagingV2.findAll({ + where: { table: 'unit', action: 'INSERT' }, + }); + const record = JSON.parse(stagingRecords[0].data)[0]; + + expect(record).to.have.property('unit_serial_id', 'BLOCK-A-BLOCK-Z'); + }); + it('should batch update existing units from CSV file (UPDATE)', async function () { - // Create units first const homeOrgId = await getV2HomeOrgId(); const unit1 = await UnitV2.create(addUuidIfNeeded('UnitV2', { unitSerialId: 'CSV-UPDATE-001', @@ -1626,7 +1711,6 @@ CSV-UNIT-002,2000,3000,75,Reduction - technical,2024,Held,${testIssuanceForAdvan orgUid: homeOrgId, })); - // Create a CSV file buffer with cadTrustUnitId to trigger UPDATE const csvContent = `cadTrustUnitId,unitSerialId,unitStartBlock,unitEndBlock,unitCount,unitType,unitVintageYear,unitStatus,cadTrustIssuanceId ${unit1.cadTrustUnitId},CSV-UPDATE-001,1000,2000,60,Avoidance - nature,2024,Issued,${testIssuanceForAdvanced.cadTrustIssuanceId} ${unit2.cadTrustUnitId},CSV-UPDATE-002,2000,3000,80,Reduction - technical,2024,Held,${testIssuanceForAdvanced.cadTrustIssuanceId}`; @@ -1639,19 +1723,40 @@ ${unit2.cadTrustUnitId},CSV-UPDATE-002,2000,3000,80,Reduction - technical,2024,H .expect(200); expect(response.body.success).to.be.true; - expect(response.body.message).to.include('CSV processing complete'); - // Verify records were staged as UPDATE const stagingRecords = await StagingV2.findAll({ - where: { - table: 'unit', - action: 'UPDATE', - }, + where: { table: 'unit', action: 'UPDATE' }, }); expect(stagingRecords.length).to.be.at.least(2); }); + it('should reject CSV with non-existent cadTrustUnitId', async function () { + const csvContent = `cadTrustUnitId,unitSerialId,unitStartBlock,unitEndBlock,unitCount,unitVintageYear,cadTrustIssuanceId +${uuidv4()},NOEXIST-001,1000,2000,50,2024,${testIssuanceForAdvanced.cadTrustIssuanceId}`; + + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + const response = await supertest(app) + .post('/v2/unit/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(400); + + expect(response.body.success).to.be.false; + }); + + it('should reject empty CSV with no data rows', async function () { + const csvContent = `unitSerialId,unitStartBlock,unitEndBlock`; + const csvBuffer = Buffer.from(csvContent, 'utf8'); + + const response = await supertest(app) + .post('/v2/unit/batch') + .attach('csv', csvBuffer, 'test.csv') + .expect(400); + + expect(response.body.success).to.be.false; + }); + it('should return error if no CSV file is provided', async function () { const response = await supertest(app) .post('/v2/unit/batch')