Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions docs/cadt_rpc_api_v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -2416,12 +2416,20 @@ Response

#### Batch upload projects from CSV

**Array Field Formatting**: For array fields like `projectType` and `projectSector`, the CSV can use any of these formats:
Stage one or more projects from a CSV file. Each row becomes a separate staging record.

**Creating vs updating:**
- To **create** a new project, omit the `cadTrustProjectId` column (or leave it blank). The server auto-generates a UUID.
- To **update** an existing project, include `cadTrustProjectId` with the existing project's UUID. The project must already exist.

> **Note**: `NEW-<n>` placeholder IDs (e.g., `NEW-1`, `NEW-2`) are only supported in the [XLSX import](#xlsx-importexport) workflow, not in CSV batch upload. For new projects via CSV, simply omit `cadTrustProjectId`.

**Array field formatting**: For array fields like `projectType` and `projectSector`, the CSV can use any of these formats:
- **Single value**: `Solar` → becomes `["Solar"]`
- **JSON array**: `["Solar","Wind"]` → becomes `["Solar","Wind"]`
- **Pipe-separated**: `Solar|Wind` → becomes `["Solar","Wind"]`

Example CSV content:
Example CSV content (creating new projects — `cadTrustProjectId` omitted):
```csv
projectRegistryName,projectId,projectName,projectType,projectSector,projectStatus
VCS,PROJ-001,Solar Farm Project,Solar,Energy,Registered
Expand Down Expand Up @@ -3689,6 +3697,23 @@ Response

#### Batch upload units from CSV

Stage one or more units from a CSV file. Each row becomes a separate staging record.

**Creating vs updating:**
- To **create** a new unit, omit the `cadTrustUnitId` column (or leave it blank). The server auto-generates a UUID.
- To **update** an existing unit, include `cadTrustUnitId` with the existing unit's UUID. The unit must already exist.

> **Note**: `NEW-<n>` placeholder IDs (e.g., `NEW-1`, `NEW-2`) are only supported in the [XLSX import](#xlsx-importexport) workflow, not in CSV batch upload. For new units via CSV, simply omit `cadTrustUnitId`.

**Serial ID derivation**: If `unitSerialId` is omitted but `unitStartBlock` and `unitEndBlock` are provided, the server derives `unitSerialId` automatically (e.g., `BLOCK001-BLOCK050`).

Example CSV content (creating new units — `cadTrustUnitId` omitted):
```csv
unitSerialId,unitStartBlock,unitEndBlock,unitVintageYear,unitCount,unitType,unitStatus,unitLink,unitMetric,cadTrustProjectId
UNIT-001,BLOCK001,BLOCK050,2024,100,Removal - nature,Held,http://example.com/unit1,tCO2e,51ca9638-22b0-4e14-ae7a-c09d23b37b58
UNIT-002,BLOCK051,BLOCK100,2024,200,Avoidance,Buffer,http://example.com/unit2,tCO2e,51ca9638-22b0-4e14-ae7a-c09d23b37b58
```

Request
```shell
curl --location --request POST 'http://localhost:31310/v2/unit/batch' --form 'csv=@"./createUnit.csv"'
Expand Down
34 changes: 14 additions & 20 deletions src/models/v2/project-v2.model.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
createXlsFromSequelizeResults,
transformFullXslsToChangeList,
} from '../../utils/xls.js';
import { parseV2Xlsx, stageV2XlsRecords } from '../../utils/v2-xls.js';
import { parseV2Xlsx, stageV2XlsRecords, toDbFieldNames } from '../../utils/v2-xls.js';
import { getDeletedItems } from '../../utils/model-utils.js';
import { keyValueToChangeList } from '../../utils/datalayer-utils.js';
import { LocationV2 } from './location-v2.model.js';
Expand Down Expand Up @@ -365,17 +365,21 @@ class ProjectV2 extends Model {

const recordsToCreate = [];

const homeOrg = await OrganizationsV2.getHomeOrg(false);
if (!homeOrg) {
throw new Error('No home organization found');
}
const orgUid = homeOrg.org_uid;

return new Promise((resolve, reject) => {
csv()
.fromStream(stream)
.subscribe(async (newRecord) => {
let action = 'UPDATE';

// Convert camelCase to snake_case for V2
const projectId = newRecord.cadTrustProjectId || newRecord.cad_trust_project_id;

if (projectId) {
// Check if project exists
const possibleExistingRecord = await ProjectV2.findByPk(projectId);

if (!possibleExistingRecord) {
Expand All @@ -386,32 +390,22 @@ class ProjectV2 extends Model {
);
return;
}

// Verify it belongs to home org (for updates)
const homeOrg = await OrganizationsV2.getHomeOrg();
if (!homeOrg) {
reject(new Error('No home organization found'));
return;
}
} else {
// New project - generate UUID
newRecord.cadTrustProjectId = uuidv4();
const homeOrg = await OrganizationsV2.getHomeOrg();
if (!homeOrg) {
reject(new Error('No home organization found'));
return;
}
action = 'INSERT';
}

// Update project properties (handle child records)
ProjectV2.updateProjectPropertiesV2(newRecord);

const uuid = newRecord.cadTrustProjectId || newRecord.cad_trust_project_id;
const dbRecord = toDbFieldNames(newRecord, ProjectV2);
dbRecord.org_uid = orgUid;

const stagedData = {
uuid: newRecord.cadTrustProjectId,
action: action,
uuid,
action,
table: 'project',
data: JSON.stringify([newRecord]),
data: JSON.stringify([dbRecord]),
};

recordsToCreate.push(stagedData);
Expand Down
38 changes: 15 additions & 23 deletions src/models/v2/unit-v2.model.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
createXlsFromSequelizeResults,
transformFullXslsToChangeList,
} from '../../utils/xls.js';
import { parseV2Xlsx, stageV2XlsRecords } from '../../utils/v2-xls.js';
import { parseV2Xlsx, stageV2XlsRecords, toDbFieldNames } from '../../utils/v2-xls.js';
import { getDeletedItems } from '../../utils/model-utils.js';
import { UnitLabelV2 } from './unit-label-v2.model.js';
import { loggerV2 } from '../../config/logger.js';
Expand Down Expand Up @@ -426,17 +426,21 @@ class UnitV2 extends Model {

const recordsToCreate = [];

const homeOrg = await OrganizationsV2.getHomeOrg(false);
if (!homeOrg) {
throw new Error('No home organization found');
}
const orgUid = homeOrg.org_uid;

return new Promise((resolve, reject) => {
csv()
.fromStream(stream)
.subscribe(async (newRecord) => {
let action = 'UPDATE';

// Convert camelCase to snake_case for V2
const unitId = newRecord.cadTrustUnitId || newRecord.cad_trust_unit_id;

if (unitId) {
// Check if unit exists
const possibleExistingRecord = await UnitV2.findByPk(unitId);

if (!possibleExistingRecord) {
Expand All @@ -447,34 +451,22 @@ class UnitV2 extends Model {
);
return;
}

// Verify it belongs to home org (for updates)
const homeOrg = await OrganizationsV2.getHomeOrg();
if (!homeOrg) {
reject(new Error('No home organization found'));
return;
}
} else {
// New unit - generate UUID
newRecord.cadTrustUnitId = uuidv4();
const homeOrg = await OrganizationsV2.getHomeOrg();
if (!homeOrg) {
reject(new Error('No home organization found'));
return;
}
action = 'INSERT';
}

// Update unit properties (handle serial ID from blocks)
if (newRecord.unitStartBlock && newRecord.unitEndBlock) {
newRecord.unitSerialId = `${newRecord.unitStartBlock}-${newRecord.unitEndBlock}`;
}
UnitV2.prepareXlsRow(newRecord);

const uuid = newRecord.cadTrustUnitId || newRecord.cad_trust_unit_id;
const dbRecord = toDbFieldNames(newRecord, UnitV2);
dbRecord.org_uid = orgUid;

const stagedData = {
uuid: newRecord.cadTrustUnitId,
action: action,
uuid,
action,
table: 'unit',
data: JSON.stringify([newRecord]),
data: JSON.stringify([dbRecord]),
};

recordsToCreate.push(stagedData);
Expand Down
2 changes: 1 addition & 1 deletion src/utils/v2-xls.js
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ function parseArrayFields(row) {
* staging data to use snake_case field names, matching what the normal API
* controllers produce.
*/
function toDbFieldNames(row, modelClass) {
export function toDbFieldNames(row, modelClass) {
const attrs = modelClass.rawAttributes;
const result = {};
for (const [key, value] of Object.entries(row)) {
Expand Down
Loading
Loading