From 3d06acc95f66c7c8fd53490daca56ac59c9ca603 Mon Sep 17 00:00:00 2001 From: Joey603 Date: Wed, 22 Apr 2026 12:44:36 -0700 Subject: [PATCH 1/5] added interface types with meter values, also updated default values --- src/client/app/types/csvUploadForm.ts | 8 ++++++++ src/client/app/utils/csvUploadDefaults.ts | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/client/app/types/csvUploadForm.ts b/src/client/app/types/csvUploadForm.ts index 5cf4f9c33f..f752f0d1fb 100644 --- a/src/client/app/types/csvUploadForm.ts +++ b/src/client/app/types/csvUploadForm.ts @@ -3,12 +3,20 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { MeterTimeSortType } from '../types/redux/meters'; +import { DisableChecksType } from './redux/units'; export interface CSVUploadPreferences { meterIdentifier: string; gzip: boolean; headerRow: boolean; update: boolean; + timeZone?: string; + minVal?: string; + maxVal?: string; + minDate?: string; + maxDate?: string; + maxError?: string; + disableChecks?: DisableChecksType; } export interface ReadingsCSVUploadPreferences extends CSVUploadPreferences { diff --git a/src/client/app/utils/csvUploadDefaults.ts b/src/client/app/utils/csvUploadDefaults.ts index 43d485a0cf..f583206c84 100644 --- a/src/client/app/utils/csvUploadDefaults.ts +++ b/src/client/app/utils/csvUploadDefaults.ts @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { DisableChecksType } from 'types/redux/units'; import { ReadingsCSVUploadPreferences, MetersCSVUploadPreferences } from '../types/csvUploadForm'; import { MeterTimeSortType } from '../types/redux/meters'; @@ -25,7 +26,14 @@ export const ReadingsCSVUploadDefaults: ReadingsCSVUploadPreferences = { timeSort: MeterTimeSortType.increasing, update: false, useMeterZone: false, - warnOnCumulativeReset: false + warnOnCumulativeReset: false, + timeZone: "", + minVal: "", + maxVal: "", + minDate: "", + maxDate: "", + maxError: "", + disableChecks: DisableChecksType.reject_all, }; export const MetersCSVUploadDefaults: MetersCSVUploadPreferences = { From 4611a82b2c49c76625a8c723787e584cd8459566 Mon Sep 17 00:00:00 2001 From: Joey603 Date: Wed, 22 Apr 2026 14:09:41 -0700 Subject: [PATCH 2/5] feat(csv pipeline): honor readings upload overrides with request-first fallback add readings upload UI fields for timeZone, minVal, maxVal, minDate, maxDate, maxError, disableChecks sync new fields from selected meter and include them in unsaved-changes tracking extend readings upload param validation schema/defaults for the 7 override fields implement request-first resolution in uploadReadings (undefined/empty fallback to meter) apply resolved values to conditionSet and pass timezone override through pipeline processing harden FormData serialization to skip null/undefined values --- .../csv/ReadingsCSVUploadComponent.tsx | 149 +++++++++++++++++- src/client/app/utils/api/UploadCSVApi.ts | 8 +- .../services/csvPipeline/uploadReadings.js | 29 ++-- .../csvPipeline/validateCsvUploadParams.js | 45 +++++- .../pipeline-in-progress/loadArrayInput.js | 5 +- .../pipeline-in-progress/loadCsvInput.js | 6 +- .../pipeline-in-progress/processData.js | 5 +- 7 files changed, 226 insertions(+), 21 deletions(-) diff --git a/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx b/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx index 9988337dc4..d36ee910b8 100644 --- a/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx +++ b/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx @@ -13,11 +13,13 @@ import { selectIsAdmin } from '../../redux/slices/currentUserSlice'; import { ReadingsCSVUploadPreferences } from '../../types/csvUploadForm'; import { TrueFalseType } from '../../types/items'; import { MeterData, MeterTimeSortType } from '../../types/redux/meters'; +import { DisableChecksType } from '../../types/redux/units'; import { submitReadings } from '../../utils/api/UploadCSVApi'; import { ReadingsCSVUploadDefaults } from '../../utils/csvUploadDefaults'; import { showErrorNotification, showSuccessNotification } from '../../utils/notifications'; import { useTranslate } from '../../redux/componentHooks'; import FormFileUploaderComponent from '../FormFileUploaderComponent'; +import TimeZoneSelect from '../TimeZoneSelect'; import TooltipHelpComponent from '../TooltipHelpComponent'; import TooltipMarkerComponent from '../TooltipMarkerComponent'; import CreateMeterModalComponent from '../meters/CreateMeterModalComponent'; @@ -116,7 +118,7 @@ export default function ReadingsCSVUploadComponent() { })); }; - const handleFileChange = (file: File) => { + const handleFileChange = (file: File | null) => { if (file) { setSelectedFile(file); if (file.name.slice(-4) === '.csv' || file.name.slice(-3) === '.gz') { @@ -132,6 +134,13 @@ export default function ReadingsCSVUploadComponent() { } } }; + + const handleTimeZoneChange = (timeZone: string) => { + setReadingsData(prevData => ({ + ...prevData, + timeZone: timeZone || '' + })); + }; /* END of Handlers for each type of input change */ useEffect(() => { @@ -178,6 +187,13 @@ export default function ReadingsCSVUploadComponent() { lengthVariation: selectedMeter.readingVariation, endOnly: selectedMeter.endOnlyTime, timeSort: MeterTimeSortType[selectedMeter.timeSort as keyof typeof MeterTimeSortType], + timeZone: selectedMeter.timeZone ?? '', + minVal: selectedMeter.minVal.toString(), + maxVal: selectedMeter.maxVal.toString(), + minDate: selectedMeter.minDate, + maxDate: selectedMeter.maxDate, + maxError: selectedMeter.maxError.toString(), + disableChecks: selectedMeter.disableChecks, useMeterZone: false, warnOnCumulativeReset: false })); @@ -283,9 +299,16 @@ export default function ReadingsCSVUploadComponent() { || readingsData.refreshReadings !== ReadingsCSVUploadDefaults.refreshReadings || readingsData.relaxedParsing !== ReadingsCSVUploadDefaults.relaxedParsing || readingsData.timeSort !== ReadingsCSVUploadDefaults.timeSort + || readingsData.timeZone !== ReadingsCSVUploadDefaults.timeZone + || readingsData.minVal !== ReadingsCSVUploadDefaults.minVal + || readingsData.maxVal !== ReadingsCSVUploadDefaults.maxVal + || readingsData.minDate !== ReadingsCSVUploadDefaults.minDate + || readingsData.maxDate !== ReadingsCSVUploadDefaults.maxDate + || readingsData.maxError !== ReadingsCSVUploadDefaults.maxError + || readingsData.disableChecks !== ReadingsCSVUploadDefaults.disableChecks || readingsData.update !== ReadingsCSVUploadDefaults.update - || readingsData.useMeterZone !== readingsData.useMeterZone - || readingsData.warnOnCumulativeReset !== readingsData.warnOnCumulativeReset + || readingsData.useMeterZone !== ReadingsCSVUploadDefaults.useMeterZone + || readingsData.warnOnCumulativeReset !== ReadingsCSVUploadDefaults.warnOnCumulativeReset // If any file is added, it will count as edit made. || selectedFile !== null || invalidFileEntry === true; @@ -673,6 +696,126 @@ export default function ReadingsCSVUploadComponent() { + + + + + handleTimeZoneChange(timeZone)} /> + + + + + + {handleChange(e);}} + > + {Object.values(DisableChecksType).map(disableChecks => { + return ( + + ); + })} + + + + + + + + + {handleChange(e);}} + /> + + + + + + {handleChange(e);}} + /> + + + + + + + + {handleChange(e);}} + /> + + + + + + {handleChange(e);}} + /> + + + + + + + + {handleChange(e);}} + /> + + + {/* TODO This feature is not working perfectly so disabling from web page but allowing in curl. Rest of changes left so easy to add back in. This feature was added to help a site import data. It works but can have issues. diff --git a/src/client/app/utils/api/UploadCSVApi.ts b/src/client/app/utils/api/UploadCSVApi.ts index e515c70fd9..2fa34be55a 100644 --- a/src/client/app/utils/api/UploadCSVApi.ts +++ b/src/client/app/utils/api/UploadCSVApi.ts @@ -35,7 +35,9 @@ export const submitReadings = async (uploadPreferences: ReadingsCSVUploadPrefere warnOnCumulativeReset: uploadPreferences.warnOnCumulativeReset }; for (const [preference, value] of Object.entries(uploadPreferencesForm)) { - formData.append(preference, value.toString()); + if (value !== undefined && value !== null) { + formData.append(preference, value.toString()); + } } formData.append('csvfile', readingsFile); // It is important for the server that the file is attached last. @@ -61,7 +63,9 @@ export const submitMeters = async (uploadPreferences: MetersCSVUploadPreferences update: uploadPreferences.update }; for (const [preference, value] of Object.entries(uploadPreferencesForm)) { - formData.append(preference, value.toString()); + if (value !== undefined && value !== null) { + formData.append(preference, value.toString()); + } } formData.append('csvfile', metersFile); // It is important for the server that the file is attached last. diff --git a/src/server/services/csvPipeline/uploadReadings.js b/src/server/services/csvPipeline/uploadReadings.js index 5247ac7c77..105e83ee79 100644 --- a/src/server/services/csvPipeline/uploadReadings.js +++ b/src/server/services/csvPipeline/uploadReadings.js @@ -18,7 +18,8 @@ const Meter = require('../../models/Meter'); */ async function uploadReadings(req, res, filepath, conn) { // extract query parameters - const { meterIdentifier, meterName, headerRow, update, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset } = req.body; + const { meterIdentifier, meterName, headerRow, update, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset, + timeZone, minVal, maxVal, minDate, maxDate, maxError, disableChecks } = req.body; // The next few have no value in the DB for a meter so always use the value passed. const hasHeaderRow = normalizeBoolean(headerRow); const shouldUpdate = normalizeBoolean(update); @@ -188,16 +189,25 @@ async function uploadReadings(req, res, filepath, conn) { } const areReadingsEndOnly = readingEndOnly; + const isMissingParam = param => param === undefined || param === ''; + const resolvedTimeZone = isMissingParam(timeZone) ? meter.meterTimezone : timeZone; + const resolvedMinVal = isMissingParam(minVal) ? meter.minVal : parseFloat(minVal); + const resolvedMaxVal = isMissingParam(maxVal) ? meter.maxVal : parseFloat(maxVal); + const resolvedMinDate = isMissingParam(minDate) ? meter.minDate : new Date(minDate); + const resolvedMaxDate = isMissingParam(maxDate) ? meter.maxDate : new Date(maxDate); + const resolvedMaxError = isMissingParam(maxError) ? meter.maxError : parseInt(maxError, 10); + const resolvedDisableChecks = isMissingParam(disableChecks) ? meter.disableChecks : disableChecks; + const mapRowToModel = row => { return row; }; // STUB function to satisfy the parameter of loadCsvInput. const conditionSet = { - minVal: meter.minVal, - maxVal: meter.maxVal, - minDate: meter.minDate, - maxDate: meter.maxDate, - threshold: meter.readingGap, - maxError: meter.maxError, - disableChecks: meter.disableChecks + minVal: resolvedMinVal, + maxVal: resolvedMaxVal, + minDate: resolvedMinDate, + maxDate: resolvedMaxDate, + threshold: readingGap, + maxError: resolvedMaxError, + disableChecks: resolvedDisableChecks } return await loadCsvInput( @@ -220,7 +230,8 @@ async function uploadReadings(req, res, filepath, conn) { shouldHonorDst, shouldRelaxedParsing, shouldUseMeterZone, - shouldWarnOnCumulativeReset + shouldWarnOnCumulativeReset, + resolvedTimeZone ); // load csv data } diff --git a/src/server/services/csvPipeline/validateCsvUploadParams.js b/src/server/services/csvPipeline/validateCsvUploadParams.js index b3b3f92fca..daccf7e720 100644 --- a/src/server/services/csvPipeline/validateCsvUploadParams.js +++ b/src/server/services/csvPipeline/validateCsvUploadParams.js @@ -19,6 +19,13 @@ MeterTimeSortTypesJS = Object.freeze({ decreasing: 'decreasing', }); +DisableChecksTypesJS = Object.freeze({ + reject_disabled: 'reject_disabled', + reject_bad: 'reject_bad', + reject_all: 'reject_all', + reject_none: 'reject_none' +}); + // This function allows for curl users to continue to use 'yes' or 'no' and also allows string // values of true or false if the change is made. const normalizeBoolean = (input) => { @@ -64,6 +71,13 @@ const DEFAULTS = { meters: { }, readings: { + timeZone: undefined, + minVal: undefined, + maxVal: undefined, + minDate: undefined, + maxDate: undefined, + maxError: undefined, + disableChecks: undefined, cumulative: undefined, cumulativeReset: undefined, cumulativeResetStart: undefined, @@ -115,6 +129,13 @@ const VALIDATION = { type: 'object', properties: { ...COMMON_PROPERTIES, + timeZone: new StringParam('timeZone', undefined, undefined), + minVal: new StringParam('minVal', undefined, undefined), + maxVal: new StringParam('maxVal', undefined, undefined), + minDate: new StringParam('minDate', undefined, undefined), + maxDate: new StringParam('maxDate', undefined, undefined), + maxError: new StringParam('maxError', undefined, undefined), + disableChecks: new EnumParam('disableChecks', Object.values(DisableChecksTypesJS)), cumulative: new EnumParam('cumulative', BooleanCheckArray), cumulativeReset: new EnumParam('cumulativeReset', BooleanCheckArray), cumulativeResetStart: new StringParam('cumulativeResetStart', undefined, undefined), @@ -182,10 +203,32 @@ function validateReadingsCsvUploadParams(req, res, next) { } // extract query parameters - const { cumulative, cumulativeReset, duplications, gzip, headerRow, timeSort, update, honorDst, + const { timeZone, minVal, maxVal, minDate, maxDate, maxError, disableChecks, + cumulative, cumulativeReset, duplications, gzip, headerRow, timeSort, update, honorDst, refreshReadings, relaxedParsing, useMeterZone, warnOnCumulativeReset } = req.body; // Set default values of not supplied parameters. + if (timeZone === undefined) { + req.body.timeZone = DEFAULTS.readings.timeZone; + } + if (minVal === undefined) { + req.body.minVal = DEFAULTS.readings.minVal; + } + if (maxVal === undefined) { + req.body.maxVal = DEFAULTS.readings.maxVal; + } + if (minDate === undefined) { + req.body.minDate = DEFAULTS.readings.minDate; + } + if (maxDate === undefined) { + req.body.maxDate = DEFAULTS.readings.maxDate; + } + if (maxError === undefined) { + req.body.maxError = DEFAULTS.readings.maxError; + } + if (disableChecks === undefined) { + req.body.disableChecks = DEFAULTS.readings.disableChecks; + } if (cumulative === undefined) { req.body.cumulative = DEFAULTS.readings.cumulative; } diff --git a/src/server/services/pipeline-in-progress/loadArrayInput.js b/src/server/services/pipeline-in-progress/loadArrayInput.js index cb81787937..5e82b4a7c7 100644 --- a/src/server/services/pipeline-in-progress/loadArrayInput.js +++ b/src/server/services/pipeline-in-progress/loadArrayInput.js @@ -30,16 +30,17 @@ const processData = require('./processData'); * @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false. * Should only be true if honorDST is true and reading does not have proper time zone information. * @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false. + * @param {string} timeZone optional timezone override provided by the upload request. * @returns {object[]} {whether readings were all process (true) or false, all the messages from processing the readings as a string} */ async function loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition, isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly, - shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false) { + shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false, timeZone = undefined) { // Get the reading, then process them for acceptance and finally insert into the DB. readingsArray = dataRows.map(mapRowToModel); let { result: readingsToInsert, isAllReadingsOk, msgTotal } = await processData(readingsArray, meterID, timeSort, readingRepetition, isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly, - conditionSet, conn, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset); + conditionSet, conn, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone); if (shouldUpdate) { // New readings should replace old ones. await Reading.insertOrUpdateAll(readingsToInsert, conn) diff --git a/src/server/services/pipeline-in-progress/loadCsvInput.js b/src/server/services/pipeline-in-progress/loadCsvInput.js index 27a08d7f83..cfc9f964a4 100644 --- a/src/server/services/pipeline-in-progress/loadCsvInput.js +++ b/src/server/services/pipeline-in-progress/loadCsvInput.js @@ -32,6 +32,7 @@ const { log } = require('../../log'); * @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false. * Should only be true if honorDST is true and reading does not have proper time zone information. * @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false. + * @param {string} timeZone optional timezone override provided by the upload request. */ async function loadCsvInput( filePath, @@ -53,14 +54,15 @@ async function loadCsvInput( honorDst = false, relaxedParsing = false, useMeterZone = false, - warnOnCumulativeReset = false + warnOnCumulativeReset = false, + timeZone = undefined ) { try { const dataRows = await readCsv(filePath, headerRow); return loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition, isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly, shouldUpdate, conditionSet, conn, - honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset); + honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone); } catch (err) { log.error(`Error updating meter ${meterID} with data from ${filePath}: ${err}`, err); } diff --git a/src/server/services/pipeline-in-progress/processData.js b/src/server/services/pipeline-in-progress/processData.js index d919812136..878fad4143 100644 --- a/src/server/services/pipeline-in-progress/processData.js +++ b/src/server/services/pipeline-in-progress/processData.js @@ -65,7 +65,8 @@ const E0 = moment(0).utc() */ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increasing, readingRepetition, isCumulative, cumulativeReset, resetStart = '00:00:00.000', resetEnd = '23:59:99.999', readingGap = 0, readingLengthVariation = 0, isEndTime = false, - conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false, useMeterFrequency = false, useMeterFrequencyVariation = 0) { + conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false, + timeZone = undefined, useMeterFrequency = false, useMeterFrequencyVariation = 0) { // Holds all the warning message to pass back to inform user. // Note they use basic HTML because the messages can be long/complex and it was felt it would be easy to put it into a web browser // to make them easier to read. @@ -125,7 +126,7 @@ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increa // These only happen if worried about DST. if (honorDst) { // Get the meter timezone since the same while processing this data. - meterZone = await meterTimezone(meter); + meterZone = (timeZone !== undefined && timeZone !== '') ? timeZone : await meterTimezone(meter); // See if were processing a shift from DST (inDst) when last batch of readings ended so need to continue. prevEndTimestamp = moment.parseZone(meter.previousEnd, true); if (!isFirst(prevEndTimestamp)) { From 14dbf2eb9c2cd41e2ca65212955095e672630a46 Mon Sep 17 00:00:00 2001 From: Joey603 Date: Thu, 23 Apr 2026 22:31:39 -0700 Subject: [PATCH 3/5] chore(pr-1613):address maintainer review feedback regarding CSV readings overrides --- src/client/app/utils/csvUploadDefaults.ts | 16 ++++++++-------- .../csvPipeline/validateCsvUploadParams.js | 10 ++-------- .../pipeline-in-progress/loadArrayInput.js | 5 +++-- .../pipeline-in-progress/loadCsvInput.js | 2 +- .../services/pipeline-in-progress/processData.js | 1 + 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/client/app/utils/csvUploadDefaults.ts b/src/client/app/utils/csvUploadDefaults.ts index f583206c84..f8d5bcfd2d 100644 --- a/src/client/app/utils/csvUploadDefaults.ts +++ b/src/client/app/utils/csvUploadDefaults.ts @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { DisableChecksType } from 'types/redux/units'; +import { DisableChecksType } from '../types/redux/units'; import { ReadingsCSVUploadPreferences, MetersCSVUploadPreferences } from '../types/csvUploadForm'; import { MeterTimeSortType } from '../types/redux/meters'; @@ -27,13 +27,13 @@ export const ReadingsCSVUploadDefaults: ReadingsCSVUploadPreferences = { update: false, useMeterZone: false, warnOnCumulativeReset: false, - timeZone: "", - minVal: "", - maxVal: "", - minDate: "", - maxDate: "", - maxError: "", - disableChecks: DisableChecksType.reject_all, + timeZone: '', + minVal: '', + maxVal: '', + minDate: '', + maxDate: '', + maxError: '', + disableChecks: DisableChecksType.reject_all }; export const MetersCSVUploadDefaults: MetersCSVUploadPreferences = { diff --git a/src/server/services/csvPipeline/validateCsvUploadParams.js b/src/server/services/csvPipeline/validateCsvUploadParams.js index daccf7e720..f12cec1c78 100644 --- a/src/server/services/csvPipeline/validateCsvUploadParams.js +++ b/src/server/services/csvPipeline/validateCsvUploadParams.js @@ -7,6 +7,7 @@ const { CSVPipelineError } = require('./CustomErrors'); const { Param, EnumParam, BooleanParam, StringParam } = require('./ValidationSchemas'); const failure = require('./failure'); const validate = require('jsonschema').validate; +const Unit = require('../../models/Unit'); // This is only used for meter page inputs but put here so next one above that related to. /** @@ -19,13 +20,6 @@ MeterTimeSortTypesJS = Object.freeze({ decreasing: 'decreasing', }); -DisableChecksTypesJS = Object.freeze({ - reject_disabled: 'reject_disabled', - reject_bad: 'reject_bad', - reject_all: 'reject_all', - reject_none: 'reject_none' -}); - // This function allows for curl users to continue to use 'yes' or 'no' and also allows string // values of true or false if the change is made. const normalizeBoolean = (input) => { @@ -135,7 +129,7 @@ const VALIDATION = { minDate: new StringParam('minDate', undefined, undefined), maxDate: new StringParam('maxDate', undefined, undefined), maxError: new StringParam('maxError', undefined, undefined), - disableChecks: new EnumParam('disableChecks', Object.values(DisableChecksTypesJS)), + disableChecks: new EnumParam('disableChecks', Object.values(Unit.disableChecksType)), cumulative: new EnumParam('cumulative', BooleanCheckArray), cumulativeReset: new EnumParam('cumulativeReset', BooleanCheckArray), cumulativeResetStart: new StringParam('cumulativeResetStart', undefined, undefined), diff --git a/src/server/services/pipeline-in-progress/loadArrayInput.js b/src/server/services/pipeline-in-progress/loadArrayInput.js index 5e82b4a7c7..b7b874e322 100644 --- a/src/server/services/pipeline-in-progress/loadArrayInput.js +++ b/src/server/services/pipeline-in-progress/loadArrayInput.js @@ -30,12 +30,13 @@ const processData = require('./processData'); * @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false. * Should only be true if honorDST is true and reading does not have proper time zone information. * @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false. - * @param {string} timeZone optional timezone override provided by the upload request. + * @param {string} timeZone timezone to use while processing data, default is undefined. * @returns {object[]} {whether readings were all process (true) or false, all the messages from processing the readings as a string} */ async function loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition, isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly, - shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false, timeZone = undefined) { + shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = + false, warnOnCumulativeReset = false, timeZone = undefined) { // Get the reading, then process them for acceptance and finally insert into the DB. readingsArray = dataRows.map(mapRowToModel); let { result: readingsToInsert, isAllReadingsOk, msgTotal } = await processData(readingsArray, meterID, timeSort, readingRepetition, diff --git a/src/server/services/pipeline-in-progress/loadCsvInput.js b/src/server/services/pipeline-in-progress/loadCsvInput.js index cfc9f964a4..f94986e896 100644 --- a/src/server/services/pipeline-in-progress/loadCsvInput.js +++ b/src/server/services/pipeline-in-progress/loadCsvInput.js @@ -32,7 +32,7 @@ const { log } = require('../../log'); * @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false. * Should only be true if honorDST is true and reading does not have proper time zone information. * @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false. - * @param {string} timeZone optional timezone override provided by the upload request. + * @param {string} timeZone timezone to use while processing data, default is undefined. */ async function loadCsvInput( filePath, diff --git a/src/server/services/pipeline-in-progress/processData.js b/src/server/services/pipeline-in-progress/processData.js index 878fad4143..dc58a4e370 100644 --- a/src/server/services/pipeline-in-progress/processData.js +++ b/src/server/services/pipeline-in-progress/processData.js @@ -53,6 +53,7 @@ const E0 = moment(0).utc() * Should only be true if honorDST is true and reading does not have proper time zone information. This feature is not great and should * be avoided except in special circumstances. * @param {boolean} warnOnCumulativeReset true if each cumulative reset generates a warning message and false if not. Default is false. + * @param {string} timeZone timezone to use while processing data, default is undefined. * @param {boolean} useMeterFrequency true if isEndTime is true then any reading found with a different reading length that is longer than the meter * frequency will make the start time by the end time minus the meter reading frequency. The idea is that a change in the length represents * missing reading(s) then it will have a longer time but that is not what is desired for this meter. This only happens if the length of the reading From 75eec8f61923826056cccb9eb7bc7e6c1e74be53 Mon Sep 17 00:00:00 2001 From: Joey603 Date: Mon, 27 Apr 2026 14:18:40 -0700 Subject: [PATCH 4/5] chore(pr-1613): address maintainer review feedback for CSV readings upload Reorder frontend fields to match meter page; fix JSX and restore missing imports/handlers. Move new validation fields to end of schema; deduplicate disableChecks enum and standardize JSDoc for timezone. Add FormData null/undefined guard and fix import path in csvUploadDefaults. Add follow-up comment in loadArrayInput.js recommending options-object refactor for trailing args. --- .../csv/ReadingsCSVUploadComponent.tsx | 88 ++++++++----------- .../csvPipeline/validateCsvUploadParams.js | 78 ++++++++-------- .../pipeline-in-progress/loadArrayInput.js | 13 ++- 3 files changed, 86 insertions(+), 93 deletions(-) diff --git a/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx b/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx index d36ee910b8..2f797db90d 100644 --- a/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx +++ b/src/client/app/components/csv/ReadingsCSVUploadComponent.tsx @@ -187,13 +187,6 @@ export default function ReadingsCSVUploadComponent() { lengthVariation: selectedMeter.readingVariation, endOnly: selectedMeter.endOnlyTime, timeSort: MeterTimeSortType[selectedMeter.timeSort as keyof typeof MeterTimeSortType], - timeZone: selectedMeter.timeZone ?? '', - minVal: selectedMeter.minVal.toString(), - maxVal: selectedMeter.maxVal.toString(), - minDate: selectedMeter.minDate, - maxDate: selectedMeter.maxDate, - maxError: selectedMeter.maxError.toString(), - disableChecks: selectedMeter.disableChecks, useMeterZone: false, warnOnCumulativeReset: false })); @@ -299,16 +292,9 @@ export default function ReadingsCSVUploadComponent() { || readingsData.refreshReadings !== ReadingsCSVUploadDefaults.refreshReadings || readingsData.relaxedParsing !== ReadingsCSVUploadDefaults.relaxedParsing || readingsData.timeSort !== ReadingsCSVUploadDefaults.timeSort - || readingsData.timeZone !== ReadingsCSVUploadDefaults.timeZone - || readingsData.minVal !== ReadingsCSVUploadDefaults.minVal - || readingsData.maxVal !== ReadingsCSVUploadDefaults.maxVal - || readingsData.minDate !== ReadingsCSVUploadDefaults.minDate - || readingsData.maxDate !== ReadingsCSVUploadDefaults.maxDate - || readingsData.maxError !== ReadingsCSVUploadDefaults.maxError - || readingsData.disableChecks !== ReadingsCSVUploadDefaults.disableChecks || readingsData.update !== ReadingsCSVUploadDefaults.update - || readingsData.useMeterZone !== ReadingsCSVUploadDefaults.useMeterZone - || readingsData.warnOnCumulativeReset !== ReadingsCSVUploadDefaults.warnOnCumulativeReset + || readingsData.useMeterZone !== readingsData.useMeterZone + || readingsData.warnOnCumulativeReset !== readingsData.warnOnCumulativeReset // If any file is added, it will count as edit made. || selectedFile !== null || invalidFileEntry === true; @@ -695,8 +681,6 @@ export default function ReadingsCSVUploadComponent() { - - - - - - {handleChange(e);}} - > - {Object.values(DisableChecksType).map(disableChecks => { - return ( - - ); - })} - - - @@ -815,6 +777,28 @@ export default function ReadingsCSVUploadComponent() { /> + + + + {handleChange(e);}} + > + {Object.values(DisableChecksType).map(disableChecks => { + return ( + + ); + })} + + + {/* TODO This feature is not working perfectly so disabling from web page but allowing in curl. Rest of changes left so easy to add back in. @@ -823,7 +807,7 @@ export default function ReadingsCSVUploadComponent() { originally added to the web page input of import but decided to not allow it at this time. Thus, the code was commented out. As of now there is no plan to make this generally available due to its limitations.*/} - {/* + {/** */} - {/* + {/** */} @@ -863,11 +847,11 @@ export default function ReadingsCSVUploadComponent() {
- - - - )} - - - ); -} + + + + )} + + + ); + } diff --git a/src/server/services/csvPipeline/validateCsvUploadParams.js b/src/server/services/csvPipeline/validateCsvUploadParams.js index f12cec1c78..0daa593d46 100644 --- a/src/server/services/csvPipeline/validateCsvUploadParams.js +++ b/src/server/services/csvPipeline/validateCsvUploadParams.js @@ -65,13 +65,6 @@ const DEFAULTS = { meters: { }, readings: { - timeZone: undefined, - minVal: undefined, - maxVal: undefined, - minDate: undefined, - maxDate: undefined, - maxError: undefined, - disableChecks: undefined, cumulative: undefined, cumulativeReset: undefined, cumulativeResetStart: undefined, @@ -85,7 +78,14 @@ const DEFAULTS = { relaxedParsing: false, timeSort: undefined, useMeterZone: false, - warnOnCumulativeReset: false + warnOnCumulativeReset: false, + timeZone: undefined, + minVal: undefined, + maxVal: undefined, + minDate: undefined, + maxDate: undefined, + maxError: undefined, + disableChecks: undefined } } @@ -123,13 +123,6 @@ const VALIDATION = { type: 'object', properties: { ...COMMON_PROPERTIES, - timeZone: new StringParam('timeZone', undefined, undefined), - minVal: new StringParam('minVal', undefined, undefined), - maxVal: new StringParam('maxVal', undefined, undefined), - minDate: new StringParam('minDate', undefined, undefined), - maxDate: new StringParam('maxDate', undefined, undefined), - maxError: new StringParam('maxError', undefined, undefined), - disableChecks: new EnumParam('disableChecks', Object.values(Unit.disableChecksType)), cumulative: new EnumParam('cumulative', BooleanCheckArray), cumulativeReset: new EnumParam('cumulativeReset', BooleanCheckArray), cumulativeResetStart: new StringParam('cumulativeResetStart', undefined, undefined), @@ -144,6 +137,13 @@ const VALIDATION = { timeSort: new EnumParam('timeSort', [MeterTimeSortTypesJS.increasing, MeterTimeSortTypesJS.decreasing]), useMeterZone: new EnumParam('useMeterZone', BooleanCheckArray), warnOnCumulativeReset: new EnumParam('warnOnCumulativeReset', BooleanCheckArray), + timeZone: new StringParam('timeZone', undefined, undefined), + minVal: new StringParam('minVal', undefined, undefined), + maxVal: new StringParam('maxVal', undefined, undefined), + minDate: new StringParam('minDate', undefined, undefined), + maxDate: new StringParam('maxDate', undefined, undefined), + maxError: new StringParam('maxError', undefined, undefined), + disableChecks: new EnumParam('disableChecks', Object.values(Unit.disableChecksType)) }, anyOf: [ { required: ['meterIdentifier'] }, @@ -197,32 +197,11 @@ function validateReadingsCsvUploadParams(req, res, next) { } // extract query parameters - const { timeZone, minVal, maxVal, minDate, maxDate, maxError, disableChecks, - cumulative, cumulativeReset, duplications, gzip, headerRow, timeSort, update, honorDst, - refreshReadings, relaxedParsing, useMeterZone, warnOnCumulativeReset } = req.body; + const { cumulative, cumulativeReset, duplications, gzip, headerRow, timeSort, update, honorDst, + refreshReadings, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone, minVal, + maxVal, minDate, maxDate, maxError, disableChecks } = req.body; // Set default values of not supplied parameters. - if (timeZone === undefined) { - req.body.timeZone = DEFAULTS.readings.timeZone; - } - if (minVal === undefined) { - req.body.minVal = DEFAULTS.readings.minVal; - } - if (maxVal === undefined) { - req.body.maxVal = DEFAULTS.readings.maxVal; - } - if (minDate === undefined) { - req.body.minDate = DEFAULTS.readings.minDate; - } - if (maxDate === undefined) { - req.body.maxDate = DEFAULTS.readings.maxDate; - } - if (maxError === undefined) { - req.body.maxError = DEFAULTS.readings.maxError; - } - if (disableChecks === undefined) { - req.body.disableChecks = DEFAULTS.readings.disableChecks; - } if (cumulative === undefined) { req.body.cumulative = DEFAULTS.readings.cumulative; } @@ -259,6 +238,27 @@ function validateReadingsCsvUploadParams(req, res, next) { if (warnOnCumulativeReset === undefined) { req.body.warnOnCumulativeReset = DEFAULTS.readings.warnOnCumulativeReset; } + if (timeZone === undefined) { + req.body.timeZone = DEFAULTS.readings.timeZone; + } + if (minVal === undefined) { + req.body.minVal = DEFAULTS.readings.minVal; + } + if (maxVal === undefined) { + req.body.maxVal = DEFAULTS.readings.maxVal; + } + if (minDate === undefined) { + req.body.minDate = DEFAULTS.readings.minDate; + } + if (maxDate === undefined) { + req.body.maxDate = DEFAULTS.readings.maxDate; + } + if (maxError === undefined) { + req.body.maxError = DEFAULTS.readings.maxError; + } + if (disableChecks === undefined) { + req.body.disableChecks = DEFAULTS.readings.disableChecks; + } next(); } diff --git a/src/server/services/pipeline-in-progress/loadArrayInput.js b/src/server/services/pipeline-in-progress/loadArrayInput.js index b7b874e322..1bb8981494 100644 --- a/src/server/services/pipeline-in-progress/loadArrayInput.js +++ b/src/server/services/pipeline-in-progress/loadArrayInput.js @@ -33,10 +33,19 @@ const processData = require('./processData'); * @param {string} timeZone timezone to use while processing data, default is undefined. * @returns {object[]} {whether readings were all process (true) or false, all the messages from processing the readings as a string} */ +// NOTE (follow-up): Callers of this function sometimes omit optional trailing +// parameters (for example `timeZone`). This works because JS allows omitted +// trailing args, but it makes call sites inconsistent and harder to maintain. +// Suggested follow-up: migrate to a single `options` object (e.g. +// `loadArrayInput(dataRows, meterID, mapRowToModel, opts)`) or mandate +// explicitly passing all arguments. Do NOT remove `timeZone` or other +// parameters here in this PR — perform a backward-compatible refactor in a +// separate change to avoid regressions. + async function loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition, isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly, - shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = - false, warnOnCumulativeReset = false, timeZone = undefined) { + shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, + warnOnCumulativeReset = false, timeZone = undefined) { // Get the reading, then process them for acceptance and finally insert into the DB. readingsArray = dataRows.map(mapRowToModel); let { result: readingsToInsert, isAllReadingsOk, msgTotal } = await processData(readingsArray, meterID, timeSort, readingRepetition, From 25b53932f4b122bbd240a896d941968653b80a9d Mon Sep 17 00:00:00 2001 From: Joey603 Date: Fri, 1 May 2026 15:30:42 -0700 Subject: [PATCH 5/5] chore(dev-setup): stabilize local docker startup so oed-web-1 reaches healthy state --- docker-compose.yml | 9 ++++----- src/scripts/installOED.sh | 16 +++++++++++++--- src/server/db.js | 2 +- src/server/log.js | 27 ++++++++++++++++++++------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b0d1b69d9e..96e2774d4e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,6 @@ # * file, You can obtain one at http://mozilla.org/MPL/2.0/. # * -version: "3.8" services: # Database service. It's PostgreSQL, see the # Dockerfile in ./database @@ -21,8 +20,8 @@ services: interval: 10s timeout: 10s retries: 3 - # ports: - # - "5432:5432" + ports: + - "5432:5432" # Uncomment the above lines to enable access to the PostgreSQL server # from the host machine. # Web service runs Node @@ -33,9 +32,9 @@ services: - OED_SERVER_PORT=3000 - OED_DB_USER=oed - OED_DB_DATABASE=oed - - OED_DB_TEST_DATABASE=oed_testing + - OED_DB_TEST_DATABASE=oed - OED_DB_PASSWORD=opened - - OED_DB_HOST=database # Docker will set this hostname + - OED_DB_HOST=database # Docker service name inside the compose network - OED_DB_PORT=5432 - OED_TOKEN_SECRET=? - OED_LOG_FILE=log.txt diff --git a/src/scripts/installOED.sh b/src/scripts/installOED.sh index 1bef1b7a5b..be7f0d6421 100755 --- a/src/scripts/installOED.sh +++ b/src/scripts/installOED.sh @@ -52,10 +52,20 @@ while test $# -gt 0; do esac done -# Load .env if it exists - +# Load .env if it exists, but do not overwrite values already provided by the +# runtime environment (for example from Docker Compose). if [ -f ".env" ]; then - source .env + while IFS='=' read -r name value; do + case "$name" in + ''|'#'*) + continue + ;; + esac + + if [ -z "${!name+x}" ]; then + export "$name=$value" + fi + done < .env fi # Skip the install if the node_modules were installed before the package files. diff --git a/src/server/db.js b/src/server/db.js index 9f356e386b..db5631dc97 100644 --- a/src/server/db.js +++ b/src/server/db.js @@ -67,7 +67,7 @@ function swapConnection(newConfig, newConnection) { if (newConnection !== null) { connmanager.connection = newConnection; } else { - connmanager = getDB(connmanager.config); + connmanager.connection = getDB(connmanager.config); } } diff --git a/src/server/log.js b/src/server/log.js index 2fbfdc4e24..ca66981e09 100644 --- a/src/server/log.js +++ b/src/server/log.js @@ -4,11 +4,22 @@ const fs = require('fs'); const logFile = require('./config').logFile; -const LogEmail = require('./models/LogEmail'); -const LogMsg = require('./models/LogMsg'); -const { getConnection } = require('./db'); const moment = require('moment'); +/** + * Get a database connection if the database module is available. + * Logging should still work even if the database is unavailable, so this + * intentionally falls back to null instead of throwing. + * @returns {object|null} + */ +function getConnection() { + try { + return require('./db').getConnection(); + } catch (err) { + return null; + } +} + /** * Represents the importance of a message to be logged */ @@ -55,6 +66,8 @@ class Logger { let messageToLog = `[${level.name}@${logTime.format('YYYY-MM-DDTHH:mm:ss.SSSZ')}] ${message}\n`; const conn = getConnection(); + const LogEmail = require('./models/LogEmail'); + const LogMsg = require('./models/LogMsg'); // Add a stacktrace to the message if one was provided. if (error !== null) { @@ -169,10 +182,10 @@ const defaultLogger = new Logger(logFile); * Wherever logging is available, the Node.js runtime will call this function to log unhandled rejections. * This helps with debugging, especially in tests. */ -process.on('unhandledRejection', (reason, p) => { - p.catch(e => { - defaultLogger.error(`Unhandled Promise Rejection: ${reason}`, e); - }); + +process.on('unhandledRejection', (reason) => { + const message = reason instanceof Error ? reason.message : String(reason); + defaultLogger.error(`Unhandled Promise Rejection: ${message}`, reason instanceof Error ? reason : null); }); defaultLogger.logToDb = true;