Worked on Issue 1307

src/server/services/pipeline-in-progress/processData.js

@@ -660,17 +660,31 @@ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increa
 		prevEndTimestampTz = endTimestampTz;
 	}
 	// Validate data if conditions given
-	if (conditionSet !== undefined && !conditionSet['disableChecks']) {
-		const { validReadings, errMsg: newErrMsg } = validateReadings(result, conditionSet, meterName);
-		({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, newErrMsg, msgTotalWarning));
-		if (!validReadings) {
-			errMsg = `<h2>For meter ${meterName}: error when validating data so all reading are rejected</h2>`;
-			log.error(errMsg);
-			({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, errMsg, msgTotalWarning));
-			// This empties the result array. Should be fast and okay with const.
-			result.splice(0, result.length);
-			isAllReadingsOk = false;
-			return { result, isAllReadingsOk, msgTotal };
+	if (conditionSet !== undefined) {
+		const disableChecks = conditionSet['disableChecks'];
+
+		if (disableChecks != 'yes_no_checks') {
+			const { validReadings, errMsg: newErrMsg } = validateReadings(result, conditionSet, meterName);
+			({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, newErrMsg, msgTotalWarning));}
+
+
+			if (!validReadings) {
+				if (disableChecks === 'no_reject_all'){
+				errMsg = `<h2>For meter ${meterName}: error when validating data so all reading are rejected</h2>`;
+				log.error(errMsg);
+				({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, errMsg, msgTotalWarning));
+				// This empties the result array. Should be fast and okay with const.
+				result.splice(0, result.length);
+				isAllReadingsOk = false;
+				return { result, isAllReadingsOk, msgTotal };
+				} else if (disableChecks === 'no_reject_bad') {
+					errMsg = '<h2>For meter ${meterName}: error when validating some readings, but valid readings are accepted</h2>';
+					log.warn(errMsg);
+					({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, errMsg, msgTotalWarning));
+					// only keep valid readings, leave the invalid ones out of the result
+					isAllReadingsOk = false;
+				}
+			}
 		}
 	}
 	// Update the meter to contain information for the last reading in the data file.

src/server/services/pipeline-in-progress/validateReadings.js

@@ -12,13 +12,17 @@ const { log } = require('../../log');
  * @param {Reading[]} arrayToValidate
  * @param {dict} conditionSet used to validate readings (minVal, maxVal, minDate, maxDate, threshold, maxError)
  * @param {string} meterIdentifier identifier of meter being checked
+ * @returns {object} { validReadings, invalidReadings, errMsg }
  */
 function validateReadings(arrayToValidate, conditionSet, meterIdentifier = undefined) {
 	/* tslint:disable:no-string-literal */
 	const { validDates, errMsg: errMsgDate } = checkDate(arrayToValidate, conditionSet['minDate'], conditionSet['maxDate'], conditionSet['maxError'] / 2, meterIdentifier);
 	const { validValues, errMsg: errMsgValue } = checkValue(arrayToValidate, conditionSet['minVal'], conditionSet['maxVal'], conditionSet['maxError'] / 2, meterIdentifier);
 	/* tslint:enable:no-string-literal */
+	const validReadings = validDates.filter(reading => validValues.includes(reading));
+	const invalidReadings = [...invalidDates, ...invalidValues];
 	const errMsg = errMsgDate + errMsgValue;
+
 	return {
 		validReadings: validDates && validValues,
 		errMsg,
@@ -32,14 +36,18 @@ function validateReadings(arrayToValidate, conditionSet, meterIdentifier = undef
  * @param {Moment} maxDate inclusive latest acceptable date (won't be rejected)
  * @param {number} maxError maximum number of errors to be reported, ignore the rest
  * @param {string} meterIdentifier identifier of meter being checked.
+ * @returns {object} { validDates, invalidDates, errMsg }
  */
 function checkDate(arrayToValidate, minDate, maxDate, maxError, meterIdentifier) {
-	let validDates = true;
+	let validDates = [];
+	let invalidDates = [];
 	let errMsg = '';
+	let readingNumber = 0;
+
 	if (minDate === null && maxDate === null) {
-		return { validDates, errMsg };
+		return { validDates: arrayToValidate, invalidDates: [], errMsg };
 	}
-	let readingNumber = 0;
+
 	for (const reading of arrayToValidate) {
 		readingNumber++;
 		if (maxError <= 0) {
@@ -53,18 +61,20 @@ function checkDate(arrayToValidate, minDate, maxDate, maxError, meterIdentifier)
 			errMsg += '<br>' + newErrMsg + '<br>';
 			--maxError;
 			validDates = false;
-		}
-		if (reading.endTimestamp > maxDate) {
+		} else if (reading.endTimestamp > maxDate) {
 			const newErrMsg = `error when checking reading time for #${readingNumber} on meter ${meterIdentifier}: ` +
 			`time ${reading.endTimestamp} is later than upper bound ${maxDate} ` +
 			`with reading ${reading.reading} and startTimestamp ${reading.startTimestamp}`;
 			log.error(newErrMsg);
 			errMsg += '<br>' + newErrMsg + '<br>';
+			invalidDates.push(reading);
 			--maxError;
-			validDates = false;
+		} else {
+			validDates.push(reading);
 		}
 	}
-	return { validDates, errMsg };
+
+	return { validDates, invalidDates, errMsg };
 }
 
 /**
@@ -74,11 +84,14 @@ function checkDate(arrayToValidate, minDate, maxDate, maxError, meterIdentifier)
  * @param {number} maxVal inclusive maximum acceptable reading value (won't be rejected)
  * @param {number} maxError maximum number of errors to be reported, ignore the rest
  * @param {string} meterIdentifier identifier of meter being checked.
+ * @returns {object} { validValues, invalidValues, errMsg }
  */
 function checkValue(arrayToValidate, minVal, maxVal, maxError, meterIdentifier) {
 	let validValues = true;
+	let invalidValues = [];
 	let errMsg = '';
 	let readingNumber = 0;
+
 	for (const reading of arrayToValidate) {
 		readingNumber++;
 		if (maxError <= 0) {
@@ -98,11 +111,14 @@ function checkValue(arrayToValidate, minVal, maxVal, maxError, meterIdentifier)
 			`with startTimestamp ${reading.startTimestamp} and endTimestamp ${reading.endTimestamp}`;
 			log.error(newErrMsg);
 			errMsg += '<br>' + newErrMsg + '<br>';
+			invalidValues.push(reading);
 			--maxError;
-			validValues = false;
+		} else {
+			validValues.push(reading);
 		}
 	}
-	return { validValues, errMsg };
+
+	return { validValues, invalidValues, errMsg };
 }
 
 /**

src/server/sql/preferences/create_preferences_table.sql

@@ -2,6 +2,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+CREATE TYPE disable_checks_enum AS ENUM (
+	'no_reject_bad',
+	'no_reject_all',
+	'yes_no_checks'
+);
 -- create preferences table
 CREATE TABLE IF NOT EXISTS preferences (
 	id SERIAL PRIMARY KEY,
@@ -21,6 +26,6 @@ CREATE TABLE IF NOT EXISTS preferences (
     default_meter_maximum_date TIMESTAMP NOT NULL,
 	default_meter_reading_gap REAL NOT NULL,
     default_meter_maximum_errors INTEGER NOT NULL,
-	default_meter_disable_checks BOOLEAN NOT NULL,
+	default_meter_disable_checks disable_checks_enum NOT NULL,
 	default_help_url TEXT DEFAULT NULL
 );

src/server/sql/preferences/insert_default_row.sql

@@ -21,7 +21,7 @@ IF NOT EXISTS(SELECT *
 	default_meter_disable_checks, default_help_url) 
 	VALUES ('', 'line', FALSE, 'en', NULL, 5, 25, FALSE, 'meters', '00:15:00', 
 	-9007199254740991, 9007199254740991, '1970-01-01 00:00:00+00:00', '6970-01-01 00:00:00+00:00',
-	0, 75, FALSE, 'https://openenergydashboard.github.io/');
+	0, 75, 'no_reject_all', 'https://openenergydashboard.github.io/');
 END IF ;
 
 END;

src/server/test/db/validateReadingsTests.js

@@ -27,12 +27,15 @@ mocha.describe('PIPELINE: Validate Readings', () => {
 		)).map(reading => checkDate([reading], minDate, maxDate, Number.MAX_VALUE));
 		for (let i = 0; i < 4; ++i) {
 			if (i % 2 === 0) {
-				expect(results[i].validDates).to.equal(true);
+				expect(results[i].validDates.length).to.be.greaterThan(0);
+				expect(results[i].invalidDates.length).to.equal(0);
 			} else {
-				expect(results[i].validDates).to.equal(false);
+				expect(results[i].validDates.length).to.equal(0);
+				expect(results[i].invalidDates.length).to.be.greaterThan(0);
 			}
 		}
 	});
+
 	mocha.it('detects out-of-bound data', async () => {
 		const minVal = 10;
 		const maxVal = 20;
@@ -46,9 +49,11 @@ mocha.describe('PIPELINE: Validate Readings', () => {
 		)).map(reading => checkValue([reading], minVal, maxVal, Number.MAX_VALUE));
 		for (let i = 0; i < 7; ++i) {
 			if (i % 2 === 0) {
-				expect(results[i].validValues).to.equal(false);
+				expect(results[i].validValues.length).to.equal(0);
+				expect(results[i].invalidValues.length).to.be.greaterThan(0);
 			} else {
-				expect(results[i].validValues).to.equal(true);
+				expect(results[i].validValues.length).to.be.greaterThan(0);
+				expect(results[i].invalidValues.length).to.equal(0);
 			}
 		}
 	});
@@ -92,10 +97,26 @@ mocha.describe('PIPELINE: Validate Readings', () => {
 						new Reading(undefined, 20, moment('1970-01-01 00:01:00'), moment('1970-01-01 00:01:01')),
 						new Reading(undefined, 0, moment('1970-01-01 00:01:30'), moment('1970-01-01 00:02:01'))];
 
+		let mixedData = [
+			new Reading(undefined, 0, moment('1970-01-01 00:00:00'), moment('1970-01-01 00:01:00')), // valid
+			new Reading(undefined, 30, moment('1970-01-01 00:01:00'), moment('1970-01-01 00:02:00')), // invalid (value too high)
+			new Reading(undefined, 0, moment('1969-01-01 00:00:00'), moment('1969-01-01 00:01:00')), // invalid (date too early)
+			new Reading(undefined, 10, moment('1980-01-01 00:00:00'), moment('1980-01-01 01:00:00'))  // valid
+		];
+
+		const result = validateReadings(mixedData, conditionSet);
+
 		expect(checkIntervals(badIntervals, conditionSet['threshold']).validIntervals).to.equal(false);
 		expect(validateReadings(badDate, conditionSet).validReadings).to.equal(false);
 		expect(validateReadings(badValue, conditionSet).validReadings).to.equal(false);
 		expect(validateReadings(goodData, conditionSet).validReadings).to.equal(true);
 
+		expect(result.validReadings.length).to.equal(2);
+		expect(result.validReadings[0].reading).to.equal(0);
+		expect(result.validReadings[1].reading).to.equal(10);
+
+		expect(result.invalidReadings.length).to.equal(2);
+		expect(result.invalidReadings[0].reading).to.equal(30);
+		expect(result.invalidReadings[1].startTimestamp.isBefore('1970-01-01 00:00:00')).to.equal(true);
 	});
 });