From 616ce6d146ff2fd2fbcb8c8b858a6859be1fe4fe Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Wed, 8 Nov 2023 13:00:13 +0100 Subject: [PATCH 01/20] Improving column name validation --- .../exceptions/FactoryException.java | 2 +- .../edu/ie3/datamodel/io/factory/Factory.java | 83 +++++++++++++++---- .../input/AssetInputEntityFactory.java | 8 +- .../input/ConnectorInputEntityFactory.java | 2 +- .../input/CylindricalStorageInputFactory.java | 8 +- .../io/factory/input/LineInputFactory.java | 4 +- .../input/MeasurementUnitInputFactory.java | 4 +- .../io/factory/input/NodeInputFactory.java | 8 +- .../factory/input/OperatorInputFactory.java | 2 +- .../input/ThermalHouseInputFactory.java | 4 +- .../input/Transformer2WInputFactory.java | 4 +- .../input/Transformer3WInputFactory.java | 4 +- .../input/graphics/GraphicInputFactory.java | 6 +- .../input/participant/BmInputFactory.java | 6 +- .../input/participant/ChpInputFactory.java | 2 +- .../input/participant/EmInputFactory.java | 4 +- .../input/participant/EvcsInputFactory.java | 8 +- .../participant/FixedFeedInInputFactory.java | 4 +- .../input/participant/LoadInputFactory.java | 8 +- .../input/participant/PvInputFactory.java | 14 ++-- .../SystemParticipantInputEntityFactory.java | 6 +- .../input/participant/WecInputFactory.java | 2 +- .../result/ConnectorResultFactory.java | 17 ++-- .../result/FlexOptionsResultFactory.java | 8 +- .../io/factory/result/NodeResultFactory.java | 6 +- .../factory/result/SwitchResultFactory.java | 2 +- .../SystemParticipantResultFactory.java | 9 +- .../factory/result/ThermalResultFactory.java | 6 +- .../timeseries/CosmoIdCoordinateFactory.java | 10 +-- .../CosmoTimeBasedWeatherValueFactory.java | 10 +-- .../timeseries/IconIdCoordinateFactory.java | 4 +- .../IconTimeBasedWeatherValueFactory.java | 2 +- .../timeseries/SqlIdCoordinateFactory.java | 2 +- .../TimeBasedSimpleValueFactory.java | 18 ++-- .../TimeBasedWeatherValueFactory.java | 2 +- .../timeseries/TimeSeriesMappingFactory.java | 2 +- .../TimeSeriesMetaInformationFactory.java | 2 +- .../typeinput/LineTypeInputFactory.java | 6 +- .../SystemParticipantTypeInputFactory.java | 44 +++++----- .../Transformer2WTypeInputFactory.java | 28 +++---- .../Transformer3WTypeInputFactory.java | 40 ++++----- .../ie3/datamodel/io/source/DataSource.java | 3 +- .../ie3/datamodel/io/source/EntitySource.java | 24 +++--- .../datamodel/io/source/GraphicSource.java | 12 +-- .../datamodel/io/source/RawGridSource.java | 12 +-- .../io/source/ResultEntitySource.java | 2 +- .../datamodel/io/source/SourceValidator.java | 19 +++++ .../io/source/SystemParticipantSource.java | 6 +- .../datamodel/io/source/ThermalSource.java | 9 +- .../io/source/TimeSeriesMappingSource.java | 4 +- .../io/source/csv/CsvDataSource.java | 20 +++-- .../io/source/csv/CsvIdCoordinateSource.java | 3 + .../csv/CsvTimeSeriesMappingSource.java | 5 +- .../io/source/csv/CsvTimeSeriesSource.java | 12 ++- .../io/source/csv/CsvWeatherSource.java | 3 + .../influxdb/InfluxDbWeatherSource.java | 3 +- .../io/source/sql/SqlDataSource.java | 26 +++++- .../io/source/sql/SqlIdCoordinateSource.java | 10 ++- .../sql/SqlTimeSeriesMappingSource.java | 14 +++- .../io/source/sql/SqlTimeSeriesSource.java | 14 +++- .../input/AssetInputEntityFactoryTest.groovy | 41 ++++----- .../Transformer2WInputFactoryTest.groovy | 9 +- .../Transformer3WInputFactoryTest.groovy | 9 +- .../participant/EvcsInputFactoryTest.groovy | 2 +- .../FixedFeedInInputFactoryTest.groovy | 31 +++---- .../FlexOptionsResultFactoryTest.groovy | 21 ++--- .../result/NodeResultFactoryTest.groovy | 19 ++--- .../SystemParticipantResultFactoryTest.groovy | 22 ++--- .../CosmoIdCoordinateFactoryTest.groovy | 41 +++------ .../IconIdCoordinateFactoryTest.groovy | 27 ++---- .../TimeBasedSimpleValueFactoryTest.groovy | 4 +- ...stemParticipantTypeInputFactoryTest.groovy | 35 ++------ .../source/csv/CsvTimeSeriesSourceIT.groovy | 4 +- .../csv/CsvWeatherSourceCosmoTest.groovy | 10 +-- .../io/source/csv/_joint_grid/evcs_input.csv | 2 +- .../csv/_participants/fixed_feed_in_input.csv | 2 +- .../source/csv/_participants/load_input.csv | 2 +- .../io/source/csv/_participants/pv_input.csv | 2 +- .../csv/_participants/time_series_mapping.csv | 8 +- .../source/csv/_results/thermal_house_res.csv | 2 +- .../io/source/csv/_types/bm_type_input.csv | 2 +- .../io/source/csv/_types/chp_type_input.csv | 2 +- .../io/source/csv/_types/ev_type_input.csv | 2 +- .../io/source/csv/_types/hp_type_input.csv | 2 +- .../source/csv/_types/storage_type_input.csv | 2 +- .../io/source/csv/_types/wec_type_input.csv | 2 +- 86 files changed, 488 insertions(+), 429 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java b/src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java index 75c0467f3..0a51c09bf 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.exceptions; -/** Is thrown, when an something went wrong during entity creation process in a EntityFactory */ +/** Is thrown, when something went wrong during entity creation process in a EntityFactory */ public class FactoryException extends RuntimeException { public FactoryException(final String message, final Throwable cause) { super(message, cause); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 0428701ec..1263cc5f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -6,10 +6,11 @@ package edu.ie3.datamodel.io.factory; import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.utils.Try; import edu.ie3.datamodel.utils.Try.*; +import edu.ie3.util.StringUtils; import java.util.*; -import java.util.function.IntFunction; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +24,7 @@ * @param Type of the intended return type (might differ slightly from target class (cf. {@link * edu.ie3.datamodel.io.factory.timeseries.TimeBasedValueFactory})). */ -public abstract class Factory { +public abstract class Factory implements SourceValidator { public static final Logger log = LoggerFactory.getLogger(Factory.class); private final List> supportedClasses; @@ -48,15 +49,10 @@ public List> getSupportedClasses() { public Try get(D data) { isSupportedClass(data.getTargetClass()); - // magic: case-insensitive get/set calls on set strings - final List> allFields = getFields(data); - try { - validateParameters(data, allFields.toArray((IntFunction[]>) Set[]::new)); - // build the model return Success.of(buildModel(data)); - } catch (FactoryException e) { + } catch (Exception e) { return Failure.of( new FactoryException( "An error occurred when creating instance of " @@ -108,10 +104,47 @@ private void isSupportedClass(Class desiredClass) { * Returns list of sets of attribute names that the entity requires to be built. At least one of * these sets needs to be delivered for entity creation to be successful. * - * @param data EntityData (or subclass) containing the data * @return list of possible attribute sets */ - protected abstract List> getFields(D data); + protected abstract List> getFields(Class entityClass); + + /** + * Method for validating the found fields. + * + * @param foundFields that were found + * @param entityClass of the build data + */ + public void validate(Set foundFields, Class entityClass) { + List> fieldSets = + getFields(entityClass).stream().map(Factory::toSnakeCase).toList(); + Set harmonizedFoundFields = toSnakeCase(foundFields); + + // comparing the found fields to a list of possible fields (allows additional fields) + // if not all fields were found in a set, this set is filtered out + // all other fields are saved as a list + // allows snake, camel and mixed cases + List> validFieldSets = + fieldSets.stream() + .filter(s -> foundFields.containsAll(s) || harmonizedFoundFields.containsAll(s)) + .toList(); + + if (validFieldSets.isEmpty()) { + // build the exception string with extensive debug information + String providedKeysString = "[" + String.join(", ", foundFields) + "]"; + + String possibleOptions = getFieldsString(fieldSets).toString(); + + throw new FactoryException( + "The provided fields " + + providedKeysString + + " are invalid for instance of " + + entityClass.getSimpleName() + + ". \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" + + entityClass.getSimpleName() + + "' are possible (NOT case-sensitive!):\n" + + possibleOptions); + } + } /** * Validates the factory specific constructor parameters in two ways. 1) the biggest set of the @@ -148,7 +181,7 @@ protected int validateParameters(D data, Set... fieldSets) { String providedKeysString = "[" + String.join(", ", fieldsToValues.keySet()) + "]"; - String possibleOptions = getFieldsString(fieldSets).toString(); + String possibleOptions = getFieldsString(List.of(fieldSets)).toString(); throw new FactoryException( "The provided fields " @@ -165,11 +198,17 @@ protected int validateParameters(D data, Set... fieldSets) { } } - protected static StringBuilder getFieldsString(Set... fieldSets) { + protected static StringBuilder getFieldsString(List> fieldSets) { StringBuilder possibleOptions = new StringBuilder(); - for (int i = 0; i < fieldSets.length; i++) { - Set fieldSet = fieldSets[i]; - String option = i + ": [" + String.join(", ", fieldSet) + "]\n"; + for (int i = 0; i < fieldSets.size(); i++) { + Set fieldSet = fieldSets.get(i); + String option = + i + + ": [" + + String.join(", ", fieldSet) + + "] or [" + + String.join(", ", toCamelCase(fieldSet)) + + "]\n"; possibleOptions.append(option); } return possibleOptions; @@ -177,7 +216,7 @@ protected static StringBuilder getFieldsString(Set... fieldSets) { /** * Creates a new set of attribute names from given list of attributes. This method should always - * be used when returning attribute sets, i.e. through {@link #getFields(FactoryData)}. + * be used when returning attribute sets, i.e. through {@link #getFields(Class)}. * * @param attributes attribute names * @return new set exactly containing attribute names @@ -203,4 +242,16 @@ protected TreeSet expandSet(Set attributeSet, String... more) { newSet.addAll(Arrays.asList(more)); return newSet; } + + private static Set toSnakeCase(Set set) { + TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + newSet.addAll(set.stream().map(StringUtils::camelCaseToSnakeCase).toList()); + return newSet; + } + + private static Set toCamelCase(Set set) { + TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + newSet.addAll(set.stream().map(StringUtils::snakeCaseToCamelCase).toList()); + return newSet; + } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java index 0da292f37..bfef41520 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java @@ -25,8 +25,8 @@ public abstract class AssetInputEntityFactory { private static final String UUID = "uuid"; - private static final String OPERATES_FROM = "operatesfrom"; - private static final String OPERATES_UNTIL = "operatesuntil"; + private static final String OPERATES_FROM = "operatesFrom"; + private static final String OPERATES_UNTIL = "operatesUntil"; private static final String ID = "id"; protected AssetInputEntityFactory(Class... allowedClasses) { @@ -39,11 +39,11 @@ protected AssetInputEntityFactory(Class... allowedClasses) { *

The mandatory attributes required to create an {@link AssetInput} are enhanced with custom * attribute names that each subclass factory determines in {@link #getAdditionalFields()}. * - * @param data EntityData (or subclass) containing the data + * @param entityClass class of the entity * @return list of possible attribute sets */ @Override - protected List> getFields(D data) { + protected List> getFields(Class entityClass) { Set constructorParamsMin = newSet(UUID, ID); Set constructorParamsFrom = expandSet(constructorParamsMin, OPERATES_FROM); Set constructorParamsUntil = expandSet(constructorParamsMin, OPERATES_UNTIL); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/ConnectorInputEntityFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/ConnectorInputEntityFactory.java index 4ef83352e..cc29c7e5e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/ConnectorInputEntityFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/ConnectorInputEntityFactory.java @@ -29,7 +29,7 @@ abstract class ConnectorInputEntityFactory< * ConnectorInput}s. Thus, this attribute name declaration can be used in subclasses of {@link * ConnectorInputEntityFactory} */ - protected static final String PARALLEL_DEVICES = "paralleldevices"; + protected static final String PARALLEL_DEVICES = "parallelDevices"; protected ConnectorInputEntityFactory(Class... allowedClasses) { super(allowedClasses); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/CylindricalStorageInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/CylindricalStorageInputFactory.java index cd517423c..3b423465c 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/CylindricalStorageInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/CylindricalStorageInputFactory.java @@ -18,10 +18,10 @@ public class CylindricalStorageInputFactory extends AssetInputEntityFactory { - private static final String STORAGE_VOLUME_LVL = "storagevolumelvl"; - private static final String STORAGE_VOLUME_LVL_MIN = "storagevolumelvlmin"; - private static final String INLET_TEMP = "inlettemp"; - private static final String RETURN_TEMP = "returntemp"; + private static final String STORAGE_VOLUME_LVL = "storageVolumeLvl"; + private static final String STORAGE_VOLUME_LVL_MIN = "storageVolumeLvlMin"; + private static final String INLET_TEMP = "inletTemp"; + private static final String RETURN_TEMP = "returnTemp"; private static final String C = "c"; public CylindricalStorageInputFactory() { diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/LineInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/LineInputFactory.java index e0c1154d8..ff8769100 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/LineInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/LineInputFactory.java @@ -23,8 +23,8 @@ public class LineInputFactory extends ConnectorInputEntityFactory> { private static final String LENGTH = "length"; - private static final String GEO_POSITION = "geoposition"; - private static final String OLM_CHARACTERISTIC = "olmcharacteristic"; + private static final String GEO_POSITION = "geoPosition"; + private static final String OLM_CHARACTERISTIC = "olmCharacteristic"; public LineInputFactory() { super(LineInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/MeasurementUnitInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/MeasurementUnitInputFactory.java index 7ed27292f..e15f6f494 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/MeasurementUnitInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/MeasurementUnitInputFactory.java @@ -13,8 +13,8 @@ public class MeasurementUnitInputFactory extends AssetInputEntityFactory { - private static final String V_MAG = "vmag"; - private static final String V_ANG = "vang"; + private static final String V_MAG = "vMag"; + private static final String V_ANG = "vAng"; private static final String P = "p"; private static final String Q = "q"; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/NodeInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/NodeInputFactory.java index 1110d5d51..96aa8859d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/NodeInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/NodeInputFactory.java @@ -16,10 +16,10 @@ import tech.units.indriya.ComparableQuantity; public class NodeInputFactory extends AssetInputEntityFactory { - private static final String V_TARGET = "vtarget"; + private static final String V_TARGET = "vTarget"; public static final String V_RATED = "vRated"; private static final String SLACK = "slack"; - private static final String GEO_POSITION = "geoposition"; + private static final String GEO_POSITION = "geoPosition"; public static final String VOLT_LVL = "voltLvl"; private static final String SUBNET = "subnet"; @@ -29,9 +29,7 @@ public NodeInputFactory() { @Override protected String[] getAdditionalFields() { - return new String[] { - V_TARGET, V_RATED.toLowerCase(), SLACK, GEO_POSITION, VOLT_LVL.toLowerCase(), SUBNET - }; + return new String[] {V_TARGET, V_RATED, SLACK, GEO_POSITION, VOLT_LVL, SUBNET}; } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/OperatorInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/OperatorInputFactory.java index 09dcdc881..5e7b0f1b5 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/OperatorInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/OperatorInputFactory.java @@ -22,7 +22,7 @@ public OperatorInputFactory() { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set constructorParams = newSet(ENTITY_UUID, ENTITY_ID); return Collections.singletonList(constructorParams); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/ThermalHouseInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/ThermalHouseInputFactory.java index f60961cb6..6fab0fc61 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/ThermalHouseInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/ThermalHouseInputFactory.java @@ -18,8 +18,8 @@ public class ThermalHouseInputFactory extends AssetInputEntityFactory { - private static final String ETH_LOSSES = "ethlosses"; - private static final String ETH_CAPA = "ethcapa"; + private static final String ETH_LOSSES = "ethLosses"; + private static final String ETH_CAPA = "ethCapa"; private static final String TARGET_TEMPERATURE = "targetTemperature"; private static final String UPPER_TEMPERATURE_LIMIT = "upperTemperatureLimit"; private static final String LOWER_TEMPERATURE_LIMIT = "lowerTemperatureLimit"; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactory.java index 5b1909993..110f0572a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactory.java @@ -16,8 +16,8 @@ public class Transformer2WInputFactory extends ConnectorInputEntityFactory< Transformer2WInput, TypedConnectorInputEntityData> { - private static final String TAP_POS = "tappos"; - private static final String AUTO_TAP = "autotap"; + private static final String TAP_POS = "tapPos"; + private static final String AUTO_TAP = "autoTap"; public Transformer2WInputFactory() { super(Transformer2WInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactory.java index 59aec043b..c9987fe1f 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactory.java @@ -15,8 +15,8 @@ public class Transformer3WInputFactory extends ConnectorInputEntityFactory { - private static final String TAP_POS = "tappos"; - private static final String AUTO_TAP = "autotap"; + private static final String TAP_POS = "tapPos"; + private static final String AUTO_TAP = "autoTap"; public Transformer3WInputFactory() { super(Transformer3WInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/graphics/GraphicInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/graphics/GraphicInputFactory.java index 4dfb98676..c3a5a058b 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/graphics/GraphicInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/graphics/GraphicInputFactory.java @@ -23,7 +23,7 @@ public abstract class GraphicInputFactory { private static final String UUID = "uuid"; - private static final String GRAPHIC_LAYER = "graphiclayer"; + private static final String GRAPHIC_LAYER = "graphicLayer"; private static final String PATH_LINE_STRING = "path"; protected GraphicInputFactory(Class... allowedClasses) { @@ -36,11 +36,11 @@ protected GraphicInputFactory(Class... allowedClasses) { *

The mandatory attributes required to create an {@link GraphicInput} are enhanced with custom * attribute names that each subclass factory determines in {@link #getAdditionalFields()}. * - * @param data EntityData (or subclass) containing the data + * @param entityClass class of the entity * @return list of possible attribute sets */ @Override - protected List> getFields(D data) { + protected List> getFields(Class entityClass) { Set constructorParamsMin = newSet(UUID, GRAPHIC_LAYER, PATH_LINE_STRING); final String[] additionalFields = getAdditionalFields(); constructorParamsMin = expandSet(constructorParamsMin, additionalFields); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/BmInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/BmInputFactory.java index 799837d3c..656e66361 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/BmInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/BmInputFactory.java @@ -19,9 +19,9 @@ public class BmInputFactory extends SystemParticipantInputEntityFactory< BmInput, SystemParticipantTypedEntityData> { - private static final String MARKET_REACTION = "marketreaction"; - private static final String COST_CONTROLLED = "costcontrolled"; - private static final String FEED_IN_TARIFF = "feedintariff"; + private static final String MARKET_REACTION = "marketReaction"; + private static final String COST_CONTROLLED = "costControlled"; + private static final String FEED_IN_TARIFF = "feedInTariff"; public BmInputFactory() { super(BmInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/ChpInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/ChpInputFactory.java index f04b85e86..fd6411508 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/ChpInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/ChpInputFactory.java @@ -14,7 +14,7 @@ public class ChpInputFactory extends SystemParticipantInputEntityFactory { - private static final String MARKET_REACTION = "marketreaction"; + private static final String MARKET_REACTION = "marketReaction"; public ChpInputFactory() { super(ChpInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EmInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EmInputFactory.java index f19475f96..aa12567a6 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EmInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EmInputFactory.java @@ -21,9 +21,9 @@ public class EmInputFactory extends SystemParticipantInputEntityFactory { private static final Logger logger = LoggerFactory.getLogger(EmInputFactory.class); - private static final String CONNECTED_ASSETS = "connectedassets"; + private static final String CONNECTED_ASSETS = "connectedAssets"; - private static final String CONTROL_STRATEGY = "controlstrategy"; + private static final String CONTROL_STRATEGY = "controlStrategy"; public EmInputFactory() { super(EmInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactory.java index fb7419a75..6eb83d0bd 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactory.java @@ -30,10 +30,10 @@ public class EvcsInputFactory extends SystemParticipantInputEntityFactory { private static final String TYPE = "type"; - private static final String CHARGING_POINTS = "chargingpoints"; - private static final String COS_PHI_RATED = "cosphirated"; - private static final String LOCATION_TYPE = "locationtype"; - private static final String V2G_SUPPORT = "v2gsupport"; + private static final String CHARGING_POINTS = "chargingPoints"; + private static final String COS_PHI_RATED = "cosPhiRated"; + private static final String LOCATION_TYPE = "locationType"; + private static final String V2G_SUPPORT = "v2gSupport"; public EvcsInputFactory() { super(EvcsInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactory.java index 03ff6636f..25ba74543 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactory.java @@ -18,8 +18,8 @@ public class FixedFeedInInputFactory extends SystemParticipantInputEntityFactory { - private static final String S_RATED = "srated"; - private static final String COSPHI_RATED = "cosphirated"; + private static final String S_RATED = "sRated"; + private static final String COSPHI_RATED = "cosPhiRated"; public FixedFeedInInputFactory() { super(FixedFeedInInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/LoadInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/LoadInputFactory.java index eb86cefc3..6ab009d90 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/LoadInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/LoadInputFactory.java @@ -24,11 +24,11 @@ public class LoadInputFactory extends SystemParticipantInputEntityFactory { private static final Logger logger = LoggerFactory.getLogger(LoadInputFactory.class); - private static final String LOAD_PROFILE = "loadprofile"; + private static final String LOAD_PROFILE = "loadProfile"; private static final String DSM = "dsm"; - private static final String E_CONS_ANNUAL = "econsannual"; - private static final String S_RATED = "srated"; - private static final String COS_PHI = "cosphirated"; + private static final String E_CONS_ANNUAL = "eConsAnnual"; + private static final String S_RATED = "sRated"; + private static final String COS_PHI = "cosPhiRated"; public LoadInputFactory() { super(LoadInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/PvInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/PvInputFactory.java index 9812354e3..34cf9cd34 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/PvInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/PvInputFactory.java @@ -21,13 +21,13 @@ public class PvInputFactory extends SystemParticipantInputEntityFactory { private static final String ALBEDO = "albedo"; private static final String AZIMUTH = "azimuth"; - private static final String ETA_CONV = "etaconv"; - private static final String ELEVATION_ANGLE = "elevationangle"; - private static final String KG = "kg"; - private static final String KT = "kt"; - private static final String MARKET_REACTION = "marketreaction"; - private static final String S_RATED = "srated"; - private static final String COS_PHI_RATED = "cosphirated"; + private static final String ETA_CONV = "etaConv"; + private static final String ELEVATION_ANGLE = "elevationAngle"; + private static final String KG = "kG"; + private static final String KT = "kT"; + private static final String MARKET_REACTION = "marketReaction"; + private static final String S_RATED = "sRated"; + private static final String COS_PHI_RATED = "cosPhiRated"; public PvInputFactory() { super(PvInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java index 212f9db96..b124ac9ec 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java @@ -30,15 +30,15 @@ abstract class SystemParticipantInputEntityFactory< T extends SystemParticipantInput, D extends NodeAssetInputEntityData> extends AssetInputEntityFactory { - private static final String Q_CHARACTERISTICS = "qcharacteristics"; + private static final String Q_CHARACTERISTICS = "qCharacteristics"; protected SystemParticipantInputEntityFactory(Class... allowedClasses) { super(allowedClasses); } @Override - protected List> getFields(D data) { - List> fields = super.getFields(data); + protected List> getFields(Class entityClass) { + List> fields = super.getFields(entityClass); for (Set set : fields) set.add(Q_CHARACTERISTICS); return fields; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/WecInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/WecInputFactory.java index 83bbbdd43..dfb2a8f8e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/input/participant/WecInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/input/participant/WecInputFactory.java @@ -16,7 +16,7 @@ public class WecInputFactory extends SystemParticipantInputEntityFactory< WecInput, SystemParticipantTypedEntityData> { - private static final String MARKET_REACTION = "marketreaction"; + private static final String MARKET_REACTION = "marketReaction"; public WecInputFactory() { super(WecInput.class); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/ConnectorResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/ConnectorResultFactory.java index 9444350ab..801341a2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/ConnectorResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/ConnectorResultFactory.java @@ -21,13 +21,13 @@ public class ConnectorResultFactory extends ResultEntityFactory { - private static final String IAMAG = "iamag"; - private static final String IAANG = "iaang"; - private static final String IBMAG = "ibmag"; - private static final String IBANG = "ibang"; - private static final String ICMAG = "icmag"; - private static final String ICANG = "icang"; - private static final String TAPPOS = "tappos"; + private static final String IAMAG = "iAMag"; + private static final String IAANG = "iAAng"; + private static final String IBMAG = "iBMag"; + private static final String IBANG = "iBAng"; + private static final String ICMAG = "iCMag"; + private static final String ICANG = "iCAng"; + private static final String TAPPOS = "tapPos"; public ConnectorResultFactory() { super(LineResult.class, Transformer2WResult.class, Transformer3WResult.class); @@ -44,12 +44,11 @@ public ConnectorResultFactory(String dtfPattern) { } @Override - protected List> getFields(SimpleEntityData simpleEntityData) { + protected List> getFields(Class entityClass) { /// all result models have the same constructor except StorageResult Set minConstructorParams = newSet(TIME, INPUT_MODEL, IAMAG, IAANG, IBMAG, IBANG); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); - final Class entityClass = simpleEntityData.getTargetClass(); if (entityClass.equals(Transformer2WResult.class)) { minConstructorParams = newSet(TIME, INPUT_MODEL, IAMAG, IAANG, IBMAG, IBANG, TAPPOS); optionalFields = expandSet(minConstructorParams, ENTITY_UUID); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactory.java index 10ccfd1ba..aa33bdaf9 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactory.java @@ -15,9 +15,9 @@ public class FlexOptionsResultFactory extends ResultEntityFactory { - private static final String P_REF = "pref"; - private static final String P_MIN = "pmin"; - private static final String P_MAX = "pmax"; + private static final String P_REF = "pRef"; + private static final String P_MIN = "pMin"; + private static final String P_MAX = "pMax"; public FlexOptionsResultFactory() { super(FlexOptionsResult.class); @@ -34,7 +34,7 @@ public FlexOptionsResultFactory(String dtfPattern) { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet(TIME, INPUT_MODEL, P_REF, P_MIN, P_MAX); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/NodeResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/NodeResultFactory.java index 9f4a9b016..855fc5d83 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/NodeResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/NodeResultFactory.java @@ -15,8 +15,8 @@ import tech.units.indriya.ComparableQuantity; public class NodeResultFactory extends ResultEntityFactory { - private static final String VMAG = "vmag"; - private static final String VANG = "vang"; + private static final String VMAG = "vMag"; + private static final String VANG = "vAng"; public NodeResultFactory() { super(NodeResult.class); @@ -33,7 +33,7 @@ public NodeResultFactory(String dtfPattern) { } @Override - protected List> getFields(SimpleEntityData entityData) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet(TIME, INPUT_MODEL, VMAG, VANG); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/SwitchResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/SwitchResultFactory.java index df24fe8ab..350885896 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/SwitchResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/SwitchResultFactory.java @@ -29,7 +29,7 @@ public SwitchResultFactory(String dtfPattern) { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet(TIME, INPUT_MODEL, CLOSED); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactory.java index 62aa0dc03..8a4b11af2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactory.java @@ -27,7 +27,7 @@ public class SystemParticipantResultFactory extends ResultEntityFactory> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { /// all result models have the same constructor except StorageResult Set minConstructorParams = newSet(TIME, INPUT_MODEL, POWER, REACTIVE_POWER); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); - if (data.getTargetClass().equals(StorageResult.class) - || data.getTargetClass().equals(EvResult.class)) { + if (entityClass.equals(StorageResult.class) || entityClass.equals(EvResult.class)) { minConstructorParams = newSet(TIME, INPUT_MODEL, POWER, REACTIVE_POWER, SOC); optionalFields = expandSet(minConstructorParams, ENTITY_UUID); } - if (SystemParticipantWithHeatResult.class.isAssignableFrom(data.getTargetClass())) { + if (SystemParticipantWithHeatResult.class.isAssignableFrom(entityClass)) { minConstructorParams = expandSet(minConstructorParams, Q_DOT); optionalFields = expandSet(minConstructorParams, ENTITY_UUID); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/result/ThermalResultFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/result/ThermalResultFactory.java index 2d3e2c77c..2848fb52b 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/result/ThermalResultFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/result/ThermalResultFactory.java @@ -41,13 +41,13 @@ public ThermalResultFactory(String dtfPattern) { } @Override - protected List> getFields(SimpleEntityData simpleEntityData) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet(TIME, INPUT_MODEL, Q_DOT); Set optionalFields = expandSet(minConstructorParams, ENTITY_UUID); - if (simpleEntityData.getTargetClass().equals(ThermalHouseResult.class)) { + if (entityClass.equals(ThermalHouseResult.class)) { minConstructorParams = newSet(TIME, INPUT_MODEL, Q_DOT, INDOOR_TEMPERATURE); - } else if (simpleEntityData.getTargetClass().equals(CylindricalStorageResult.class)) { + } else if (entityClass.equals(CylindricalStorageResult.class)) { minConstructorParams = newSet(TIME, INPUT_MODEL, Q_DOT, ENERGY, FILL_LEVEL); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactory.java index 18920df21..3dcb3e083 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactory.java @@ -20,10 +20,10 @@ public class CosmoIdCoordinateFactory extends IdCoordinateFactory { private static final String TID = "tid"; private static final String COORDINATE_ID = "id"; - private static final String LONG_GEO = "longgeo"; - private static final String LAT_GEO = "latgeo"; - private static final String LONG_ROT = "longrot"; - private static final String LAT_ROT = "latrot"; + private static final String LONG_GEO = "longGeo"; + private static final String LAT_GEO = "latGeo"; + private static final String LONG_ROT = "longRot"; + private static final String LAT_ROT = "latRot"; @Override protected Pair buildModel(SimpleFactoryData data) { @@ -34,7 +34,7 @@ protected Pair buildModel(SimpleFactoryData data) { } @Override - protected List> getFields(SimpleFactoryData data) { + protected List> getFields(Class entityClass) { return Collections.singletonList( newSet(TID, COORDINATE_ID, LONG_GEO, LAT_GEO, LONG_ROT, LAT_ROT)); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index a95affade..1bce91906 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -26,11 +26,11 @@ * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; - private static final String DIRECT_IRRADIANCE = "directirradiance"; + private static final String DIFFUSE_IRRADIANCE = "diffuseIrradiance"; + private static final String DIRECT_IRRADIANCE = "directIrradiance"; private static final String TEMPERATURE = "temperature"; - private static final String WIND_DIRECTION = "winddirection"; - private static final String WIND_VELOCITY = "windvelocity"; + private static final String WIND_DIRECTION = "windDirection"; + private static final String WIND_VELOCITY = "windVelocity"; public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); @@ -50,7 +50,7 @@ public String getTimeFieldString() { } @Override - protected List> getFields(TimeBasedWeatherValueData data) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet( UUID, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactory.java index 454c73157..39843178c 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactory.java @@ -21,7 +21,7 @@ public class IconIdCoordinateFactory extends IdCoordinateFactory { private static final String COORDINATE_ID = "id"; private static final String LONG = "longitude"; private static final String LAT = "latitude"; - private static final String TYPE = "coordinatetype"; + private static final String TYPE = "coordinateType"; @Override protected Pair buildModel(SimpleFactoryData data) { @@ -32,7 +32,7 @@ protected Pair buildModel(SimpleFactoryData data) { } @Override - protected List> getFields(SimpleFactoryData data) { + protected List> getFields(Class entityClass) { return Collections.singletonList(newSet(COORDINATE_ID, LAT, LONG, TYPE)); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index 1949bb2db..20d8e5bef 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -53,7 +53,7 @@ public String getTimeFieldString() { } @Override - protected List> getFields(TimeBasedWeatherValueData data) { + protected List> getFields(Class entityClass) { Set minParameters = newSet( TIME, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/SqlIdCoordinateFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/SqlIdCoordinateFactory.java index fb1d053f2..141d8dfd5 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/SqlIdCoordinateFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/SqlIdCoordinateFactory.java @@ -39,7 +39,7 @@ protected Pair buildModel(SimpleFactoryData data) { } @Override - protected List> getFields(SimpleFactoryData data) { + protected List> getFields(Class entityClass) { return Collections.singletonList(newSet(COORDINATE_ID, COORDINATE)); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java index f71a00940..243c3986f 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java @@ -25,7 +25,7 @@ public class TimeBasedSimpleValueFactory /* Energy / Power */ private static final String ACTIVE_POWER = "p"; private static final String REACTIVE_POWER = "q"; - private static final String HEAT_DEMAND = "heatdemand"; + private static final String HEAT_DEMAND = "heatDemand"; private final TimeUtil timeUtil; @@ -87,24 +87,24 @@ protected TimeBasedValue buildModel(SimpleTimeBasedValueData data) { } @Override - protected List> getFields(SimpleTimeBasedValueData data) { + protected List> getFields(Class entityClass) { Set minConstructorParams = newSet(UUID, TIME); - if (EnergyPriceValue.class.isAssignableFrom(data.getTargetClass())) { + if (EnergyPriceValue.class.isAssignableFrom(entityClass)) { minConstructorParams.add(PRICE); - } else if (HeatAndSValue.class.isAssignableFrom(data.getTargetClass())) { + } else if (HeatAndSValue.class.isAssignableFrom(entityClass)) { minConstructorParams.addAll(Arrays.asList(ACTIVE_POWER, REACTIVE_POWER, HEAT_DEMAND)); - } else if (HeatAndPValue.class.isAssignableFrom(data.getTargetClass())) { + } else if (HeatAndPValue.class.isAssignableFrom(entityClass)) { minConstructorParams.addAll(Arrays.asList(ACTIVE_POWER, HEAT_DEMAND)); - } else if (HeatDemandValue.class.isAssignableFrom(data.getTargetClass())) { + } else if (HeatDemandValue.class.isAssignableFrom(entityClass)) { minConstructorParams.add(HEAT_DEMAND); - } else if (SValue.class.isAssignableFrom(data.getTargetClass())) { + } else if (SValue.class.isAssignableFrom(entityClass)) { minConstructorParams.addAll(Arrays.asList(ACTIVE_POWER, REACTIVE_POWER)); - } else if (PValue.class.isAssignableFrom(data.getTargetClass())) { + } else if (PValue.class.isAssignableFrom(entityClass)) { minConstructorParams.add(ACTIVE_POWER); } else { throw new FactoryException( - "The given factory cannot handle target class '" + data.getTargetClass() + "'."); + "The given factory cannot handle target class '" + entityClass + "'."); } return Collections.singletonList(minConstructorParams); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 4ad3b3348..254de4a74 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -18,7 +18,7 @@ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { protected static final String UUID = "uuid"; protected static final String TIME = "time"; - protected static final String COORDINATE_ID = "coordinateid"; + protected static final String COORDINATE_ID = "coordinateId"; protected final TimeUtil timeUtil; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMappingFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMappingFactory.java index 64aa90af4..5962ae2b7 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMappingFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMappingFactory.java @@ -26,7 +26,7 @@ public TimeSeriesMappingFactory() { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { return Collections.singletonList( Stream.of(UUID, PARTICIPANT, TIME_SERIES).collect(Collectors.toSet())); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java index fba7317a7..a5237275a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java @@ -30,7 +30,7 @@ public TimeSeriesMetaInformationFactory() { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { return Collections.singletonList( Stream.of(TIME_SERIES, COLUMN_SCHEME).collect(Collectors.toSet())); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/LineTypeInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/LineTypeInputFactory.java index 609a11849..a5a0faa5d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/LineTypeInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/LineTypeInputFactory.java @@ -23,15 +23,15 @@ public class LineTypeInputFactory extends AssetTypeInputEntityFactory> getFields(SimpleEntityData entityData) { + protected List> getFields(Class entityClass) { Set constructorParams = newSet(ENTITY_UUID, ENTITY_ID, B, G, R, X, I_MAX, V_RATED); return Collections.singletonList(constructorParams); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactory.java index e0aa91bdf..7d6596cc6 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactory.java @@ -27,35 +27,35 @@ public class SystemParticipantTypeInputFactory // SystemParticipantTypeInput parameters private static final String CAP_EX = "capex"; private static final String OP_EX = "opex"; - private static final String S_RATED = "srated"; - private static final String COS_PHI_RATED = "cosphirated"; + private static final String S_RATED = "sRated"; + private static final String COS_PHI_RATED = "cosPhiRated"; // required in multiple types - private static final String ETA_CONV = "etaconv"; - private static final String P_THERMAL = "pthermal"; - private static final String E_STORAGE = "estorage"; + private static final String ETA_CONV = "etaConv"; + private static final String P_THERMAL = "pThermal"; + private static final String E_STORAGE = "eStorage"; // EvTypeInput - private static final String E_CONS = "econs"; + private static final String E_CONS = "eCons"; // BmTypeInput - private static final String ACTIVE_POWER_GRADIENT = "activepowergradient"; + private static final String ACTIVE_POWER_GRADIENT = "activePowerGradient"; // WecTypeInput - private static final String ROTOR_AREA = "rotorarea"; - private static final String HUB_HEIGHT = "hubheight"; + private static final String ROTOR_AREA = "rotorArea"; + private static final String HUB_HEIGHT = "hubHeight"; // ChpTypeInput - private static final String ETA_EL = "etael"; - private static final String ETA_THERMAL = "etathermal"; - private static final String P_OWN = "pown"; + private static final String ETA_EL = "etaEl"; + private static final String ETA_THERMAL = "etaThermal"; + private static final String P_OWN = "pOwn"; // StorageTypeInput - private static final String P_MAX = "pmax"; + private static final String P_MAX = "pMax"; private static final String ETA = "eta"; private static final String DOD = "dod"; - private static final String LIFETIME = "lifetime"; - private static final String LIFECYCLE = "lifecycle"; + private static final String LIFETIME = "lifeTime"; + private static final String LIFECYCLE = "lifeCycle"; // WecTypeInput private static final String CP_CHARACTERISTIC = "cpCharacteristic"; @@ -71,24 +71,24 @@ public SystemParticipantTypeInputFactory() { } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set standardConstructorParams = newSet(ENTITY_UUID, ENTITY_ID, CAP_EX, OP_EX, S_RATED, COS_PHI_RATED); Set constructorParameters = null; - if (data.getTargetClass().equals(EvTypeInput.class)) { + if (entityClass.equals(EvTypeInput.class)) { constructorParameters = expandSet(standardConstructorParams, E_STORAGE, E_CONS); - } else if (data.getTargetClass().equals(HpTypeInput.class)) { + } else if (entityClass.equals(HpTypeInput.class)) { constructorParameters = expandSet(standardConstructorParams, P_THERMAL); - } else if (data.getTargetClass().equals(BmTypeInput.class)) { + } else if (entityClass.equals(BmTypeInput.class)) { constructorParameters = expandSet(standardConstructorParams, ACTIVE_POWER_GRADIENT, ETA_CONV); - } else if (data.getTargetClass().equals(WecTypeInput.class)) { + } else if (entityClass.equals(WecTypeInput.class)) { constructorParameters = expandSet(standardConstructorParams, CP_CHARACTERISTIC, ETA_CONV, ROTOR_AREA, HUB_HEIGHT); - } else if (data.getTargetClass().equals(ChpTypeInput.class)) { // into new file + } else if (entityClass.equals(ChpTypeInput.class)) { // into new file constructorParameters = expandSet(standardConstructorParams, ETA_EL, ETA_THERMAL, P_THERMAL, P_OWN); - } else if (data.getTargetClass().equals(StorageTypeInput.class)) { + } else if (entityClass.equals(StorageTypeInput.class)) { constructorParameters = expandSet( standardConstructorParams, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer2WTypeInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer2WTypeInputFactory.java index a79236d49..8274d0665 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer2WTypeInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer2WTypeInputFactory.java @@ -17,26 +17,26 @@ public class Transformer2WTypeInputFactory extends AssetTypeInputEntityFactory { - private static final String R_SC = "rsc"; - private static final String X_SC = "xsc"; - private static final String S_RATED = "srated"; - private static final String V_RATED_A = "vrateda"; - private static final String V_RATED_B = "vratedb"; - private static final String G_M = "gm"; - private static final String B_M = "bm"; - private static final String D_V = "dv"; - private static final String D_PHI = "dphi"; - private static final String TAP_SIDE = "tapside"; - private static final String TAP_NEUTR = "tapneutr"; - private static final String TAP_MIN = "tapmin"; - private static final String TAP_MAX = "tapmax"; + private static final String R_SC = "rSc"; + private static final String X_SC = "xSc"; + private static final String S_RATED = "sRated"; + private static final String V_RATED_A = "vRatedA"; + private static final String V_RATED_B = "vRatedB"; + private static final String G_M = "gM"; + private static final String B_M = "bM"; + private static final String D_V = "dV"; + private static final String D_PHI = "dPhi"; + private static final String TAP_SIDE = "tapSide"; + private static final String TAP_NEUTR = "tapNeutr"; + private static final String TAP_MIN = "tapMin"; + private static final String TAP_MAX = "tapMax"; public Transformer2WTypeInputFactory() { super(Transformer2WTypeInput.class); } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set constructorParams = newSet( ENTITY_UUID, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer3WTypeInputFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer3WTypeInputFactory.java index a51f2ec7b..345b0728c 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer3WTypeInputFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/typeinput/Transformer3WTypeInputFactory.java @@ -17,32 +17,32 @@ public class Transformer3WTypeInputFactory extends AssetTypeInputEntityFactory { - private static final String S_RATED_A = "srateda"; - private static final String S_RATED_B = "sratedb"; - private static final String S_RATED_C = "sratedc"; - private static final String V_RATED_A = "vrateda"; - private static final String V_RATED_B = "vratedb"; - private static final String V_RATED_C = "vratedc"; - private static final String R_SC_A = "rsca"; - private static final String R_SC_B = "rscb"; - private static final String R_SC_C = "rscc"; - private static final String X_SC_A = "xsca"; - private static final String X_SC_B = "xscb"; - private static final String X_SC_C = "xscc"; - private static final String G_M = "gm"; - private static final String B_M = "bm"; - private static final String D_V = "dv"; - private static final String D_PHI = "dphi"; - private static final String TAP_NEUTR = "tapneutr"; - private static final String TAP_MIN = "tapmin"; - private static final String TAP_MAX = "tapmax"; + private static final String S_RATED_A = "sRatedA"; + private static final String S_RATED_B = "sRatedB"; + private static final String S_RATED_C = "sRatedC"; + private static final String V_RATED_A = "vRatedA"; + private static final String V_RATED_B = "vRatedB"; + private static final String V_RATED_C = "vRatedC"; + private static final String R_SC_A = "rScA"; + private static final String R_SC_B = "rScB"; + private static final String R_SC_C = "rScC"; + private static final String X_SC_A = "xScA"; + private static final String X_SC_B = "xScB"; + private static final String X_SC_C = "xScC"; + private static final String G_M = "gM"; + private static final String B_M = "bM"; + private static final String D_V = "dV"; + private static final String D_PHI = "dPhi"; + private static final String TAP_NEUTR = "tapNeutr"; + private static final String TAP_MIN = "tapMin"; + private static final String TAP_MAX = "tapMax"; public Transformer3WTypeInputFactory() { super(Transformer3WTypeInput.class); } @Override - protected List> getFields(SimpleEntityData data) { + protected List> getFields(Class entityClass) { Set constructorParams = newSet( ENTITY_UUID, diff --git a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java index 0687e9e50..f52d9b55e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java @@ -13,5 +13,6 @@ public interface DataSource { /** Creates a stream of maps that represent the rows in the database */ - Stream> getSourceData(Class entityClass); + Stream> getSourceData( + Class entityClass, SourceValidator validateSource); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index eec826f97..64a7b66ab 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -251,9 +251,9 @@ protected Stream> nodeAssetInputE * @return stream of the entity data wrapped in a {@link Try} */ protected Stream assetInputEntityDataStream( - Class entityClass, Collection operators) { + Class entityClass, SourceValidator validator, Collection operators) { return dataSource - .getSourceData(entityClass) + .getSourceData(entityClass, validator) .map( fieldsToAttributes -> assetInputEntityDataStream(entityClass, fieldsToAttributes, operators)); @@ -288,19 +288,12 @@ protected AssetInputEntityData assetInputEntityDataStream * @return stream of {@link SimpleEntityData} */ protected Stream simpleEntityDataStream( - Class entityClass) { + Class entityClass, SourceValidator validator) { return dataSource - .getSourceData(entityClass) + .getSourceData(entityClass, validator) .map(fieldsToAttributes -> new SimpleEntityData(fieldsToAttributes, entityClass)); } - protected Stream> assetInputEntityStream( - Class entityClass, - EntityFactory factory, - Collection operators) { - return assetInputEntityDataStream(entityClass, operators).map(factory::get); - } - /** * Returns a stream of {@link Try} entities that can be build by using {@link * NodeAssetInputEntityData} and their corresponding factory. @@ -319,7 +312,8 @@ protected Stream> nodeAssetEntit EntityFactory factory, Collection nodes, Collection operators) { - return nodeAssetInputEntityDataStream(assetInputEntityDataStream(entityClass, operators), nodes) + return nodeAssetInputEntityDataStream( + assetInputEntityDataStream(entityClass, factory, operators), nodes) .map(factory::get); } @@ -338,14 +332,16 @@ public Set> buildAssetInputEntit Class entityClass, EntityFactory factory, Collection operators) { - return assetInputEntityStream(entityClass, factory, operators).collect(Collectors.toSet()); + return assetInputEntityDataStream(entityClass, factory, operators) + .map(factory::get) + .collect(Collectors.toSet()); } @SuppressWarnings("unchecked") public Set> buildEntities( Class entityClass, EntityFactory factory) { return dataSource - .getSourceData(entityClass) + .getSourceData(entityClass, factory) .map( fieldsToAttributes -> { SimpleEntityData data = new SimpleEntityData(fieldsToAttributes, entityClass); diff --git a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java index 7cdc097ca..bc237aa1b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java @@ -91,7 +91,7 @@ public Set getNodeGraphicInput() throws SourceException { public Set getNodeGraphicInput(Set nodes) throws SourceException { return Try.scanCollection( - buildNodeGraphicEntityData(nodes) + buildNodeGraphicEntityData(nodes, nodeGraphicInputFactory) .map(nodeGraphicInputFactory::get) .collect(Collectors.toSet()), NodeGraphicInput.class) @@ -113,7 +113,7 @@ public Set getLineGraphicInput() throws SourceException { public Set getLineGraphicInput(Set lines) throws SourceException { return Try.scanCollection( - buildLineGraphicEntityData(lines) + buildLineGraphicEntityData(lines, lineGraphicInputFactory) .map(lineGraphicInputFactory::get) .collect(Collectors.toSet()), LineGraphicInput.class) @@ -140,9 +140,9 @@ public Set getLineGraphicInput(Set lines) throws So * @return a stream of tries of {@link NodeGraphicInput} entities */ protected Stream> buildNodeGraphicEntityData( - Set nodes) { + Set nodes, SourceValidator validator) { return dataSource - .getSourceData(NodeGraphicInput.class) + .getSourceData(NodeGraphicInput.class, validator) .map(fieldsToAttributes -> buildNodeGraphicEntityData(fieldsToAttributes, nodes)); } @@ -187,9 +187,9 @@ protected Try buildNodeGraphicEntit * @return a stream of tries of {@link LineGraphicInput} entities */ protected Stream> buildLineGraphicEntityData( - Set lines) { + Set lines, SourceValidator validator) { return dataSource - .getSourceData(LineGraphicInput.class) + .getSourceData(LineGraphicInput.class, validator) .map(fieldsToAttributes -> buildLineGraphicEntityData(fieldsToAttributes, lines)); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java index b06d55d4d..986059dfc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java @@ -166,7 +166,7 @@ public Set getNodes() throws SourceException { */ public Set getNodes(Set operators) throws SourceException { return Try.scanCollection( - assetInputEntityDataStream(NodeInput.class, operators) + assetInputEntityDataStream(NodeInput.class, nodeInputFactory, operators) .map(nodeInputFactory::get) .collect(Collectors.toSet()), NodeInput.class) @@ -414,7 +414,7 @@ public Set> buildNodeInputEntiti Class entityClass, EntityFactory factory, Collection operators) { - return assetInputEntityDataStream(entityClass, operators) + return assetInputEntityDataStream(entityClass, factory, operators) .map(factory::get) .collect(Collectors.toSet()); } @@ -441,7 +441,9 @@ public Set> buildTransformer3WEntities return buildTransformer3WEntityData( buildTypedConnectorEntityData( buildUntypedConnectorInputEntityData( - assetInputEntityDataStream(Transformer3WInput.class, operators), nodes), + assetInputEntityDataStream( + Transformer3WInput.class, transformer3WInputFactory, operators), + nodes), transformer3WTypeInputs), nodes) .map(transformer3WInputFactory::get) @@ -560,7 +562,7 @@ Stream> typedEntityStream( Collection types) { return buildTypedConnectorEntityData( buildUntypedConnectorInputEntityData( - assetInputEntityDataStream(entityClass, operators), nodes), + assetInputEntityDataStream(entityClass, factory, operators), nodes), types) .map(factory::get); } @@ -572,7 +574,7 @@ Stream> untypedConnectorInputEntityStream( Set nodes, Set operators) { return buildUntypedConnectorInputEntityData( - assetInputEntityDataStream(entityClass, operators), nodes) + assetInputEntityDataStream(entityClass, factory, operators), nodes) .map(factory::get); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java index 87d9fcc21..b4cc69334 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java @@ -318,7 +318,7 @@ public Set getEmResults() { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Set getResultEntities( Class entityClass, SimpleEntityFactory factory) { - return simpleEntityDataStream(entityClass) + return simpleEntityDataStream(entityClass, factory) .map( entityData -> factory diff --git a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java new file mode 100644 index 000000000..f3f8ee9ef --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java @@ -0,0 +1,19 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source; + +import java.util.Set; + +public interface SourceValidator { + + /** + * Method for validating a data source. + * + * @param foundFields fields that were found in the source data + * @param entityClass that should be buildable from the source data + */ + void validate(Set foundFields, Class entityClass); +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index 3754362dd..293dc856d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -693,7 +693,7 @@ Stream> typedSystemParticipantEntityStream( Collection types) { return buildTypedSystemParticipantEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(entityClass, operators), nodes), + assetInputEntityDataStream(entityClass, factory, operators), nodes), types) .map(factory::get); } @@ -708,7 +708,7 @@ private Stream> chpInputStream( return buildChpEntityData( buildTypedEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(ChpInput.class, operators), nodes), + assetInputEntityDataStream(ChpInput.class, factory, operators), nodes), types), thermalStorages, thermalBuses) @@ -724,7 +724,7 @@ private Stream> hpInputStream( return buildHpEntityData( buildTypedEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(HpInput.class, operators), nodes), + assetInputEntityDataStream(HpInput.class, factory, operators), nodes), types), thermalBuses) .map(factory::get); diff --git a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java index 48384565f..60da88bc1 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java @@ -262,7 +262,7 @@ public Try, FailureException> buildThermalHouseInputEntit Set thermalBuses = getThermalBuses(); return Try.scanCollection( - assetInputEntityDataStream(ThermalHouseInput.class, typeSource.getOperators()) + assetInputEntityDataStream(ThermalHouseInput.class, factory, typeSource.getOperators()) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -276,7 +276,7 @@ public Try, FailureException> buildThermalHouseInputEntit Collection operators, Collection thermalBuses) { return Try.scanCollection( - assetInputEntityDataStream(ThermalHouseInput.class, operators) + assetInputEntityDataStream(ThermalHouseInput.class, factory, operators) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -290,7 +290,8 @@ public Try, FailureException> buildCylindricalStora Set thermalBuses = getThermalBuses(); return Try.scanCollection( - assetInputEntityDataStream(CylindricalStorageInput.class, typeSource.getOperators()) + assetInputEntityDataStream( + CylindricalStorageInput.class, factory, typeSource.getOperators()) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -303,7 +304,7 @@ public Set> buildCylindricalStora CylindricalStorageInputFactory factory, Collection operators, Collection thermalBuses) { - return assetInputEntityDataStream(CylindricalStorageInput.class, operators) + return assetInputEntityDataStream(CylindricalStorageInput.class, factory, operators) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) diff --git a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java index 09d4530f9..03377241f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java @@ -36,7 +36,7 @@ protected TimeSeriesMappingSource() { * @return That mapping */ public Map getMapping() { - return getMappingSourceData() + return getMappingSourceData(mappingFactory) .map(this::createMappingEntry) .filter(Try::isSuccess) .map(t -> (Success) t) @@ -59,7 +59,7 @@ public Optional getTimeSeriesUuid(UUID modelIdentifier) { * * @return Stream of maps */ - public abstract Stream> getMappingSourceData(); + public abstract Stream> getMappingSourceData(SourceValidator validator); // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 9a355e746..75fea3036 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -10,6 +10,7 @@ import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.utils.validation.ValidationUtils; import edu.ie3.util.StringUtils; @@ -59,8 +60,9 @@ public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNami } @Override - public Stream> getSourceData(Class entityClass) { - return buildStreamWithFieldsToAttributesMap(entityClass, connector); + public Stream> getSourceData( + Class entityClass, SourceValidator validator) { + return buildStreamWithFieldsToAttributesMap(entityClass, connector, validator); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -266,9 +268,12 @@ protected Predicate> isPresentCollectIfNot( * mapping (fieldName to fieldValue) */ protected Stream> buildStreamWithFieldsToAttributesMap( - Class entityClass, CsvFileConnector connector) { + Class entityClass, + CsvFileConnector connector, + SourceValidator validator) { try { - return buildStreamWithFieldsToAttributesMap(entityClass, connector.initReader(entityClass)); + return buildStreamWithFieldsToAttributesMap( + entityClass, connector.initReader(entityClass), validator); } catch (FileNotFoundException | ConnectorException e) { log.warn( "Unable to find file for entity '{}': {}", entityClass.getSimpleName(), e.getMessage()); @@ -287,10 +292,15 @@ protected Stream> buildStreamWithFieldsToAttributesMap( * mapping (fieldName to fieldValue) */ protected Stream> buildStreamWithFieldsToAttributesMap( - Class entityClass, BufferedReader bufferedReader) { + Class entityClass, + BufferedReader bufferedReader, + SourceValidator validator) { try (BufferedReader reader = bufferedReader) { final String[] headline = parseCsvRow(reader.readLine(), csvSep); + // validating read file + validator.validate(Set.of(headline), entityClass); + // sanity check for headline if (!Arrays.asList(headline).contains("uuid")) { throw new SourceException( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java index d7f293f0f..81d40e0bb 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java @@ -162,6 +162,9 @@ protected Stream> buildStreamWithFieldsToAttributesMap() { try (BufferedReader reader = dataSource.connector.initIdCoordinateReader()) { final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); + // validating read file + factory.validate(Set.of(headline), Pair.class); + // by default try-with-resources closes the reader directly when we leave this method (which // is wanted to avoid a lock on the file), but this causes a closing of the stream as well. // As we still want to consume the data at other places, we start a new stream instead of diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index c6affdb71..8e9996f8d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source.csv; import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.nio.file.Path; import java.util.Map; @@ -21,8 +22,8 @@ public CsvTimeSeriesMappingSource( } @Override - public Stream> getMappingSourceData() { + public Stream> getMappingSourceData(SourceValidator validator) { return dataSource.buildStreamWithFieldsToAttributesMap( - MappingEntry.class, dataSource.connector); + MappingEntry.class, dataSource.connector, validator); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java index 4b8d55604..1ad3d5e5d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java @@ -12,6 +12,7 @@ import edu.ie3.datamodel.io.factory.timeseries.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesSource; +import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.*; @@ -99,7 +100,11 @@ public CsvTimeSeriesSource( /* Read in the full time series */ try { this.timeSeries = - buildIndividualTimeSeries(timeSeriesUuid, filePath, this::createTimeBasedValue); + buildIndividualTimeSeries( + timeSeriesUuid, + filePath, + this::createTimeBasedValue, + (Class) valueClass); } catch (SourceException e) { throw new IllegalArgumentException( "Unable to obtain time series with UUID '" @@ -140,13 +145,14 @@ public Optional getValue(ZonedDateTime time) { protected IndividualTimeSeries buildIndividualTimeSeries( UUID timeSeriesUuid, Path filePath, - Function, Try, FactoryException>> fieldToValueFunction) + Function, Try, FactoryException>> fieldToValueFunction, + Class entityClass) throws SourceException { try (BufferedReader reader = dataSource.connector.initReader(filePath)) { Try>, FailureException> timeBasedValues = Try.scanStream( dataSource - .buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader) + .buildStreamWithFieldsToAttributesMap(entityClass, reader, valueFactory) .map(fieldToValueFunction), "TimeBasedValue"); return new IndividualTimeSeries<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 1997edbd1..6ece1f5c6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -179,6 +179,9 @@ private Stream> buildStreamWithFieldsToAttributesMap( try (BufferedReader reader = bufferedReader) { final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); + // validating read file + weatherFactory.validate(Set.of(headline), WeatherValue.class); + // by default try-with-resources closes the reader directly when we leave this method (which // is wanted to avoid a lock on the file), but this causes a closing of the stream as well. // As we still want to consume the data at other places, we start a new stream instead of diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index de3269b61..cd769fc26 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -159,8 +159,7 @@ private Stream>> optTimeBasedValueStream( fieldToValue.entrySet().stream() .collect( Collectors.toMap( - entry -> - StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), + entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()), Map.Entry::getValue)); /* Add a random UUID if necessary */ diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 9cd4e413f..ce1338158 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -9,6 +9,7 @@ import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.util.StringUtils; import java.sql.PreparedStatement; @@ -35,6 +36,28 @@ public SqlDataSource( this.databaseNamingStrategy = databaseNamingStrategy; } + /** + * This method should be used to validate a given table. + * + * @param tableName name of the table + * @param entityClass class of the entity + * @param validator for validation + * @throws SQLException – if the connection could not be established + */ + protected void validateDBTable(String tableName, Class entityClass, SourceValidator validator) + throws SQLException { + ResultSet rs = connector.getConnection().getMetaData().getColumns(null, null, tableName, null); + + Set columnNames = new HashSet<>(); + + while (rs.next()) { + String name = rs.getString("COLUMN_NAME"); + columnNames.add(StringUtils.snakeCaseToCamelCase(name)); + } + + validator.validate(columnNames, entityClass); + } + /** * Creates a base query string without closing semicolon of the following pattern:
* {@code SELECT * FROM .} @@ -109,7 +132,8 @@ protected List getDbTables(String schemaPattern, String tableNamePattern } @Override - public Stream> getSourceData(Class entityClass) { + public Stream> getSourceData( + Class entityClass, SourceValidator validator) { String explicitTableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); return buildStreamByTableName(explicitTableName); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index 9736d8beb..fd2ec84e8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -17,6 +17,7 @@ import edu.ie3.util.geo.GeoUtils; import java.sql.Array; import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.*; import javax.measure.quantity.Length; import org.apache.commons.lang3.tuple.Pair; @@ -45,7 +46,8 @@ public class SqlIdCoordinateSource implements IdCoordinateSource { private final SqlIdCoordinateFactory factory; public SqlIdCoordinateSource( - SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) { + SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) + throws SQLException { this.factory = factory; this.dataSource = dataSource; @@ -53,6 +55,9 @@ public SqlIdCoordinateSource( String dbPointColumnName = dataSource.getDbColumnName(factory.getCoordinateField(), coordinateTableName); + // validating table + dataSource.validateDBTable(coordinateTableName, Pair.class, factory); + // setup queries this.basicQuery = createBaseQueryString(dataSource.schemaName, coordinateTableName); this.queryForPoint = createQueryForPoint(dbIdColumnName); @@ -76,7 +81,8 @@ public SqlIdCoordinateSource( SqlConnector connector, String schemaName, String coordinateTableName, - SqlIdCoordinateFactory factory) { + SqlIdCoordinateFactory factory) + throws SQLException { this( factory, coordinateTableName, diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index 65f65f436..7f952d4e4 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -7,16 +7,20 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; +import java.sql.SQLException; import java.util.Map; import java.util.stream.Stream; public class SqlTimeSeriesMappingSource extends TimeSeriesMappingSource { private final EntityPersistenceNamingStrategy entityPersistenceNamingStrategy; private final String queryFull; + private final String tableName; private final SqlDataSource dataSource; public SqlTimeSeriesMappingSource( @@ -28,13 +32,19 @@ public SqlTimeSeriesMappingSource( connector, schemaName, new DatabaseNamingStrategy(entityPersistenceNamingStrategy)); this.entityPersistenceNamingStrategy = entityPersistenceNamingStrategy; - final String tableName = + this.tableName = entityPersistenceNamingStrategy.getEntityName(MappingEntry.class).orElseThrow(); this.queryFull = createBaseQueryString(schemaName, tableName); } @Override - public Stream> getMappingSourceData() { + public Stream> getMappingSourceData(SourceValidator validator) { + try { + dataSource.validateDBTable(tableName, TimeSeriesMappingSource.class, validator); + } catch (SQLException e) { + throw new FactoryException(e); + } + return dataSource.executeQuery(queryFull); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java index 126aefae5..3784d13c8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java @@ -19,6 +19,7 @@ import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.utils.TimeSeriesUtils; import edu.ie3.util.interval.ClosedInterval; +import java.sql.SQLException; import java.sql.Timestamp; import java.time.ZonedDateTime; import java.util.*; @@ -50,7 +51,8 @@ public SqlTimeSeriesSource( SqlDataSource sqlDataSource, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) { + TimeBasedSimpleValueFactory factory) + throws SQLException { super(valueClass, factory); this.dataSource = sqlDataSource; @@ -63,6 +65,8 @@ public SqlTimeSeriesSource( final String tableName = sqlDataSource.databaseNamingStrategy.getTimeSeriesEntityName(columnScheme); + dataSource.validateDBTable(tableName, valueClass, factory); + String dbTimeColumnName = sqlDataSource.getDbColumnName(factory.getTimeFieldString(), tableName); @@ -88,7 +92,8 @@ public SqlTimeSeriesSource( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) { + TimeBasedSimpleValueFactory factory) + throws SQLException { this( new SqlDataSource(connector, schemaName, namingStrategy), timeSeriesUuid, @@ -113,7 +118,7 @@ public static SqlTimeSeriesSource createSource( DatabaseNamingStrategy namingStrategy, IndividualTimeSeriesMetaInformation metaInformation, String timePattern) - throws SourceException { + throws SourceException, SQLException { if (!TimeSeriesUtils.isSchemeAccepted(metaInformation.getColumnScheme())) throw new SourceException( "Unsupported column scheme '" + metaInformation.getColumnScheme() + "'."); @@ -130,7 +135,8 @@ private static SqlTimeSeriesSource create( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valClass, - String timePattern) { + String timePattern) + throws SQLException { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass, timePattern); return new SqlTimeSeriesSource<>( diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy index 424ad10ec..10395b2c8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy @@ -274,31 +274,34 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe } } - def "An AssetInputFactory should throw an exception on invalid or incomplete data "() { - given: "a system participant input type factory and model data" + def "An AssetInputFactory should throw an exception on invalid or incomplete fields"() { + given: def inputFactory = new TestAssetInputFactory() - Map parameter = [ - "uuid" : "91ec3bcf-1777-4d38-af67-0bf7c9fa73c7", - "operatesfrom" : "2019-01-01T00:00:00+01:00[Europe/Berlin]", - "operatesuntil": "2019-12-31T00:00:00+01:00[Europe/Berlin]" - ] - def inputClass = TestAssetInput + def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until") when: - Try input = inputFactory.get(new AssetInputEntityData(parameter, inputClass)) + Try input = Try.of(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) then: input.failure - input.exception.get().cause.message == - "The provided fields [operatesfrom, operatesuntil, uuid] with data \n" + - "{operatesfrom -> 2019-01-01T00:00:00+01:00[Europe/Berlin],\n" + - "operatesuntil -> 2019-12-31T00:00:00+01:00[Europe/Berlin],\n" + - "uuid -> 91ec3bcf-1777-4d38-af67-0bf7c9fa73c7} are invalid for instance of TestAssetInput. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + - "0: [id, uuid]\n" + - "1: [id, operatesfrom, uuid]\n" + - "2: [id, operatesuntil, uuid]\n" + - "3: [id, operatesfrom, operatesuntil, uuid]\n" + input.exception.get().message == "The provided fields [operates_from, operates_until, uuid] are invalid for instance of TestAssetInput. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + + "0: [id, uuid] or [id, uuid]\n" + + "1: [id, operates_from, uuid] or [id, operatesFrom, uuid]\n" + + "2: [id, operates_until, uuid] or [id, operatesUntil, uuid]\n" + + "3: [id, operates_from, operates_until, uuid] or [id, operatesFrom, operatesUntil, uuid]\n" + } + + def "An AssetInputFactory should allow additional fields"() { + given: + def inputFactory = new TestAssetInputFactory() + def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until", "id", "additional_field") + + when: + Try input = Try.of(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) + + then: + input.success } private static class TestAssetInput extends AssetInput { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy index 1c12b1e27..c357481f1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.factory.input import edu.ie3.datamodel.exceptions.FactoryException +import edu.ie3.datamodel.exceptions.TryException import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.OperatorInput import edu.ie3.datamodel.models.input.connector.Transformer2WInput @@ -85,10 +86,12 @@ class Transformer2WInputFactoryTest extends Specification implements FactoryTest def typeInput = Mock(Transformer2WTypeInput) when: - inputFactory.get(new TypedConnectorInputEntityData(parameter, inputClass, operatorInput, nodeInputA, nodeInputB, typeInput)) + Try input = inputFactory.get(new TypedConnectorInputEntityData(parameter, inputClass, operatorInput, nodeInputA, nodeInputB, typeInput)) then: - def e = thrown(IllegalArgumentException) - e.message == "nodeA must be on the higher voltage side of the transformer" + input.failure + def e = input.exception.get() + e.cause.class == IllegalArgumentException + e.cause.message == "nodeA must be on the higher voltage side of the transformer" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactoryTest.groovy index f518207c2..4c74bf061 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer3WInputFactoryTest.groovy @@ -9,6 +9,7 @@ import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.models.OperationTime import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.OperatorInput +import edu.ie3.datamodel.models.input.connector.Transformer2WInput import edu.ie3.datamodel.models.input.connector.Transformer3WInput import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput import edu.ie3.datamodel.utils.Try @@ -79,11 +80,13 @@ class Transformer3WInputFactoryTest extends Specification implements FactoryTes def typeInput = Mock(Transformer3WTypeInput) when: - inputFactory.get(new Transformer3WInputEntityData(parameter, inputClass, nodeInputA, nodeInputB, nodeInputC, typeInput)) + Try input = inputFactory.get(new Transformer3WInputEntityData(parameter, inputClass, nodeInputA, nodeInputB, nodeInputC, typeInput)) then: - def e = thrown(IllegalArgumentException) - e.message == "Voltage level of node a must be greater than voltage level of node b and voltage level of node b must be greater than voltage level of node c" + input.failure + def e = input.exception.get() + e.cause.class == IllegalArgumentException + e.cause.message == "Voltage level of node a must be greater than voltage level of node b and voltage level of node b must be greater than voltage level of node c" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactoryTest.groovy index 4f3412e23..50ea280f1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/EvcsInputFactoryTest.groovy @@ -139,6 +139,6 @@ class EvcsInputFactoryTest extends Specification implements FactoryTestHelper { then: input.failure - input.exception.get().cause.message == "Exception while trying to parse field \"locationtype\" with supposed int value \"-- invalid --\"" + input.exception.get().cause.message == "Exception while trying to parse field \"locationType\" with supposed int value \"-- invalid --\"" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy index fa7184b4b..e7a46b0c8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy @@ -73,32 +73,21 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe } } - def "A FixedFeedInInputFactory should throw an exception on invalid or incomplete data (parameter missing)"() { - given: "a system participant input type factory and model data" + def "A FixedFeedInInputFactory should throw an exception on invalid or incomplete data fields"() { + given: def inputFactory = new FixedFeedInInputFactory() - Map parameter = [ - "uuid" : "91ec3bcf-1777-4d38-af67-0bf7c9fa73c7", - "id" : "TestID", - "srated" : "3", - "cosphirated": "4" - ] - def inputClass = FixedFeedInInput - def nodeInput = Mock(NodeInput) + def foundFields = inputFactory.newSet("uuid", "id", "s_rated", "cosphi_rated") when: - Try input = inputFactory.get(new NodeAssetInputEntityData(parameter, inputClass, nodeInput)) + Try input = Try.of(() -> inputFactory.validate(foundFields, FixedFeedInInput), FactoryException) then: input.failure - input.exception.get().cause.message == "The provided fields [cosphirated, id, srated, uuid] with data \n" + - "{cosphirated -> 4,\n" + - "id -> TestID,\n" + - "srated -> 3,\n" + - "uuid -> 91ec3bcf-1777-4d38-af67-0bf7c9fa73c7} are invalid for instance of FixedFeedInInput. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + - "0: [cosphirated, id, qcharacteristics, srated, uuid]\n" + - "1: [cosphirated, id, operatesfrom, qcharacteristics, srated, uuid]\n" + - "2: [cosphirated, id, operatesuntil, qcharacteristics, srated, uuid]\n" + - "3: [cosphirated, id, operatesfrom, operatesuntil, qcharacteristics, srated, uuid]\n" + input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of FixedFeedInInput. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + + "0: [cos_phi_rated, id, q_characteristics, s_rated, uuid] or [cosPhiRated, id, qCharacteristics, sRated, uuid]\n" + + "1: [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid]\n" + + "2: [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid]\n" + + "3: [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy index c46eabaea..b9bfa7709 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy @@ -53,25 +53,16 @@ class FlexOptionsResultFactoryTest extends Specification implements FactoryTestH def "A FlexOptionsResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new FlexOptionsResultFactory() - Map parameter = [ - "time" : "2020-01-30 17:26:44", - "inputModel": "91ec3bcf-1897-4d38-af67-0bf7c9fa73c7", - "pref" : "2", - "pmin" : "-1", - ] + def foundFields = resultFactory.newSet("time", "input_model", "p_ref", "p_min") when: - Try input = resultFactory.get(new SimpleEntityData(parameter, FlexOptionsResult)) + Try input = Try.of(() -> resultFactory.validate(foundFields, FlexOptionsResult), FactoryException) then: input.failure - input.exception.get().cause.message == "The provided fields [inputModel, pmin, pref, time] with data \n" + - "{inputModel -> 91ec3bcf-1897-4d38-af67-0bf7c9fa73c7,\n" + - "pmin -> -1,\n" + - "pref -> 2,\n" + - "time -> 2020-01-30 17:26:44} are invalid for instance of FlexOptionsResult. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, pmax, pmin, pref, time]\n" + - "1: [inputModel, pmax, pmin, pref, time, uuid]\n" + input.exception.get().message == "The provided fields [input_model, p_min, p_ref, time] are invalid for instance of FlexOptionsResult. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + + "0: [input_model, p_max, p_min, p_ref, time] or [inputModel, pMax, pMin, pRef, time]\n" + + "1: [input_model, p_max, p_min, p_ref, time, uuid] or [inputModel, pMax, pMin, pRef, time, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy index 9dbc4e294..2e6f50a85 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy @@ -51,23 +51,16 @@ class NodeResultFactoryTest extends Specification implements FactoryTestHelper { def "A NodeResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new NodeResultFactory() - Map parameter = [ - "time" : "2020-01-30 17:26:44", - "inputModel": "91ec3bcf-1897-4d38-af67-0bf7c9fa73c7", - "vmag" : "2" - ] + def foundFields = resultFactory.newSet("time", "input_model", "v_mag") when: - Try input = resultFactory.get(new SimpleEntityData(parameter, NodeResult)) + Try input = Try.of(() -> resultFactory.validate(foundFields, NodeResult), FactoryException) then: input.failure - input.exception.get().cause.message == "The provided fields [inputModel, time, vmag] with data \n" + - "{inputModel -> 91ec3bcf-1897-4d38-af67-0bf7c9fa73c7,\n" + - "time -> 2020-01-30 17:26:44,\n" + - "vmag -> 2} are invalid for instance of NodeResult. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, time, vang, vmag]\n" + - "1: [inputModel, time, uuid, vang, vmag]\n" + input.exception.get().message == "The provided fields [input_model, time, v_mag] are invalid for instance of NodeResult. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + + "0: [input_model, time, v_ang, v_mag] or [inputModel, time, vAng, vMag]\n" + + "1: [input_model, time, uuid, v_ang, v_mag] or [inputModel, time, uuid, vAng, vMag]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy index 5ca875cda..947a5f5c5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy @@ -127,23 +127,17 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor def "A SystemParticipantResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new SystemParticipantResultFactory() - Map parameter = [ - "time" : "2020-01-30 17:26:44", - "inputModel": "91ec3bcf-1777-4d38-af67-0bf7c9fa73c7", - "q" : "2" - ] + def foundFields = resultFactory.newSet("time", "input_model", "q") + when: - Try result = resultFactory.get(new SimpleEntityData(parameter, WecResult)) + Try result = Try.of(() -> resultFactory.validate(foundFields, WecResult), FactoryException) then: result.failure - result.exception.get().cause.message == "The provided fields [inputModel, q, time] with data \n" + - "{inputModel -> 91ec3bcf-1777-4d38-af67-0bf7c9fa73c7,\n" + - "q -> 2,\n" + - "time -> 2020-01-30 17:26:44} are invalid for instance of WecResult. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, p, q, time]\n" + - "1: [inputModel, p, q, time, uuid]\n" + result.exception.get().message == "The provided fields [input_model, q, time] are invalid for instance of WecResult. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + + "0: [input_model, p, q, time] or [inputModel, p, q, time]\n" + + "1: [input_model, p, q, time, uuid] or [inputModel, p, q, time, uuid]\n" } def "A SystemParticipantResultFactory should be performant"() { @@ -162,7 +156,7 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor resultFactory.get(new SimpleEntityData(parameter, StorageResult)) } BigDecimal elapsedTime = (System - .currentTimeMillis() - startTime) / 1000.0 + .currentTimeMillis() - startTime) / 1000.0 elapsedTime < 2 } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index 117999ce4..ad189ae73 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -5,7 +5,9 @@ */ package edu.ie3.datamodel.io.factory.timeseries +import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.io.factory.SimpleFactoryData +import edu.ie3.datamodel.utils.Try import edu.ie3.util.geo.GeoUtils import org.apache.commons.lang3.tuple.Pair import org.locationtech.jts.geom.Point @@ -25,27 +27,14 @@ class CosmoIdCoordinateFactoryTest extends Specification { def expectedFields = [ "tid", "id", - "latrot", - "longrot", - "latgeo", - "longgeo" + "latRot", + "longRot", + "latGeo", + "longGeo" ] as Set - Map parameter = [ - "tid": "1", - "id": "106580", - "latgeo": "39.602772", - "longgeo": "1.279336", - "latrot": "-10", - "longrot": "-6.8125" - ] - - - def validSimpleFactoryData = new SimpleFactoryData(parameter, Pair) - - when: - def actual = factory.getFields(validSimpleFactoryData) + def actual = factory.getFields(Pair) then: actual.size() == 1 @@ -54,22 +43,16 @@ class CosmoIdCoordinateFactoryTest extends Specification { def "A COSMO id to coordinate factory refuses to build from invalid data"() { given: - Map parameter = [ - "tid": "1", - "id": "106580", - "latrot": "-10", - "longrot": "-6.8125" - ] - - def invalidSimpleFactoryData = new SimpleFactoryData(parameter, Pair) + def foundFields = factory.newSet("tid", "id", "latrot", "longrot") when: - def actual = factory.get(invalidSimpleFactoryData) + def actual = Try.of(() -> factory.validate(foundFields, Pair), FactoryException) then: actual.failure - actual.exception.get().cause.message.startsWith("The provided fields [id, latrot, longrot, tid] with data \n{id -> 106580,\nlatrot" + - " -> -10,\nlongrot -> -6.8125,\ntid -> 1} are invalid for instance of Pair.") + actual.exception.get().message == "The provided fields [id, latrot, longrot, tid] are invalid for instance of Pair. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + + "0: [id, lat_geo, lat_rot, long_geo, long_rot, tid] or [id, latGeo, latRot, longGeo, longRot, tid]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index b40b44387..a38901edd 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -5,7 +5,9 @@ */ package edu.ie3.datamodel.io.factory.timeseries +import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.io.factory.SimpleFactoryData +import edu.ie3.datamodel.utils.Try import edu.ie3.util.geo.GeoUtils import org.apache.commons.lang3.tuple.Pair import org.locationtech.jts.geom.Point @@ -26,18 +28,11 @@ class IconIdCoordinateFactoryTest extends Specification { "id", "latitude", "longitude", - "coordinatetype" + "coordinateType" ] as Set - Map parameter = [ - "id":"477295", - "latitude":"52.312", - "longitude":"12.812", - "coordinatetype":"ICON"] - - def validSimpleFactoryData = new SimpleFactoryData(parameter, Pair) when: - def actual = factory.getFields(validSimpleFactoryData) + def actual = factory.getFields(Pair) then: actual.size() == 1 @@ -46,20 +41,16 @@ class IconIdCoordinateFactoryTest extends Specification { def "A COSMO id to coordinate factory refuses to build from invalid data"() { given: - Map parameter = [ - "id":"477295", - "latitude":"52.312", - "coordinatetype":"ICON"] - - def invalidSimpleFactoryData = new SimpleFactoryData(parameter, Pair) + def foundFields = factory.newSet("id", "latitude", "coordinatetype") when: - def actual = factory.get(invalidSimpleFactoryData) + def actual = Try.of(() -> factory.validate(foundFields, Pair), FactoryException) then: actual.failure - actual.exception.get().cause.message.startsWith("The provided fields [coordinatetype, id, latitude] with data \n{coordinatetype -> " + - "ICON,\nid -> 477295,\nlatitude -> 52.312} are invalid for instance of Pair. ") + actual.exception.get().message == "The provided fields [coordinatetype, id, latitude] are invalid for instance of Pair. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + + "0: [coordinate_type, id, latitude, longitude] or [coordinateType, id, latitude, longitude]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy index a91436c4e..e3169fc9e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy @@ -34,7 +34,7 @@ class TimeBasedSimpleValueFactoryTest extends Specification { data.targetClass >> valueClass expect: - factory.getFields(data) == expectedFields + factory.getFields(data.targetClass) == expectedFields where: valueClass || expectedFields @@ -93,7 +93,7 @@ class TimeBasedSimpleValueFactoryTest extends Specification { data.targetClass >> NodeInput when: - factory.getFields(data) + factory.getFields(data.targetClass) then: def e = thrown(FactoryException) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy index b5f40c2ac..7e44fc75c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy @@ -280,40 +280,15 @@ class SystemParticipantTypeInputFactoryTest extends Specification implements Fac def "A SystemParticipantTypeInputFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def typeInputFactory = new SystemParticipantTypeInputFactory() - Map parameter = [ - "uuid": "91ec3bcf-1777-4d38-af67-0bf7c9fa73c7", - "id": "blablub", - "capex": "3", - "opex": "4", - "srated": "5", - "cosPhiRated": "6", - "estorage": "6", - "pmin": "7", - "pmax": "8", - "eta": "9", - "dod": "10", - "lifetime": "11" - ] + def foundFields = typeInputFactory.newSet("uuid", "id", "capex", "opex", "srated", "cosPhiRated", "estorage", "pmin", "pmax", "eta", "dod", "lifetime",) when: - Try input = typeInputFactory.get(new SimpleEntityData(parameter, StorageTypeInput)) + def input = Try.of(() -> typeInputFactory.validate(foundFields, StorageTypeInput), FactoryException) then: input.failure - input.exception.get().cause.message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] with data \n" + - "{capex -> 3,\n" + - "cosPhiRated -> 6,\n" + - "dod -> 10,\n" + - "estorage -> 6,\n" + - "eta -> 9,\n" + - "id -> blablub,\n" + - "lifetime -> 11,\n" + - "opex -> 4,\n" + - "pmax -> 8,\n" + - "pmin -> 7,\n" + - "srated -> 5,\n" + - "uuid -> 91ec3bcf-1777-4d38-af67-0bf7c9fa73c7} are invalid for instance of StorageTypeInput. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + - "0: [activepowergradient, capex, cosphirated, dod, estorage, eta, id, lifecycle, lifetime, opex, pmax, srated, uuid]\n" + input.exception.get().message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] are invalid for instance of StorageTypeInput. \n" + + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + + "0: [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid] or [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy index de1857954..3dbf7940c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy @@ -35,7 +35,7 @@ class CsvTimeSeriesSourceIT extends Specification implements CsvTestDataMeta { def filePath = Path.of("file/not/found.csv") when: - source.buildIndividualTimeSeries(UUID.fromString("fbc59b5b-9307-4fb4-a406-c1f08f26fee5"), filePath, { null }) + source.buildIndividualTimeSeries(UUID.fromString("fbc59b5b-9307-4fb4-a406-c1f08f26fee5"), filePath, { null }, null) then: def ex = thrown(SourceException) @@ -49,7 +49,7 @@ class CsvTimeSeriesSourceIT extends Specification implements CsvTestDataMeta { def tsUuid = UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7") when: - def actual = source.buildIndividualTimeSeries(tsUuid, filePath, { source.createTimeBasedValue(it) }) + def actual = source.buildIndividualTimeSeries(tsUuid, filePath, { source.createTimeBasedValue(it) }, HeatAndPValue) then: noExceptionThrown() diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 25fc33d40..e4873c75e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -120,12 +120,12 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", - "coordinateid" : "5", - "directirradiance" : "1.234", - "diffuseirradiance": "5.678", + "coordinateId" : "5", + "directIrradiance" : "1.234", + "diffuseIrradiance": "5.678", "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "windVelocity" : "12.1314", + "windDirection" : "15.1617" ] def expectedValue = new TimeBasedValue( UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv index b9ddc711d..cf9538fba 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv @@ -1,3 +1,3 @@ -"uuid","cos_phi_rated","id","node","operates_from","operates_until","operator","q_characteristics","chargingpoints","type","location_type","v2gSupport" +"uuid","cos_phi_rated","id","node","operates_from","operates_until","operator","q_characteristics","charging_points","type","location_type","v2gSupport" 06a14909-366e-4e94-a593-1016e1455b30,0.9,test_evcs_1,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,cosPhiFixed:{(0.00,1.0)},4,ChargingStationType1,HOME,false 104acdaa-5dc5-4197-aed2-2fddb3c4f237,0.9,test_evcs_2,ed4697fd-016c-40c2-a66b-e793878dadea,,,,cosPhiFixed:{(0.00,1.0)},4,ChargingStationType1,HOME,false \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/fixed_feed_in_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/fixed_feed_in_input.csv index 0cf4a3ec4..b96a7bb37 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/fixed_feed_in_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/fixed_feed_in_input.csv @@ -1,2 +1,2 @@ -uuid,cosphi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated +uuid,cos_phi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated 717af017-cc69-406f-b452-e022d7fb516a,0.95,test_fixedFeedInInput,4ca90220-74c2-4369-9afa-a18bf068840d,2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,"cosPhiFixed:{(0.00,0.95)}",25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/load_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/load_input.csv index d961a158b..c35f3fe6a 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/load_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/load_input.csv @@ -1,2 +1,2 @@ -uuid,cosphi_rated,dsm,e_cons_annual,id,node,operates_from,operates_until,operator,q_characteristics,s_rated,load_profile +uuid,cos_phi_rated,dsm,e_cons_annual,id,node,operates_from,operates_until,operator,q_characteristics,s_rated,load_profile eaf77f7e-9001-479f-94ca-7fb657766f5f,0.95,false,4000.0,test_loadInput,4ca90220-74c2-4369-9afa-a18bf068840d,2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,"cosPhiFixed:{(0.00,0.95)}",25.0,h0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/pv_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/pv_input.csv index 70b1881a7..795d9a173 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/pv_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/pv_input.csv @@ -1,2 +1,2 @@ -uuid,albedo,azimuth,cosphi_rated,eta_conv,elevation_angle,id,k_g,k_t,market_reaction,node,operates_from,operates_until,operator,q_characteristics,s_rated +uuid,albedo,azimuth,cos_phi_rated,eta_conv,elevation_angle,id,k_g,k_t,market_reaction,node,operates_from,operates_until,operator,q_characteristics,s_rated d56f15b7-8293-4b98-b5bd-58f6273ce229,0.20000000298023224,-8.926613807678223,0.95,98.0,41.01871871948242,test_pvInput,0.8999999761581421,1.0,false,4ca90220-74c2-4369-9afa-a18bf068840d,2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,"cosPhiFixed:{(0.00,0.95)}",25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/time_series_mapping.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/time_series_mapping.csv index e03b276ad..565fcf11c 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/time_series_mapping.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_participants/time_series_mapping.csv @@ -1,4 +1,4 @@ -"uuid";"participant";"time_series" -"58167015-d760-4f90-8109-f2ebd94cda91";"b86e95b0-e579-4a80-a534-37c7a470a409";"67600124-2475-4a62-a410-0dd6eabb9441" -"9a9ebfda-dc26-4a40-b9ca-25cd42f6cc3f";"c7ebcc6c-55fc-479b-aa6b-6fa82ccac6b8";"05a25fe7-11e5-4732-92b0-490cec171c78" -"9c1c53ea-e575-41a2-a373-a8b2d3ed2c39";"90a96daa-012b-4fea-82dc-24ba7a7ab81c";"05a25fe7-11e5-4732-92b0-490cec171c78" \ No newline at end of file +"uuid","participant","time_series" +"58167015-d760-4f90-8109-f2ebd94cda91","b86e95b0-e579-4a80-a534-37c7a470a409","67600124-2475-4a62-a410-0dd6eabb9441" +"9a9ebfda-dc26-4a40-b9ca-25cd42f6cc3f","c7ebcc6c-55fc-479b-aa6b-6fa82ccac6b8","05a25fe7-11e5-4732-92b0-490cec171c78" +"9c1c53ea-e575-41a2-a373-a8b2d3ed2c39","90a96daa-012b-4fea-82dc-24ba7a7ab81c","05a25fe7-11e5-4732-92b0-490cec171c78" \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_results/thermal_house_res.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_results/thermal_house_res.csv index 19d5398ac..9b92587d4 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_results/thermal_house_res.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_results/thermal_house_res.csv @@ -1 +1 @@ -uuid,inputModel,qDot,indoorTemperature \ No newline at end of file +uuid,inputModel,qDot,indoorTemperature,time \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/bm_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/bm_type_input.csv index 742bbfb0e..e00adbd76 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/bm_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/bm_type_input.csv @@ -1,2 +1,2 @@ -uuid,active_power_gradient,capex,cosphi_rated,eta_conv,id,opex,s_rated +uuid,active_power_gradient,capex,cos_phi_rated,eta_conv,id,opex,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/chp_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/chp_type_input.csv index 4e19b862a..416890a66 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/chp_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/chp_type_input.csv @@ -1,2 +1,2 @@ -uuid,capex,cosphi_rated,eta_el,eta_thermal,id,opex,p_own,p_thermal,s_rated +uuid,capex,cos_phi_rated,eta_el,eta_thermal,id,opex,p_own,p_thermal,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,100.0,0.95,19.0,76.0,test_chpType,50.0,0.0,9.0,25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/ev_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/ev_type_input.csv index ab07e35fa..b27fe828c 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/ev_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/ev_type_input.csv @@ -1,2 +1,2 @@ -uuid,capex,cosphi_rated,e_cons,e_storage,id,opex,s_rated +uuid,capex,cos_phi_rated,e_cons,e_storage,id,opex,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,100.0,0.95,5.0,100.0,test_evTypeInput,50.0,25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/hp_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/hp_type_input.csv index 6f943036a..d41d6d38c 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/hp_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/hp_type_input.csv @@ -1,2 +1,2 @@ -uuid,capex,cosphi_rated,id,opex,p_thermal,s_rated +uuid,capex,cos_phi_rated,id,opex,p_thermal,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,100.0,0.95,test_hpTypeInput,50.0,9.0,25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/storage_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/storage_type_input.csv index 1c437b952..4497b6159 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/storage_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/storage_type_input.csv @@ -1,2 +1,2 @@ -uuid,active_power_gradient,capex,cosphi_rated,dod,e_storage,eta,id,life_cycle,life_time,opex,p_max,s_rated +uuid,active_power_gradient,capex,cos_phi_rated,dod,e_storage,eta,id,life_cycle,life_time,opex,p_max,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,100.0,100.0,0.95,10.0,100.0,95.0,test_storageTypeInput,100,175316.4,50.0,15.0,25.0 diff --git a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/wec_type_input.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/wec_type_input.csv index 43184d4e0..0fdac436e 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/wec_type_input.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/_types/wec_type_input.csv @@ -1,2 +1,2 @@ -uuid,capex,cosphi_rated,cp_characteristic,eta_conv,hub_height,id,opex,rotor_area,s_rated +uuid,capex,cos_phi_rated,cp_characteristic,eta_conv,hub_height,id,opex,rotor_area,s_rated 5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,100.0,0.95,"cP:{(10.00,0.05),(15.00,0.10),(20.00,0.20)}",98.0,200.0,test_wecType,50.0,20.0,25.0 From 065e15b2268ae3f3e4c5c2a54dc007ae9554de57 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 13 Nov 2023 15:27:55 +0100 Subject: [PATCH 02/20] Some improvements. --- .../io/connectors/CouchbaseConnector.java | 32 +++++++++ .../io/connectors/InfluxDbConnector.java | 23 +++++++ .../datamodel/io/connectors/SqlConnector.java | 23 +++++++ .../edu/ie3/datamodel/io/factory/Factory.java | 66 ++++--------------- .../CosmoTimeBasedWeatherValueFactory.java | 6 +- .../IconTimeBasedWeatherValueFactory.java | 14 ++-- .../couchbase/CouchbaseWeatherSource.java | 2 + .../influxdb/InfluxDbWeatherSource.java | 4 +- .../io/source/sql/SqlDataSource.java | 24 +------ .../io/source/sql/SqlIdCoordinateSource.java | 2 +- .../sql/SqlTimeSeriesMappingSource.java | 2 +- .../io/source/sql/SqlTimeSeriesSource.java | 2 +- .../io/source/sql/SqlWeatherSource.java | 5 +- .../CosmoIdCoordinateFactoryTest.groovy | 8 +-- ...smoTimeBasedWeatherValueFactoryTest.groovy | 16 ++--- .../IconIdCoordinateFactoryTest.groovy | 2 +- .../TimeBasedSimpleValueFactoryTest.groovy | 6 +- 17 files changed, 127 insertions(+), 110 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java index 196a3beef..99230558e 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java @@ -9,10 +9,15 @@ import com.couchbase.client.java.AsyncCollection; import com.couchbase.client.java.Cluster; import com.couchbase.client.java.Collection; +import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.kv.MutationResult; import com.couchbase.client.java.query.QueryResult; +import edu.ie3.datamodel.io.source.SourceValidator; +import java.time.Duration; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; /** @@ -36,6 +41,33 @@ public CouchbaseConnector(String url, String bucketName, String username, String cluster = Cluster.connect(url, username, password); } + /** + * This method should be used to validate a given couchbaseDb. + * + * @param entityClass class of the entity + * @param validator for validation + */ + @SuppressWarnings("unchecked") + public final void validateDb(Class entityClass, SourceValidator validator) { + String query = + "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM " + + bucketName + + " b UNNEST OBJECT_NAMES(b) AS v"; + cluster.bucket(bucketName).waitUntilReady(Duration.ofSeconds(30)); + + QueryResult queryResult = query(query).join(); + JsonObject jsonObject = queryResult.rowsAsObject().get(0); + Object columns = jsonObject.toMap().get("column"); + + Set set = new HashSet<>(); + + if (columns != null) { + set.addAll((List) columns); + } + + validator.validate(set, entityClass); + } + /** * Return the couchbase java sdk equivalent of a session - a collection - to the previously set * bucket diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java index 685e16614..bd1b700cc 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.connectors; +import edu.ie3.datamodel.io.source.SourceValidator; import java.util.*; import java.util.function.BinaryOperator; import java.util.stream.Collectors; @@ -38,6 +39,7 @@ public class InfluxDbConnector implements DataConnector { return maps; }; + private final String databaseName; private final String scenarioName; private final InfluxDB session; @@ -77,6 +79,7 @@ public InfluxDbConnector( */ public InfluxDbConnector( InfluxDB session, String scenarioName, String databaseName, boolean createDb) { + this.databaseName = databaseName; this.scenarioName = scenarioName; this.session = session; @@ -107,6 +110,26 @@ public InfluxDbConnector(String url, String databaseName) { this(url, databaseName, NO_SCENARIO, true, InfluxDB.LogLevel.NONE, BatchOptions.DEFAULTS); } + /** + * This method should be used to validate a given {@link InfluxDB}. + * + * @param entityClass class of the entity + * @param validator for validation + */ + public final void validateDb(Class entityClass, SourceValidator validator) { + QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName)); + Map>> tagResults = parseQueryResult(tagKeys); + + QueryResult fieldKeys = session.query(new Query("SHOW FIELD KEYS ON " + databaseName)); + Map>> fieldResults = parseQueryResult(fieldKeys); + + Set set = new HashSet<>(); + tagResults.values().forEach(v -> v.stream().map(m -> m.get("tagKey")).forEach(set::add)); + fieldResults.values().forEach(v -> v.stream().map(m -> m.get("fieldKey")).forEach(set::add)); + + validator.validate(set, entityClass); + } + /** * Create the database of this connector if it doesn't exist yet * diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index 8b13ec0d0..d88b4894e 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.connectors; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.util.StringUtils; import edu.ie3.util.TimeUtil; import java.sql.*; @@ -41,6 +42,28 @@ public SqlConnector(String jdbcUrl, String userName, String password) { connectionProps.put("password", password); } + /** + * This method should be used to validate a given sql table. + * + * @param tableName name of the table + * @param entityClass class of the entity + * @param validator for validation + * @throws SQLException – if the connection could not be established + */ + public void validateDBTable(String tableName, Class entityClass, SourceValidator validator) + throws SQLException { + ResultSet rs = getConnection().getMetaData().getColumns(null, null, tableName, null); + + Set columnNames = new HashSet<>(); + + while (rs.next()) { + String name = rs.getString("COLUMN_NAME"); + columnNames.add(StringUtils.snakeCaseToCamelCase(name)); + } + + validator.validate(columnNames, entityClass); + } + /** * Executes the given query. For update queries please use {@link * SqlConnector#executeUpdate(String)}. diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 1263cc5f2..e9e7a93f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -109,7 +109,9 @@ private void isSupportedClass(Class desiredClass) { protected abstract List> getFields(Class entityClass); /** - * Method for validating the found fields. + * Method for validating the found fields. The found fields needs to fully contain at least one of the sets returned by {@link #getFields(Class)}. + * If the found fields don't contain all necessary fields, an {@link FactoryException} with a detail message is thrown. + * If the found fields contain more fields than necessary, these fields are ignored. * * @param foundFields that were found * @param entityClass of the build data @@ -146,58 +148,6 @@ public void validate(Set foundFields, Class entityClass) { } } - /** - * Validates the factory specific constructor parameters in two ways. 1) the biggest set of the - * provided field sets is compared against fields the class implements. If this test passes then - * we know for sure that the field names at least in the biggest constructor are equal to the - * provided factory strings 2) if 1) passes, the provided entity data (which is equal to the data - * e.g. read from the outside) is compared to all available constructor parameters provided by the - * fieldSets Array. If we find exactly one constructor, that matches the field names we can - * proceed. Otherwise a detailed exception message is thrown. - * - * @param data the entity containing at least the entity class as well a mapping of the provided - * field name strings to its value (e.g. a headline of a csv to column values) - * @param fieldSets a set containing all available constructor combinations as field names - * @return the index of the set in the fieldSets array that fits the provided entity data - */ - protected int validateParameters(D data, Set... fieldSets) { - Map fieldsToValues = data.getFieldsToValues(); - - // get all sets that match the fields to attributes - List> validFieldSets = - Arrays.stream(fieldSets).filter(x -> x.equals(fieldsToValues.keySet())).toList(); - - if (validFieldSets.size() == 1) { - // if we can identify a unique parameter set for a constructor, we take it and return the - // index - Set validFieldSet = validFieldSets.get(0); - return Arrays.asList(fieldSets).indexOf(validFieldSet); - } else { - // build the exception string with extensive debug information - String providedFieldMapString = - fieldsToValues.keySet().stream() - .map(key -> key + " -> " + fieldsToValues.get(key)) - .collect(Collectors.joining(",\n")); - - String providedKeysString = "[" + String.join(", ", fieldsToValues.keySet()) + "]"; - - String possibleOptions = getFieldsString(List.of(fieldSets)).toString(); - - throw new FactoryException( - "The provided fields " - + providedKeysString - + " with data \n{" - + providedFieldMapString - + "}" - + " are invalid for instance of " - + data.getTargetClass().getSimpleName() - + ". \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" - + data.getTargetClass().getSimpleName() - + "' are possible (NOT case-sensitive!):\n" - + possibleOptions); - } - } - protected static StringBuilder getFieldsString(List> fieldSets) { StringBuilder possibleOptions = new StringBuilder(); for (int i = 0; i < fieldSets.size(); i++) { @@ -243,15 +193,21 @@ protected TreeSet expandSet(Set attributeSet, String... more) { return newSet; } - private static Set toSnakeCase(Set set) { + protected static Set toSnakeCase(Set set) { TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); newSet.addAll(set.stream().map(StringUtils::camelCaseToSnakeCase).toList()); return newSet; } - private static Set toCamelCase(Set set) { + protected static Set toCamelCase(Set set) { TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); newSet.addAll(set.stream().map(StringUtils::snakeCaseToCamelCase).toList()); return newSet; } + + protected static Set toLowerCase(Set set) { + TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + newSet.addAll(set.stream().map(String::toLowerCase).toList()); + return newSet; + } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 1bce91906..21c61a9ec 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -12,7 +12,6 @@ import edu.ie3.util.quantities.PowerSystemUnits; import edu.ie3.util.quantities.interfaces.Irradiance; import java.time.ZonedDateTime; -import java.util.Collections; import java.util.List; import java.util.Set; import javax.measure.quantity.Angle; @@ -53,14 +52,13 @@ public String getTimeFieldString() { protected List> getFields(Class entityClass) { Set minConstructorParams = newSet( - UUID, - TIME, + COORDINATE_ID, DIFFUSE_IRRADIANCE, DIRECT_IRRADIANCE, TEMPERATURE, WIND_DIRECTION, WIND_VELOCITY); - return Collections.singletonList(minConstructorParams); + return List.of(minConstructorParams, toLowerCase(minConstructorParams)); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index 20d8e5bef..e1b095116 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -56,12 +56,7 @@ public String getTimeFieldString() { protected List> getFields(Class entityClass) { Set minParameters = newSet( - TIME, - DIFFUSE_IRRADIANCE, - DIRECT_IRRADIANCE, - TEMPERATURE, - WIND_VELOCITY_U, - WIND_VELOCITY_V); + DIFFUSE_IRRADIANCE, DIRECT_IRRADIANCE, TEMPERATURE, WIND_VELOCITY_U, WIND_VELOCITY_V); Set allParameters = expandSet( minParameters, @@ -92,7 +87,12 @@ protected List> getFields(Class entityClass) { Set allParametersWithUuid = expandSet(allParameters, UUID); return Arrays.asList( - minParameters, allParameters, minParametersWithUuid, allParametersWithUuid); + minParameters, + allParameters, + minParametersWithUuid, + allParametersWithUuid, + toLowerCase(minParameters), + toLowerCase(allParameters)); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index abfaca204..e84aef130 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -94,6 +94,8 @@ public CouchbaseWeatherSource( this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.timeStampPattern = timeStampPattern; + + connector.validateDb(WeatherValue.class, weatherFactory); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index cd769fc26..78a317c4b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -50,6 +50,8 @@ public InfluxDbWeatherSource( TimeBasedWeatherValueFactory weatherValueFactory) { super(idCoordinateSource, weatherValueFactory); this.connector = connector; + + connector.validateDb(WeatherValue.class, weatherValueFactory); } @Override @@ -224,6 +226,6 @@ private String createCoordinateConstraintString(int coordinateId) { */ protected Stream> filterEmptyOptionals( Stream>> elements) { - return elements.flatMap(Optional::stream).map(TimeBasedValue.class::cast); + return elements.flatMap(Optional::stream); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index ce1338158..8f15ba0a8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -36,28 +36,6 @@ public SqlDataSource( this.databaseNamingStrategy = databaseNamingStrategy; } - /** - * This method should be used to validate a given table. - * - * @param tableName name of the table - * @param entityClass class of the entity - * @param validator for validation - * @throws SQLException – if the connection could not be established - */ - protected void validateDBTable(String tableName, Class entityClass, SourceValidator validator) - throws SQLException { - ResultSet rs = connector.getConnection().getMetaData().getColumns(null, null, tableName, null); - - Set columnNames = new HashSet<>(); - - while (rs.next()) { - String name = rs.getString("COLUMN_NAME"); - columnNames.add(StringUtils.snakeCaseToCamelCase(name)); - } - - validator.validate(columnNames, entityClass); - } - /** * Creates a base query string without closing semicolon of the following pattern:
* {@code SELECT * FROM .
} @@ -146,7 +124,7 @@ public Stream> getSourceData( *

(We cannot use {@link java.util.function.Function} here because it throws SQLException). */ @FunctionalInterface - interface AddParams { + protected interface AddParams { /** * Enhance a PreparedStatement by inserting parameters for wildcards * diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index fd2ec84e8..4349351e0 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -56,7 +56,7 @@ public SqlIdCoordinateSource( dataSource.getDbColumnName(factory.getCoordinateField(), coordinateTableName); // validating table - dataSource.validateDBTable(coordinateTableName, Pair.class, factory); + dataSource.connector.validateDBTable(coordinateTableName, Pair.class, factory); // setup queries this.basicQuery = createBaseQueryString(dataSource.schemaName, coordinateTableName); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index 7f952d4e4..8ec86c162 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -40,7 +40,7 @@ public SqlTimeSeriesMappingSource( @Override public Stream> getMappingSourceData(SourceValidator validator) { try { - dataSource.validateDBTable(tableName, TimeSeriesMappingSource.class, validator); + dataSource.connector.validateDBTable(tableName, TimeSeriesMappingSource.class, validator); } catch (SQLException e) { throw new FactoryException(e); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java index 3784d13c8..1100ad3c2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java @@ -65,7 +65,7 @@ public SqlTimeSeriesSource( final String tableName = sqlDataSource.databaseNamingStrategy.getTimeSeriesEntityName(columnScheme); - dataSource.validateDBTable(tableName, valueClass, factory); + dataSource.connector.validateDBTable(tableName, valueClass, factory); String dbTimeColumnName = sqlDataSource.getDbColumnName(factory.getTimeFieldString(), tableName); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index d2aab44e1..4045d128e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -54,11 +54,14 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, - TimeBasedWeatherValueFactory weatherFactory) { + TimeBasedWeatherValueFactory weatherFactory) + throws SQLException { super(idCoordinateSource, weatherFactory); this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); this.dataSource = new SqlDataSource(connector, schemaName, new DatabaseNamingStrategy()); + dataSource.connector.validateDBTable(weatherTableName, WeatherValue.class, weatherFactory); + String dbTimeColumnName = dataSource.getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName); String dbCoordinateIdColumnName = diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index ad189ae73..2a18ea0e5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -60,10 +60,10 @@ class CosmoIdCoordinateFactoryTest extends Specification { Map parameter = [ "tid": "1", "id": "106580", - "latgeo": "39.602772", - "longgeo": "1.279336", - "latrot": "-10", - "longrot": "-6.8125" + "latGeo": "39.602772", + "longGeo": "1.279336", + "latRot": "-10", + "longRot": "-6.8125" ] def validSimpleFactoryData = new SimpleFactoryData(parameter, Pair) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy index f83ec6a29..ac44095d8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy @@ -24,11 +24,11 @@ class CosmoTimeBasedWeatherValueFactoryTest extends Specification { Map parameter = [ "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", "time" : TimeUtil.withDefaults.toString(time), - "diffuseirradiance": "282.671997070312", - "directirradiance" : "286.872985839844", + "diffuseIrradiance": "282.671997070312", + "directIrradiance" : "286.872985839844", "temperature" : "", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "windDirection" : "0", + "windVelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) @@ -57,11 +57,11 @@ class CosmoTimeBasedWeatherValueFactoryTest extends Specification { Map parameter = [ "time" : TimeUtil.withDefaults.toString(time), "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "diffuseirradiance": "282.671997070312", - "directirradiance" : "286.872985839844", + "diffuseIrradiance": "282.671997070312", + "directIrradiance" : "286.872985839844", "temperature" : "278.019012451172", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "windDirection" : "0", + "windVelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index a38901edd..e1ec0a0c0 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -59,7 +59,7 @@ class IconIdCoordinateFactoryTest extends Specification { "id":"477295", "latitude":"52.312", "longitude":"12.812", - "coordinatetype":"ICON"] + "coordinateType":"ICON"] def validSimpleFactoryData = new SimpleFactoryData(parameter, Pair) Pair expectedPair = Pair.of(477295, GeoUtils.buildPoint(52.312, 12.812)) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy index e3169fc9e..5b72b7e3a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactoryTest.groovy @@ -128,7 +128,7 @@ class TimeBasedSimpleValueFactoryTest extends Specification { "time": defaultTimeUtil.toString(time), "p": "500.0", "q": "165.0", - "heatdemand": "8.0" + "heatDemand": "8.0" ], HeatAndSValue) def expected = new TimeBasedValue( UUID.fromString("78ca078a-e6e9-4972-a58d-b2cadbc2df2c"), @@ -148,7 +148,7 @@ class TimeBasedSimpleValueFactoryTest extends Specification { "uuid": "78ca078a-e6e9-4972-a58d-b2cadbc2df2c", "time": defaultTimeUtil.toString(time), "p": "500.0", - "heatdemand": "8.0" + "heatDemand": "8.0" ], HeatAndPValue) def expected = new TimeBasedValue( UUID.fromString("78ca078a-e6e9-4972-a58d-b2cadbc2df2c"), @@ -167,7 +167,7 @@ class TimeBasedSimpleValueFactoryTest extends Specification { def data = new SimpleTimeBasedValueData([ "uuid": "78ca078a-e6e9-4972-a58d-b2cadbc2df2c", "time": defaultTimeUtil.toString(time), - "heatdemand": "8.0" + "heatDemand": "8.0" ], HeatDemandValue) def expected = new TimeBasedValue( UUID.fromString("78ca078a-e6e9-4972-a58d-b2cadbc2df2c"), From 651d9d126d8a4dcb01d1ffd8caa11131a1f709ac Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 13 Nov 2023 15:32:02 +0100 Subject: [PATCH 03/20] fmt --- src/main/java/edu/ie3/datamodel/io/factory/Factory.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index e9e7a93f2..c8b7d4af4 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -109,9 +109,10 @@ private void isSupportedClass(Class desiredClass) { protected abstract List> getFields(Class entityClass); /** - * Method for validating the found fields. The found fields needs to fully contain at least one of the sets returned by {@link #getFields(Class)}. - * If the found fields don't contain all necessary fields, an {@link FactoryException} with a detail message is thrown. - * If the found fields contain more fields than necessary, these fields are ignored. + * Method for validating the found fields. The found fields needs to fully contain at least one of + * the sets returned by {@link #getFields(Class)}. If the found fields don't contain all necessary + * fields, an {@link FactoryException} with a detail message is thrown. If the found fields + * contain more fields than necessary, these fields are ignored. * * @param foundFields that were found * @param entityClass of the build data From fdbe8a8f1f9e0bc4e7e807d39108e7f16c03bce8 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 14 Nov 2023 11:10:53 +0100 Subject: [PATCH 04/20] Some small changes. --- CHANGELOG.md | 3 +++ .../edu/ie3/datamodel/io/factory/Factory.java | 21 +++++++++++++++++-- .../input/AssetInputEntityFactoryTest.groovy | 6 +++--- .../Transformer2WInputFactoryTest.groovy | 4 +--- .../FixedFeedInInputFactoryTest.groovy | 4 ++-- .../FlexOptionsResultFactoryTest.groovy | 4 ++-- .../result/NodeResultFactoryTest.groovy | 4 ++-- .../SystemParticipantResultFactoryTest.groovy | 4 ++-- .../CosmoIdCoordinateFactoryTest.groovy | 2 +- .../IconIdCoordinateFactoryTest.groovy | 2 +- ...stemParticipantTypeInputFactoryTest.groovy | 2 +- 11 files changed, 37 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1a341398..c1ba25466 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Changed +- Enhancing the error message for coordinate sources with invalid column names [#670](https://github.com/ie3-institute/PowerSystemDataModel/issues/670) +- Allowing for additional unused columns in sources [#839](https://github.com/ie3-institute/PowerSystemDataModel/issues/839) +- Improving column name validation to only run once per source [#849](https://github.com/ie3-institute/PowerSystemDataModel/issues/849) ## [4.1.0] - 2023-11-02 diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index c8b7d4af4..6681ad6f1 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -140,12 +140,29 @@ public void validate(Set foundFields, Class entityClass) { throw new FactoryException( "The provided fields " + providedKeysString - + " are invalid for instance of " + + " are invalid for instance of '" + entityClass.getSimpleName() - + ". \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" + + "'. \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" + entityClass.getSimpleName() + "' are possible (NOT case-sensitive!):\n" + possibleOptions); + } else { + // checking for additional fields + Set additionalFields = new HashSet<>(); + Set allFields = + validFieldSets.stream().flatMap(Collection::stream).collect(Collectors.toSet()); + + foundFields.stream().filter(e -> !allFields.contains(e)).forEach(additionalFields::add); + harmonizedFoundFields.stream() + .filter(e -> !allFields.contains(e)) + .forEach(additionalFields::add); + + if (!additionalFields.isEmpty()) { + log.debug( + "The following additional fields were found for instance of '{}': {}", + entityClass.getSimpleName(), + additionalFields); + } } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy index 10395b2c8..11d613bec 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy @@ -280,11 +280,11 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until") when: - Try input = Try.of(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) + Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) then: input.failure - input.exception.get().message == "The provided fields [operates_from, operates_until, uuid] are invalid for instance of TestAssetInput. \n" + + input.exception.get().message == "The provided fields [operates_from, operates_until, uuid] are invalid for instance of 'TestAssetInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + "0: [id, uuid] or [id, uuid]\n" + "1: [id, operates_from, uuid] or [id, operatesFrom, uuid]\n" + @@ -298,7 +298,7 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until", "id", "additional_field") when: - Try input = Try.of(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) + Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) then: input.success diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy index c357481f1..99bd62bd3 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/Transformer2WInputFactoryTest.groovy @@ -6,15 +6,13 @@ package edu.ie3.datamodel.io.factory.input import edu.ie3.datamodel.exceptions.FactoryException -import edu.ie3.datamodel.exceptions.TryException -import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.OperatorInput import edu.ie3.datamodel.models.input.connector.Transformer2WInput import edu.ie3.datamodel.models.input.connector.type.Transformer2WTypeInput import edu.ie3.datamodel.utils.Try +import edu.ie3.test.common.GridTestData import edu.ie3.test.helper.FactoryTestHelper import spock.lang.Specification -import edu.ie3.test.common.GridTestData import java.time.ZonedDateTime diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy index e7a46b0c8..3532e2554 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy @@ -79,11 +79,11 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe def foundFields = inputFactory.newSet("uuid", "id", "s_rated", "cosphi_rated") when: - Try input = Try.of(() -> inputFactory.validate(foundFields, FixedFeedInInput), FactoryException) + Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, FixedFeedInInput), FactoryException) then: input.failure - input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of FixedFeedInInput. \n" + + input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of 'FixedFeedInInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + "0: [cos_phi_rated, id, q_characteristics, s_rated, uuid] or [cosPhiRated, id, qCharacteristics, sRated, uuid]\n" + "1: [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid]\n" + diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy index b9bfa7709..bab574a5e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy @@ -56,11 +56,11 @@ class FlexOptionsResultFactoryTest extends Specification implements FactoryTestH def foundFields = resultFactory.newSet("time", "input_model", "p_ref", "p_min") when: - Try input = Try.of(() -> resultFactory.validate(foundFields, FlexOptionsResult), FactoryException) + Try input = Try.ofVoid(() -> resultFactory.validate(foundFields, FlexOptionsResult), FactoryException) then: input.failure - input.exception.get().message == "The provided fields [input_model, p_min, p_ref, time] are invalid for instance of FlexOptionsResult. \n" + + input.exception.get().message == "The provided fields [input_model, p_min, p_ref, time] are invalid for instance of 'FlexOptionsResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + "0: [input_model, p_max, p_min, p_ref, time] or [inputModel, pMax, pMin, pRef, time]\n" + "1: [input_model, p_max, p_min, p_ref, time, uuid] or [inputModel, pMax, pMin, pRef, time, uuid]\n" diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy index 2e6f50a85..aea5c3be8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy @@ -54,11 +54,11 @@ class NodeResultFactoryTest extends Specification implements FactoryTestHelper { def foundFields = resultFactory.newSet("time", "input_model", "v_mag") when: - Try input = Try.of(() -> resultFactory.validate(foundFields, NodeResult), FactoryException) + Try input = Try.ofVoid(() -> resultFactory.validate(foundFields, NodeResult), FactoryException) then: input.failure - input.exception.get().message == "The provided fields [input_model, time, v_mag] are invalid for instance of NodeResult. \n" + + input.exception.get().message == "The provided fields [input_model, time, v_mag] are invalid for instance of 'NodeResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + "0: [input_model, time, v_ang, v_mag] or [inputModel, time, vAng, vMag]\n" + "1: [input_model, time, uuid, v_ang, v_mag] or [inputModel, time, uuid, vAng, vMag]\n" diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy index 947a5f5c5..fbc741f50 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy @@ -130,11 +130,11 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor def foundFields = resultFactory.newSet("time", "input_model", "q") when: - Try result = Try.of(() -> resultFactory.validate(foundFields, WecResult), FactoryException) + Try result = Try.ofVoid(() -> resultFactory.validate(foundFields, WecResult), FactoryException) then: result.failure - result.exception.get().message == "The provided fields [input_model, q, time] are invalid for instance of WecResult. \n" + + result.exception.get().message == "The provided fields [input_model, q, time] are invalid for instance of 'WecResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + "0: [input_model, p, q, time] or [inputModel, p, q, time]\n" + "1: [input_model, p, q, time, uuid] or [inputModel, p, q, time, uuid]\n" diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index 2a18ea0e5..00115184e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -50,7 +50,7 @@ class CosmoIdCoordinateFactoryTest extends Specification { then: actual.failure - actual.exception.get().message == "The provided fields [id, latrot, longrot, tid] are invalid for instance of Pair. \n" + + actual.exception.get().message == "The provided fields [id, latrot, longrot, tid] are invalid for instance of 'Pair'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + "0: [id, lat_geo, lat_rot, long_geo, long_rot, tid] or [id, latGeo, latRot, longGeo, longRot, tid]\n" } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index e1ec0a0c0..be32388b0 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -48,7 +48,7 @@ class IconIdCoordinateFactoryTest extends Specification { then: actual.failure - actual.exception.get().message == "The provided fields [coordinatetype, id, latitude] are invalid for instance of Pair. \n" + + actual.exception.get().message == "The provided fields [coordinatetype, id, latitude] are invalid for instance of 'Pair'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + "0: [coordinate_type, id, latitude, longitude] or [coordinateType, id, latitude, longitude]\n" } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy index 7e44fc75c..1f5856a47 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy @@ -287,7 +287,7 @@ class SystemParticipantTypeInputFactoryTest extends Specification implements Fac then: input.failure - input.exception.get().message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] are invalid for instance of StorageTypeInput. \n" + + input.exception.get().message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] are invalid for instance of 'StorageTypeInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + "0: [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid] or [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid]\n" } From ad379c29de46a671409c5974e0929c1d357b5bd1 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 24 Nov 2023 10:54:21 +0100 Subject: [PATCH 05/20] Implementing requested changes. --- .../io/connectors/CouchbaseConnector.java | 5 +- .../io/connectors/CsvFileConnector.java | 59 ++++++++++++++ .../edu/ie3/datamodel/io/factory/Factory.java | 57 ++++++++----- .../CosmoTimeBasedWeatherValueFactory.java | 3 +- .../IconTimeBasedWeatherValueFactory.java | 9 +-- .../ie3/datamodel/io/source/DataSource.java | 3 +- .../ie3/datamodel/io/source/EntitySource.java | 18 +++-- .../datamodel/io/source/GraphicSource.java | 21 +++-- .../datamodel/io/source/RawGridSource.java | 24 ++++-- .../io/source/ResultEntitySource.java | 30 ++++++- .../datamodel/io/source/SourceValidator.java | 4 +- .../io/source/SystemParticipantSource.java | 24 +++++- .../datamodel/io/source/ThermalSource.java | 19 +++-- .../io/source/TimeSeriesMappingSource.java | 6 +- .../ie3/datamodel/io/source/TypeSource.java | 18 +++++ .../io/source/csv/CsvDataSource.java | 20 ++--- .../csv/CsvJointGridContainerSource.java | 11 +++ .../csv/CsvTimeSeriesMappingSource.java | 8 +- .../io/source/csv/CsvTimeSeriesSource.java | 12 +-- .../io/source/sql/SqlDataSource.java | 4 +- .../sql/SqlTimeSeriesMappingSource.java | 15 ++-- .../datamodel/io/factory/FactoryTest.groovy | 80 +++++++++++++++++++ .../input/AssetInputEntityFactoryTest.groovy | 18 +---- .../FixedFeedInInputFactoryTest.groovy | 8 +- .../FlexOptionsResultFactoryTest.groovy | 4 +- .../result/NodeResultFactoryTest.groovy | 4 +- .../SystemParticipantResultFactoryTest.groovy | 6 +- .../CosmoIdCoordinateFactoryTest.groovy | 2 +- .../IconIdCoordinateFactoryTest.groovy | 2 +- ...stemParticipantTypeInputFactoryTest.groovy | 2 +- .../io/source/EntitySourceTest.groovy | 5 ++ .../source/csv/CsvTimeSeriesSourceIT.groovy | 4 +- 32 files changed, 365 insertions(+), 140 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java index 51f4dd377..0686c640e 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java @@ -14,8 +14,8 @@ import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.kv.MutationResult; import com.couchbase.client.java.query.QueryResult; -import java.time.Duration; import edu.ie3.datamodel.io.source.SourceValidator; +import edu.ie3.datamodel.models.value.WeatherValue; import java.time.Duration; import java.util.HashSet; import java.util.List; @@ -68,7 +68,8 @@ public CouchbaseConnector( * @param validator for validation */ @SuppressWarnings("unchecked") - public final void validateDb(Class entityClass, SourceValidator validator) { + public final void validateDb( + Class entityClass, SourceValidator validator) { String query = "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM " + bucketName diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 71147a7db..6fc303fbd 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -6,16 +6,22 @@ package edu.ie3.datamodel.io.connectors; import edu.ie3.datamodel.exceptions.ConnectorException; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.IoUtil; import edu.ie3.datamodel.io.csv.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; +import edu.ie3.datamodel.utils.ExceptionUtils; +import edu.ie3.datamodel.utils.Try; +import edu.ie3.util.StringUtils; import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -51,6 +57,59 @@ public CsvFileConnector(Path baseDirectoryName, FileNamingStrategy fileNamingStr this.fileNamingStrategy = fileNamingStrategy; } + @SuppressWarnings("unchecked") + public void validate(Map, SourceValidator> pairs, String csvSep) { + List exceptions = new ArrayList<>(); + + pairs.forEach( + ((clz, validator) -> { + Class c = (Class) clz; + SourceValidator v = (SourceValidator) validator; + + Try.of(() -> getHeadlineFields(c, csvSep), SourceException.class) + .getData() + .flatMap( + headline -> + Try.ofVoid(() -> v.validate(headline, c), FactoryException.class) + .getException()) + .ifPresent(exceptions::add); + })); + + if (!exceptions.isEmpty()) { + throw new FactoryException( + "The following exception(s) occurred during validation: " + + ExceptionUtils.getMessages(exceptions)); + } + } + + /** + * Method for extracting the headline fields of a csv file. + * + * @param entityClass that should be buildable from the source data + * @param csvSep separator for headline + * @return a set of headline fields + * @throws SourceException if the file could not be read + */ + public Set getHeadlineFields(Class entityClass, String csvSep) + throws SourceException { + try (BufferedReader reader = initReader(entityClass)) { + // extracting headline fields + return Arrays.stream(reader.readLine().split(csvSep + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1)) + .map( + maybeStartEndQuotedString -> + StringUtils.unquoteStartEnd(maybeStartEndQuotedString.trim()) + .replaceAll("\"{2}", "\"") + .trim()) + .collect(Collectors.toSet()); + } catch (ConnectorException | IOException e) { + throw new SourceException( + "Unable to find file for entity '" + + entityClass.getSimpleName() + + "': " + + e.getMessage()); + } + } + public synchronized BufferedCsvWriter getOrInitWriter( Class clz, String[] headerElements, String csvSep) throws ConnectorException { diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 6681ad6f1..61cbb7978 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -24,7 +24,7 @@ * @param Type of the intended return type (might differ slightly from target class (cf. {@link * edu.ie3.datamodel.io.factory.timeseries.TimeBasedValueFactory})). */ -public abstract class Factory implements SourceValidator { +public abstract class Factory implements SourceValidator { public static final Logger log = LoggerFactory.getLogger(Factory.class); private final List> supportedClasses; @@ -52,7 +52,7 @@ public Try get(D data) { try { // build the model return Success.of(buildModel(data)); - } catch (Exception e) { + } catch (FactoryException | IllegalArgumentException e) { return Failure.of( new FactoryException( "An error occurred when creating instance of " @@ -104,36 +104,57 @@ private void isSupportedClass(Class desiredClass) { * Returns list of sets of attribute names that the entity requires to be built. At least one of * these sets needs to be delivered for entity creation to be successful. * + * @param entityClass class that can be used to specify the fields that are returned * @return list of possible attribute sets */ protected abstract List> getFields(Class entityClass); + /** + * Method to find and return additional fields that were found in a source. This method will + * return the additional fields that were found in the valid set with the least additional fields. + * + * @param actualFields found in the source + * @param validFieldSets that contains at least all fields found in the source + * @return a set of additional fields + */ + protected Set getAdditionalFields( + Set actualFields, List> validFieldSets) { + // checking for additional fields + // and returning the set with the least additional fields + return validFieldSets.stream() + .map( + s -> { + Set set = new HashSet<>(actualFields); + set.removeAll(s); + return set; + }) + .min(Comparator.comparing(Collection::size)) + .orElse(Collections.emptySet()); + } + /** * Method for validating the found fields. The found fields needs to fully contain at least one of * the sets returned by {@link #getFields(Class)}. If the found fields don't contain all necessary * fields, an {@link FactoryException} with a detail message is thrown. If the found fields * contain more fields than necessary, these fields are ignored. * - * @param foundFields that were found + * @param actualFields that were found * @param entityClass of the build data */ - public void validate(Set foundFields, Class entityClass) { - List> fieldSets = - getFields(entityClass).stream().map(Factory::toSnakeCase).toList(); - Set harmonizedFoundFields = toSnakeCase(foundFields); + public void validate(Set actualFields, Class entityClass) { + List> fieldSets = getFields(entityClass); + Set harmonizedFoundFields = toCamelCase(actualFields); // comparing the found fields to a list of possible fields (allows additional fields) // if not all fields were found in a set, this set is filtered out // all other fields are saved as a list // allows snake, camel and mixed cases List> validFieldSets = - fieldSets.stream() - .filter(s -> foundFields.containsAll(s) || harmonizedFoundFields.containsAll(s)) - .toList(); + fieldSets.stream().filter(harmonizedFoundFields::containsAll).toList(); if (validFieldSets.isEmpty()) { // build the exception string with extensive debug information - String providedKeysString = "[" + String.join(", ", foundFields) + "]"; + String providedKeysString = "[" + String.join(", ", actualFields) + "]"; String possibleOptions = getFieldsString(fieldSets).toString(); @@ -147,19 +168,11 @@ public void validate(Set foundFields, Class entityClass) { + "' are possible (NOT case-sensitive!):\n" + possibleOptions); } else { - // checking for additional fields - Set additionalFields = new HashSet<>(); - Set allFields = - validFieldSets.stream().flatMap(Collection::stream).collect(Collectors.toSet()); - - foundFields.stream().filter(e -> !allFields.contains(e)).forEach(additionalFields::add); - harmonizedFoundFields.stream() - .filter(e -> !allFields.contains(e)) - .forEach(additionalFields::add); + Set additionalFields = getAdditionalFields(harmonizedFoundFields, validFieldSets); if (!additionalFields.isEmpty()) { log.debug( - "The following additional fields were found for instance of '{}': {}", + "The following additional fields were found for entity class of '{}': {}", entityClass.getSimpleName(), additionalFields); } @@ -175,7 +188,7 @@ protected static StringBuilder getFieldsString(List> fieldSets) { + ": [" + String.join(", ", fieldSet) + "] or [" - + String.join(", ", toCamelCase(fieldSet)) + + String.join(", ", toSnakeCase(fieldSet)) + "]\n"; possibleOptions.append(option); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 21c61a9ec..f8d267187 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -12,6 +12,7 @@ import edu.ie3.util.quantities.PowerSystemUnits; import edu.ie3.util.quantities.interfaces.Irradiance; import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import java.util.Set; import javax.measure.quantity.Angle; @@ -58,7 +59,7 @@ protected List> getFields(Class entityClass) { TEMPERATURE, WIND_DIRECTION, WIND_VELOCITY); - return List.of(minConstructorParams, toLowerCase(minConstructorParams)); + return Collections.singletonList(minConstructorParams); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index e1b095116..6b3fd419e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -62,7 +62,7 @@ protected List> getFields(Class entityClass) { minParameters, "albrad", "asobs", - "aswdifus", + "aswdifuS", "tG", "u10m", "u20m", @@ -87,12 +87,7 @@ protected List> getFields(Class entityClass) { Set allParametersWithUuid = expandSet(allParameters, UUID); return Arrays.asList( - minParameters, - allParameters, - minParametersWithUuid, - allParametersWithUuid, - toLowerCase(minParameters), - toLowerCase(allParameters)); + minParameters, allParameters, minParametersWithUuid, allParametersWithUuid); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java index f52d9b55e..0687e9e50 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java @@ -13,6 +13,5 @@ public interface DataSource { /** Creates a stream of maps that represent the rows in the database */ - Stream> getSourceData( - Class entityClass, SourceValidator validateSource); + Stream> getSourceData(Class entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index 3c7a880d5..91be9395e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -39,6 +39,9 @@ public abstract class EntitySource { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + /** Returns a map: class to {@link SourceValidator}. */ + public abstract Map, SourceValidator> getValidationMapping(); + protected String buildSkippingMessage( String entityDesc, String entityUuid, String entityId, String missingElementsString) { return "Skipping " @@ -249,9 +252,9 @@ protected Stream> nodeAssetInputE * @return stream of the entity data wrapped in a {@link Try} */ protected Stream assetInputEntityDataStream( - Class entityClass, SourceValidator validator, Collection operators) { + Class entityClass, Collection operators) { return dataSource - .getSourceData(entityClass, validator) + .getSourceData(entityClass) .map( fieldsToAttributes -> assetInputEntityDataStream(entityClass, fieldsToAttributes, operators)); @@ -289,9 +292,9 @@ protected AssetInputEntityData assetInputEntityDataStream * @return stream of {@link SimpleEntityData} */ protected Stream simpleEntityDataStream( - Class entityClass, SourceValidator validator) { + Class entityClass) { return dataSource - .getSourceData(entityClass, validator) + .getSourceData(entityClass) .map(fieldsToAttributes -> new SimpleEntityData(fieldsToAttributes, entityClass)); } @@ -313,8 +316,7 @@ protected Stream> nodeAssetEntit EntityFactory factory, Collection nodes, Collection operators) { - return nodeAssetInputEntityDataStream( - assetInputEntityDataStream(entityClass, factory, operators), nodes) + return nodeAssetInputEntityDataStream(assetInputEntityDataStream(entityClass, operators), nodes) .map(factory::get); } @@ -333,7 +335,7 @@ public Set> buildAssetInputEntit Class entityClass, EntityFactory factory, Collection operators) { - return assetInputEntityDataStream(entityClass, factory, operators) + return assetInputEntityDataStream(entityClass, operators) .map(factory::get) .collect(Collectors.toSet()); } @@ -342,7 +344,7 @@ public Set> buildAssetInputEntit public Set> buildEntities( Class entityClass, EntityFactory factory) { return dataSource - .getSourceData(entityClass, factory) + .getSourceData(entityClass) .map( fieldsToAttributes -> { SimpleEntityData data = new SimpleEntityData(fieldsToAttributes, entityClass); diff --git a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java index ae97091de..5767e7f03 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.exceptions.GraphicSourceException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.graphics.LineGraphicInputEntityData; @@ -50,6 +52,13 @@ public GraphicSource(TypeSource typeSource, RawGridSource rawGridSource, DataSou this.nodeGraphicInputFactory = new NodeGraphicInputFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(NodeGraphicInput.class, nodeGraphicInputFactory), + entry(LineGraphicInput.class, lineGraphicInputFactory)); + } + /** Returns the graphic elements of the grid or throws a {@link SourceException} */ public GraphicElements getGraphicElements() throws SourceException { @@ -91,7 +100,7 @@ public Set getNodeGraphicInput() throws SourceException { public Set getNodeGraphicInput(Set nodes) throws SourceException { return Try.scanCollection( - buildNodeGraphicEntityData(nodes, nodeGraphicInputFactory) + buildNodeGraphicEntityData(nodes) .map(nodeGraphicInputFactory::get) .collect(Collectors.toSet()), NodeGraphicInput.class) @@ -113,7 +122,7 @@ public Set getLineGraphicInput() throws SourceException { public Set getLineGraphicInput(Set lines) throws SourceException { return Try.scanCollection( - buildLineGraphicEntityData(lines, lineGraphicInputFactory) + buildLineGraphicEntityData(lines) .map(lineGraphicInputFactory::get) .collect(Collectors.toSet()), LineGraphicInput.class) @@ -140,9 +149,9 @@ public Set getLineGraphicInput(Set lines) throws So * @return a stream of tries of {@link NodeGraphicInput} entities */ protected Stream> buildNodeGraphicEntityData( - Set nodes, SourceValidator validator) { + Set nodes) { return dataSource - .getSourceData(NodeGraphicInput.class, validator) + .getSourceData(NodeGraphicInput.class) .map(fieldsToAttributes -> buildNodeGraphicEntityData(fieldsToAttributes, nodes)); } @@ -187,9 +196,9 @@ protected Try buildNodeGraphicEntit * @return a stream of tries of {@link LineGraphicInput} entities */ protected Stream> buildLineGraphicEntityData( - Set lines, SourceValidator validator) { + Set lines) { return dataSource - .getSourceData(LineGraphicInput.class, validator) + .getSourceData(LineGraphicInput.class) .map(fieldsToAttributes -> buildLineGraphicEntityData(fieldsToAttributes, lines)); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java index d5f0e3cbd..2ca424231 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.RawGridException; import edu.ie3.datamodel.exceptions.SourceException; @@ -69,6 +71,16 @@ public RawGridSource(TypeSource typeSource, DataSource dataSource) { this.measurementUnitInputFactory = new MeasurementUnitInputFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(NodeInput.class, nodeInputFactory), + entry(LineInput.class, lineInputFactory), + entry(Transformer2WInput.class, transformer2WInputFactory), + entry(SwitchInput.class, switchInputFactory), + entry(MeasurementUnitInput.class, measurementUnitInputFactory)); + } + /** * Should return either a consistent instance of {@link RawGridElements} or throw a {@link * SourceException}. The decision to throw a {@link SourceException} instead of returning the @@ -166,7 +178,7 @@ public Set getNodes() throws SourceException { */ public Set getNodes(Set operators) throws SourceException { return Try.scanCollection( - assetInputEntityDataStream(NodeInput.class, nodeInputFactory, operators) + assetInputEntityDataStream(NodeInput.class, operators) .map(nodeInputFactory::get) .collect(Collectors.toSet()), NodeInput.class) @@ -414,7 +426,7 @@ public Set> buildNodeInputEntiti Class entityClass, EntityFactory factory, Collection operators) { - return assetInputEntityDataStream(entityClass, factory, operators) + return assetInputEntityDataStream(entityClass, operators) .map(factory::get) .collect(Collectors.toSet()); } @@ -441,9 +453,7 @@ public Set> buildTransformer3WEntities return buildTransformer3WEntityData( buildTypedConnectorEntityData( buildUntypedConnectorInputEntityData( - assetInputEntityDataStream( - Transformer3WInput.class, transformer3WInputFactory, operators), - nodes), + assetInputEntityDataStream(Transformer3WInput.class, operators), nodes), transformer3WTypeInputs), nodes) .map(transformer3WInputFactory::get) @@ -562,7 +572,7 @@ Stream> typedEntityStream( Collection types) { return buildTypedConnectorEntityData( buildUntypedConnectorInputEntityData( - assetInputEntityDataStream(entityClass, factory, operators), nodes), + assetInputEntityDataStream(entityClass, operators), nodes), types) .map(factory::get); } @@ -574,7 +584,7 @@ Stream> untypedConnectorInputEntityStream( Set nodes, Set operators) { return buildUntypedConnectorInputEntityData( - assetInputEntityDataStream(entityClass, factory, operators), nodes) + assetInputEntityDataStream(entityClass, operators), nodes) .map(factory::get); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java index b4cc69334..0378f0ab4 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.io.factory.SimpleEntityFactory; import edu.ie3.datamodel.io.factory.result.*; import edu.ie3.datamodel.models.result.NodeResult; @@ -16,6 +18,7 @@ import edu.ie3.datamodel.models.result.system.*; import edu.ie3.datamodel.models.result.thermal.CylindricalStorageResult; import edu.ie3.datamodel.models.result.thermal.ThermalHouseResult; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -60,6 +63,31 @@ public ResultEntitySource(DataSource dataSource, String dtfPattern) { this.flexOptionsResultFactory = new FlexOptionsResultFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(LoadResult.class, systemParticipantResultFactory), + entry(FixedFeedInResult.class, systemParticipantResultFactory), + entry(BmResult.class, systemParticipantResultFactory), + entry(PvResult.class, systemParticipantResultFactory), + entry(ChpResult.class, systemParticipantResultFactory), + entry(WecResult.class, systemParticipantResultFactory), + entry(StorageResult.class, systemParticipantResultFactory), + entry(EvcsResult.class, systemParticipantResultFactory), + entry(EvResult.class, systemParticipantResultFactory), + entry(HpResult.class, systemParticipantResultFactory), + entry(EmResult.class, systemParticipantResultFactory), + entry(ThermalHouseResult.class, thermalResultFactory), + entry(CylindricalStorageResult.class, thermalResultFactory), + entry(SwitchResult.class, switchResultFactory), + entry(NodeResult.class, nodeResultFactory), + entry(SystemParticipantResult.class, connectorResultFactory), + entry(LineResult.class, connectorResultFactory), + entry(Transformer2WResult.class, connectorResultFactory), + entry(Transformer3WResult.class, connectorResultFactory), + entry(FlexOptionsResult.class, flexOptionsResultFactory)); + } + /** * Returns a unique set of {@link NodeResult} instances. * @@ -318,7 +346,7 @@ public Set getEmResults() { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Set getResultEntities( Class entityClass, SimpleEntityFactory factory) { - return simpleEntityDataStream(entityClass, factory) + return simpleEntityDataStream(entityClass) .map( entityData -> factory diff --git a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java index f3f8ee9ef..39aeb1f98 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java @@ -7,7 +7,7 @@ import java.util.Set; -public interface SourceValidator { +public interface SourceValidator { /** * Method for validating a data source. @@ -15,5 +15,5 @@ public interface SourceValidator { * @param foundFields fields that were found in the source data * @param entityClass that should be buildable from the source data */ - void validate(Set foundFields, Class entityClass); + void validate(Set foundFields, Class entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index 19295f7a1..a0bf8d2c6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.SystemParticipantsException; @@ -77,6 +79,22 @@ public SystemParticipantSource( this.emInputFactory = new EmInputFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(BmInput.class, bmInputFactory), + entry(ChpTypeInput.class, chpInputFactory), + entry(EvInput.class, evcsInputFactory), + entry(FixedFeedInInput.class, fixedFeedInInputFactory), + entry(HpInput.class, hpInputFactory), + entry(LoadInput.class, loadInputFactory), + entry(PvInput.class, pvInputFactory), + entry(StorageInput.class, storageInputFactory), + entry(WecInput.class, wecInputFactory), + entry(EvcsInput.class, evcsInputFactory), + entry(EmInput.class, emInputFactory)); + } + /** * Should return either a consistent instance of {@link SystemParticipants} or throw a {@link * SourceException}. The decision to throw a {@link SourceException} instead of returning the @@ -693,7 +711,7 @@ Stream> typedSystemParticipantEntityStream( Collection types) { return buildTypedSystemParticipantEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(entityClass, factory, operators), nodes), + assetInputEntityDataStream(entityClass, operators), nodes), types) .map(factory::get); } @@ -708,7 +726,7 @@ private Stream> chpInputStream( return buildChpEntityData( buildTypedEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(ChpInput.class, factory, operators), nodes), + assetInputEntityDataStream(ChpInput.class, operators), nodes), types), thermalStorages, thermalBuses) @@ -724,7 +742,7 @@ private Stream> hpInputStream( return buildHpEntityData( buildTypedEntityData( nodeAssetInputEntityDataStream( - assetInputEntityDataStream(HpInput.class, factory, operators), nodes), + assetInputEntityDataStream(HpInput.class, operators), nodes), types), thermalBuses) .map(factory::get); diff --git a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java index 60da88bc1..af41fbb2b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FailureException; import edu.ie3.datamodel.exceptions.SourceException; @@ -46,6 +48,14 @@ public ThermalSource(TypeSource typeSource, DataSource dataSource) { this.thermalHouseInputFactory = new ThermalHouseInputFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(ThermalBusInput.class, thermalBusInputFactory), + entry(CylindricalStorageInput.class, cylindricalStorageInputFactory), + entry(ThermalHouseInput.class, thermalHouseInputFactory)); + } + /** * Returns a unique set of {@link ThermalBusInput} instances. * @@ -262,7 +272,7 @@ public Try, FailureException> buildThermalHouseInputEntit Set thermalBuses = getThermalBuses(); return Try.scanCollection( - assetInputEntityDataStream(ThermalHouseInput.class, factory, typeSource.getOperators()) + assetInputEntityDataStream(ThermalHouseInput.class, typeSource.getOperators()) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -276,7 +286,7 @@ public Try, FailureException> buildThermalHouseInputEntit Collection operators, Collection thermalBuses) { return Try.scanCollection( - assetInputEntityDataStream(ThermalHouseInput.class, factory, operators) + assetInputEntityDataStream(ThermalHouseInput.class, operators) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -290,8 +300,7 @@ public Try, FailureException> buildCylindricalStora Set thermalBuses = getThermalBuses(); return Try.scanCollection( - assetInputEntityDataStream( - CylindricalStorageInput.class, factory, typeSource.getOperators()) + assetInputEntityDataStream(CylindricalStorageInput.class, typeSource.getOperators()) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) @@ -304,7 +313,7 @@ public Set> buildCylindricalStora CylindricalStorageInputFactory factory, Collection operators, Collection thermalBuses) { - return assetInputEntityDataStream(CylindricalStorageInput.class, factory, operators) + return assetInputEntityDataStream(CylindricalStorageInput.class, operators) .flatMap( assetInputEntityData -> buildThermalUnitInputEntityData(assetInputEntityData, thermalBuses) diff --git a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java index 03377241f..0f85a4084 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java @@ -24,7 +24,7 @@ */ public abstract class TimeSeriesMappingSource { - private final TimeSeriesMappingFactory mappingFactory; + protected final TimeSeriesMappingFactory mappingFactory; protected TimeSeriesMappingSource() { this.mappingFactory = new TimeSeriesMappingFactory(); @@ -36,7 +36,7 @@ protected TimeSeriesMappingSource() { * @return That mapping */ public Map getMapping() { - return getMappingSourceData(mappingFactory) + return getMappingSourceData() .map(this::createMappingEntry) .filter(Try::isSuccess) .map(t -> (Success) t) @@ -59,7 +59,7 @@ public Optional getTimeSeriesUuid(UUID modelIdentifier) { * * @return Stream of maps */ - public abstract Stream> getMappingSourceData(SourceValidator validator); + public abstract Stream> getMappingSourceData(); // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java index a7412cedc..f7bea01c8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import static java.util.Map.entry; + import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.OperatorInputFactory; import edu.ie3.datamodel.io.factory.typeinput.LineTypeInputFactory; @@ -17,6 +19,7 @@ import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput; import edu.ie3.datamodel.models.input.system.type.*; import edu.ie3.datamodel.utils.Try; +import java.util.Map; import java.util.Set; /** @@ -45,6 +48,21 @@ public TypeSource(DataSource dataSource) { this.systemParticipantTypeInputFactory = new SystemParticipantTypeInputFactory(); } + @Override + public Map, SourceValidator> getValidationMapping() { + return Map.ofEntries( + entry(OperatorInput.class, operatorInputFactory), + entry(LineTypeInput.class, lineTypeInputFactory), + entry(Transformer2WTypeInput.class, transformer2WTypeInputFactory), + entry(Transformer3WTypeInput.class, transformer3WTypeInputFactory), + entry(BmTypeInput.class, systemParticipantTypeInputFactory), + entry(ChpTypeInput.class, systemParticipantTypeInputFactory), + entry(EvTypeInput.class, systemParticipantTypeInputFactory), + entry(HpTypeInput.class, systemParticipantTypeInputFactory), + entry(StorageTypeInput.class, systemParticipantTypeInputFactory), + entry(WecTypeInput.class, systemParticipantTypeInputFactory)); + } + /** * Returns a set of {@link Transformer2WTypeInput} instances. This set has to be unique in the * sense of object uniqueness but also in the sense of {@link java.util.UUID} uniqueness of the diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 75fea3036..9a355e746 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -10,7 +10,6 @@ import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.utils.validation.ValidationUtils; import edu.ie3.util.StringUtils; @@ -60,9 +59,8 @@ public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNami } @Override - public Stream> getSourceData( - Class entityClass, SourceValidator validator) { - return buildStreamWithFieldsToAttributesMap(entityClass, connector, validator); + public Stream> getSourceData(Class entityClass) { + return buildStreamWithFieldsToAttributesMap(entityClass, connector); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -268,12 +266,9 @@ protected Predicate> isPresentCollectIfNot( * mapping (fieldName to fieldValue) */ protected Stream> buildStreamWithFieldsToAttributesMap( - Class entityClass, - CsvFileConnector connector, - SourceValidator validator) { + Class entityClass, CsvFileConnector connector) { try { - return buildStreamWithFieldsToAttributesMap( - entityClass, connector.initReader(entityClass), validator); + return buildStreamWithFieldsToAttributesMap(entityClass, connector.initReader(entityClass)); } catch (FileNotFoundException | ConnectorException e) { log.warn( "Unable to find file for entity '{}': {}", entityClass.getSimpleName(), e.getMessage()); @@ -292,15 +287,10 @@ protected Stream> buildStreamWithFieldsToAttributesMap( * mapping (fieldName to fieldValue) */ protected Stream> buildStreamWithFieldsToAttributesMap( - Class entityClass, - BufferedReader bufferedReader, - SourceValidator validator) { + Class entityClass, BufferedReader bufferedReader) { try (BufferedReader reader = bufferedReader) { final String[] headline = parseCsvRow(reader.readLine(), csvSep); - // validating read file - validator.validate(Set.of(headline), entityClass); - // sanity check for headline if (!Arrays.asList(headline).contains("uuid")) { throw new SourceException( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java index 2fa9e0f2a..5420b85f8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java @@ -18,7 +18,9 @@ import edu.ie3.datamodel.models.input.container.SystemParticipants; import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** Convenience class for cases where all used data comes from CSV sources */ public class CsvJointGridContainerSource { @@ -52,6 +54,15 @@ public static JointGridContainer read( new SystemParticipantSource(typeSource, thermalSource, rawGridSource, dataSource); GraphicSource graphicSource = new GraphicSource(typeSource, rawGridSource, dataSource); + /* validating sources */ + Map, SourceValidator> validationMapping = + new HashMap<>(typeSource.getValidationMapping()); + validationMapping.putAll(rawGridSource.getValidationMapping()); + validationMapping.putAll(thermalSource.getValidationMapping()); + validationMapping.putAll(systemParticipantSource.getValidationMapping()); + validationMapping.putAll(graphicSource.getValidationMapping()); + dataSource.connector.validate(validationMapping, csvSep); + /* Loading models */ Try rawGridElements = Try.of(rawGridSource::getGridData, SourceException.class); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index 8e9996f8d..12c4182c8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -6,7 +6,6 @@ package edu.ie3.datamodel.io.source.csv; import edu.ie3.datamodel.io.naming.FileNamingStrategy; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.nio.file.Path; import java.util.Map; @@ -19,11 +18,14 @@ public class CsvTimeSeriesMappingSource extends TimeSeriesMappingSource { public CsvTimeSeriesMappingSource( String csvSep, Path gridFolderPath, FileNamingStrategy fileNamingStrategy) { this.dataSource = new CsvDataSource(csvSep, gridFolderPath, fileNamingStrategy); + + // validating + dataSource.connector.validate(Map.of(MappingEntry.class, mappingFactory), dataSource.csvSep); } @Override - public Stream> getMappingSourceData(SourceValidator validator) { + public Stream> getMappingSourceData() { return dataSource.buildStreamWithFieldsToAttributesMap( - MappingEntry.class, dataSource.connector, validator); + MappingEntry.class, dataSource.connector); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java index 1ad3d5e5d..4b8d55604 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java @@ -12,7 +12,6 @@ import edu.ie3.datamodel.io.factory.timeseries.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesSource; -import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.*; @@ -100,11 +99,7 @@ public CsvTimeSeriesSource( /* Read in the full time series */ try { this.timeSeries = - buildIndividualTimeSeries( - timeSeriesUuid, - filePath, - this::createTimeBasedValue, - (Class) valueClass); + buildIndividualTimeSeries(timeSeriesUuid, filePath, this::createTimeBasedValue); } catch (SourceException e) { throw new IllegalArgumentException( "Unable to obtain time series with UUID '" @@ -145,14 +140,13 @@ public Optional getValue(ZonedDateTime time) { protected IndividualTimeSeries buildIndividualTimeSeries( UUID timeSeriesUuid, Path filePath, - Function, Try, FactoryException>> fieldToValueFunction, - Class entityClass) + Function, Try, FactoryException>> fieldToValueFunction) throws SourceException { try (BufferedReader reader = dataSource.connector.initReader(filePath)) { Try>, FailureException> timeBasedValues = Try.scanStream( dataSource - .buildStreamWithFieldsToAttributesMap(entityClass, reader, valueFactory) + .buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader) .map(fieldToValueFunction), "TimeBasedValue"); return new IndividualTimeSeries<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 8f15ba0a8..f6841c3e2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -9,7 +9,6 @@ import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.util.StringUtils; import java.sql.PreparedStatement; @@ -110,8 +109,7 @@ protected List getDbTables(String schemaPattern, String tableNamePattern } @Override - public Stream> getSourceData( - Class entityClass, SourceValidator validator) { + public Stream> getSourceData(Class entityClass) { String explicitTableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); return buildStreamByTableName(explicitTableName); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index 8ec86c162..fa084c3d8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -7,11 +7,9 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.sql.SQLException; import java.util.Map; @@ -26,7 +24,8 @@ public class SqlTimeSeriesMappingSource extends TimeSeriesMappingSource { public SqlTimeSeriesMappingSource( SqlConnector connector, String schemaName, - EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) { + EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) + throws SQLException { this.dataSource = new SqlDataSource( connector, schemaName, new DatabaseNamingStrategy(entityPersistenceNamingStrategy)); @@ -35,16 +34,12 @@ public SqlTimeSeriesMappingSource( this.tableName = entityPersistenceNamingStrategy.getEntityName(MappingEntry.class).orElseThrow(); this.queryFull = createBaseQueryString(schemaName, tableName); + + dataSource.connector.validateDBTable(tableName, TimeSeriesMappingSource.class, mappingFactory); } @Override - public Stream> getMappingSourceData(SourceValidator validator) { - try { - dataSource.connector.validateDBTable(tableName, TimeSeriesMappingSource.class, validator); - } catch (SQLException e) { - throw new FactoryException(e); - } - + public Stream> getMappingSourceData() { return dataSource.executeQuery(queryFull); } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy new file mode 100644 index 000000000..e7dc42212 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy @@ -0,0 +1,80 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory + +import edu.ie3.datamodel.exceptions.FactoryException +import edu.ie3.datamodel.utils.Try +import spock.lang.Shared +import spock.lang.Specification + +class FactoryTest extends Specification { + @Shared + private DummyFactory factory = new DummyFactory(String) + + + def "A Factory can return additional fields correctly"() { + when: + def additional = factory.getAdditionalFields(actualFields as Set, validFieldSets) + + then: + additional == expected as Set + + where: + actualFields | validFieldSets | expected + [ + "uuid", + "id", + "time", + "value_1" + ] | [ + ["uuid", "value_1"] as Set, + ["id", "time", "value_1"] as Set + ] | ["uuid"] + [ + "uuid", + "id", + "time", + "value_1", + "value_2", + "value_3" + ] | [ + ["uuid", "value_1", "value_3"] as Set, + ["id", "time", "value_1"] as Set + ] | ["id", "time", "value_2"] + } + + def "A Factory should allow additional fields"() { + given: + def foundFields = factory.newSet("uuid", "id", "time", "value_1", "value_2", "value_3") + + when: + Try input = Try.ofVoid(() -> factory.validate(foundFields, String), FactoryException) + + then: + input.success + } + + + private class DummyFactory extends Factory { + + protected DummyFactory(Class... supportedClasses) { + super(supportedClasses) + } + + @Override + protected String buildModel(SimpleFactoryData data) { + return null + } + + @Override + protected List> getFields(Class entityClass) { + return [ + ["uuid", "value1", "value3"] as Set, + ["id", "time", "value1"] as Set + ] + } + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy index 11d613bec..8243a5b3d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy @@ -287,21 +287,9 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe input.exception.get().message == "The provided fields [operates_from, operates_until, uuid] are invalid for instance of 'TestAssetInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + "0: [id, uuid] or [id, uuid]\n" + - "1: [id, operates_from, uuid] or [id, operatesFrom, uuid]\n" + - "2: [id, operates_until, uuid] or [id, operatesUntil, uuid]\n" + - "3: [id, operates_from, operates_until, uuid] or [id, operatesFrom, operatesUntil, uuid]\n" - } - - def "An AssetInputFactory should allow additional fields"() { - given: - def inputFactory = new TestAssetInputFactory() - def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until", "id", "additional_field") - - when: - Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) - - then: - input.success + "1: [id, operatesFrom, uuid] or [id, operates_from, uuid]\n" + + "2: [id, operatesUntil, uuid] or [id, operates_until, uuid]\n" + + "3: [id, operatesFrom, operatesUntil, uuid] or [id, operates_from, operates_until, uuid]\n" } private static class TestAssetInput extends AssetInput { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy index 3532e2554..1091238cb 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy @@ -85,9 +85,9 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe input.failure input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of 'FixedFeedInInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + - "0: [cos_phi_rated, id, q_characteristics, s_rated, uuid] or [cosPhiRated, id, qCharacteristics, sRated, uuid]\n" + - "1: [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid]\n" + - "2: [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid]\n" + - "3: [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid] or [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid]\n" + "0: [cosPhiRated, id, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, q_characteristics, s_rated, uuid]\n" + + "1: [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid]\n" + + "2: [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid]\n" + + "3: [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy index bab574a5e..6831a7746 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy @@ -62,7 +62,7 @@ class FlexOptionsResultFactoryTest extends Specification implements FactoryTestH input.failure input.exception.get().message == "The provided fields [input_model, p_min, p_ref, time] are invalid for instance of 'FlexOptionsResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + - "0: [input_model, p_max, p_min, p_ref, time] or [inputModel, pMax, pMin, pRef, time]\n" + - "1: [input_model, p_max, p_min, p_ref, time, uuid] or [inputModel, pMax, pMin, pRef, time, uuid]\n" + "0: [inputModel, pMax, pMin, pRef, time] or [input_model, p_max, p_min, p_ref, time]\n" + + "1: [inputModel, pMax, pMin, pRef, time, uuid] or [input_model, p_max, p_min, p_ref, time, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy index aea5c3be8..7767998d6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy @@ -60,7 +60,7 @@ class NodeResultFactoryTest extends Specification implements FactoryTestHelper { input.failure input.exception.get().message == "The provided fields [input_model, time, v_mag] are invalid for instance of 'NodeResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + - "0: [input_model, time, v_ang, v_mag] or [inputModel, time, vAng, vMag]\n" + - "1: [input_model, time, uuid, v_ang, v_mag] or [inputModel, time, uuid, vAng, vMag]\n" + "0: [inputModel, time, vAng, vMag] or [input_model, time, v_ang, v_mag]\n" + + "1: [inputModel, time, uuid, vAng, vMag] or [input_model, time, uuid, v_ang, v_mag]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy index efa30db11..5c4983db3 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy @@ -136,8 +136,8 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor result.failure result.exception.get().message == "The provided fields [input_model, q, time] are invalid for instance of 'WecResult'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + - "0: [input_model, p, q, time] or [inputModel, p, q, time]\n" + - "1: [input_model, p, q, time, uuid] or [inputModel, p, q, time, uuid]\n" + "0: [inputModel, p, q, time] or [input_model, p, q, time]\n" + + "1: [inputModel, p, q, time, uuid] or [input_model, p, q, time, uuid]\n" } def "A SystemParticipantResultFactory should be performant"() { @@ -156,7 +156,7 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor resultFactory.get(new SimpleEntityData(parameter, StorageResult)) } BigDecimal elapsedTime = (System - .currentTimeMillis() - startTime) / 1000.0 + .currentTimeMillis() - startTime) / 1000.0 elapsedTime < 3 } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index 00115184e..fc7534a04 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -52,7 +52,7 @@ class CosmoIdCoordinateFactoryTest extends Specification { actual.failure actual.exception.get().message == "The provided fields [id, latrot, longrot, tid] are invalid for instance of 'Pair'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + - "0: [id, lat_geo, lat_rot, long_geo, long_rot, tid] or [id, latGeo, latRot, longGeo, longRot, tid]\n" + "0: [id, latGeo, latRot, longGeo, longRot, tid] or [id, lat_geo, lat_rot, long_geo, long_rot, tid]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index be32388b0..fbc664d34 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -50,7 +50,7 @@ class IconIdCoordinateFactoryTest extends Specification { actual.failure actual.exception.get().message == "The provided fields [coordinatetype, id, latitude] are invalid for instance of 'Pair'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + - "0: [coordinate_type, id, latitude, longitude] or [coordinateType, id, latitude, longitude]\n" + "0: [coordinateType, id, latitude, longitude] or [coordinate_type, id, latitude, longitude]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy index 1f5856a47..8f6340c8c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy @@ -289,6 +289,6 @@ class SystemParticipantTypeInputFactoryTest extends Specification implements Fac input.failure input.exception.get().message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] are invalid for instance of 'StorageTypeInput'. \n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + - "0: [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid] or [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid]\n" + "0: [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid] or [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy index 4b51be15b..32f4fd439 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy @@ -24,6 +24,11 @@ class EntitySourceTest extends Specification { DummyEntitySource(CsvDataSource dataSource) { this.dataSource = dataSource } + + @Override + Map, SourceValidator> getValidationMapping() { + return null + } } @Shared diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy index 3dbf7940c..de1857954 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceIT.groovy @@ -35,7 +35,7 @@ class CsvTimeSeriesSourceIT extends Specification implements CsvTestDataMeta { def filePath = Path.of("file/not/found.csv") when: - source.buildIndividualTimeSeries(UUID.fromString("fbc59b5b-9307-4fb4-a406-c1f08f26fee5"), filePath, { null }, null) + source.buildIndividualTimeSeries(UUID.fromString("fbc59b5b-9307-4fb4-a406-c1f08f26fee5"), filePath, { null }) then: def ex = thrown(SourceException) @@ -49,7 +49,7 @@ class CsvTimeSeriesSourceIT extends Specification implements CsvTestDataMeta { def tsUuid = UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7") when: - def actual = source.buildIndividualTimeSeries(tsUuid, filePath, { source.createTimeBasedValue(it) }, HeatAndPValue) + def actual = source.buildIndividualTimeSeries(tsUuid, filePath, { source.createTimeBasedValue(it) }) then: noExceptionThrown() From f37d7b6fa8cd9f32be8e677962ff26ba3e2a01bd Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 5 Dec 2023 13:55:00 +0100 Subject: [PATCH 06/20] Implementing requested changes. --- .../io/connectors/CouchbaseConnector.java | 14 +--- .../io/connectors/CsvFileConnector.java | 59 ---------------- .../io/connectors/InfluxDbConnector.java | 12 +--- .../datamodel/io/connectors/SqlConnector.java | 23 ------- .../edu/ie3/datamodel/io/factory/Factory.java | 25 ++++--- .../ie3/datamodel/io/source/DataSource.java | 9 +++ .../ie3/datamodel/io/source/EntitySource.java | 21 +++++- .../datamodel/io/source/GraphicSource.java | 14 ++-- .../io/source/IdCoordinateSource.java | 9 +++ .../datamodel/io/source/RawGridSource.java | 31 ++++----- .../io/source/ResultEntitySource.java | 68 ++++++++++++------- .../datamodel/io/source/SourceValidator.java | 6 +- .../io/source/SystemParticipantSource.java | 31 +++++---- .../datamodel/io/source/ThermalSource.java | 15 ++-- .../io/source/TimeSeriesMappingSource.java | 9 +-- .../ie3/datamodel/io/source/TypeSource.java | 49 +++++++++---- .../datamodel/io/source/WeatherSource.java | 9 +++ .../couchbase/CouchbaseWeatherSource.java | 15 +++- .../io/source/csv/CsvDataSource.java | 10 +++ .../io/source/csv/CsvIdCoordinateSource.java | 10 +++ .../csv/CsvJointGridContainerSource.java | 24 ++++--- .../csv/CsvTimeSeriesMappingSource.java | 12 +++- .../io/source/csv/CsvWeatherSource.java | 8 +++ .../influxdb/InfluxDbWeatherSource.java | 12 +++- .../io/source/sql/SqlDataSource.java | 45 ++++++++++++ .../io/source/sql/SqlIdCoordinateSource.java | 15 ++-- .../sql/SqlTimeSeriesMappingSource.java | 12 +++- .../io/source/sql/SqlTimeSeriesSource.java | 11 ++- .../io/source/sql/SqlWeatherSource.java | 12 +++- .../ie3/datamodel/utils/ExceptionUtils.java | 26 +++++++ .../java/edu/ie3/datamodel/utils/Try.java | 46 +++++++++++++ .../input/AssetInputEntityFactoryTest.groovy | 12 ++-- .../FixedFeedInInputFactoryTest.groovy | 12 ++-- .../FlexOptionsResultFactoryTest.groovy | 8 +-- .../result/NodeResultFactoryTest.groovy | 8 +-- .../SystemParticipantResultFactoryTest.groovy | 10 +-- .../CosmoIdCoordinateFactoryTest.groovy | 6 +- .../IconIdCoordinateFactoryTest.groovy | 6 +- ...stemParticipantTypeInputFactoryTest.groovy | 6 +- .../io/source/EntitySourceTest.groovy | 5 +- .../io/source/IdCoordinateSourceMock.groovy | 6 ++ .../ie3/test/common/WeatherTestData.groovy | 6 ++ 42 files changed, 474 insertions(+), 273 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java index 0686c640e..42f7a8f82 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java @@ -14,8 +14,6 @@ import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.kv.MutationResult; import com.couchbase.client.java.query.QueryResult; -import edu.ie3.datamodel.io.source.SourceValidator; -import edu.ie3.datamodel.models.value.WeatherValue; import java.time.Duration; import java.util.HashSet; import java.util.List; @@ -61,15 +59,9 @@ public CouchbaseConnector( cluster = Cluster.connect(url, clusterOptions); } - /** - * This method should be used to validate a given couchbaseDb. - * - * @param entityClass class of the entity - * @param validator for validation - */ + /** Returns a set of found fields. */ @SuppressWarnings("unchecked") - public final void validateDb( - Class entityClass, SourceValidator validator) { + public Set getSourceFields(Class entityClass) { String query = "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM " + bucketName @@ -86,7 +78,7 @@ public final void validateDb( set.addAll((List) columns); } - validator.validate(set, entityClass); + return set; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 6fc303fbd..71147a7db 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -6,22 +6,16 @@ package edu.ie3.datamodel.io.connectors; import edu.ie3.datamodel.exceptions.ConnectorException; -import edu.ie3.datamodel.exceptions.FactoryException; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.IoUtil; import edu.ie3.datamodel.io.csv.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; -import edu.ie3.datamodel.utils.ExceptionUtils; -import edu.ie3.datamodel.utils.Try; -import edu.ie3.util.StringUtils; import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -57,59 +51,6 @@ public CsvFileConnector(Path baseDirectoryName, FileNamingStrategy fileNamingStr this.fileNamingStrategy = fileNamingStrategy; } - @SuppressWarnings("unchecked") - public void validate(Map, SourceValidator> pairs, String csvSep) { - List exceptions = new ArrayList<>(); - - pairs.forEach( - ((clz, validator) -> { - Class c = (Class) clz; - SourceValidator v = (SourceValidator) validator; - - Try.of(() -> getHeadlineFields(c, csvSep), SourceException.class) - .getData() - .flatMap( - headline -> - Try.ofVoid(() -> v.validate(headline, c), FactoryException.class) - .getException()) - .ifPresent(exceptions::add); - })); - - if (!exceptions.isEmpty()) { - throw new FactoryException( - "The following exception(s) occurred during validation: " - + ExceptionUtils.getMessages(exceptions)); - } - } - - /** - * Method for extracting the headline fields of a csv file. - * - * @param entityClass that should be buildable from the source data - * @param csvSep separator for headline - * @return a set of headline fields - * @throws SourceException if the file could not be read - */ - public Set getHeadlineFields(Class entityClass, String csvSep) - throws SourceException { - try (BufferedReader reader = initReader(entityClass)) { - // extracting headline fields - return Arrays.stream(reader.readLine().split(csvSep + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1)) - .map( - maybeStartEndQuotedString -> - StringUtils.unquoteStartEnd(maybeStartEndQuotedString.trim()) - .replaceAll("\"{2}", "\"") - .trim()) - .collect(Collectors.toSet()); - } catch (ConnectorException | IOException e) { - throw new SourceException( - "Unable to find file for entity '" - + entityClass.getSimpleName() - + "': " - + e.getMessage()); - } - } - public synchronized BufferedCsvWriter getOrInitWriter( Class clz, String[] headerElements, String csvSep) throws ConnectorException { diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java index bd1b700cc..86805779f 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.connectors; -import edu.ie3.datamodel.io.source.SourceValidator; import java.util.*; import java.util.function.BinaryOperator; import java.util.stream.Collectors; @@ -110,13 +109,8 @@ public InfluxDbConnector(String url, String databaseName) { this(url, databaseName, NO_SCENARIO, true, InfluxDB.LogLevel.NONE, BatchOptions.DEFAULTS); } - /** - * This method should be used to validate a given {@link InfluxDB}. - * - * @param entityClass class of the entity - * @param validator for validation - */ - public final void validateDb(Class entityClass, SourceValidator validator) { + /** Returns the fields found in the source. */ + public Set getSourceFields(Class entityClass) { QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName)); Map>> tagResults = parseQueryResult(tagKeys); @@ -127,7 +121,7 @@ public final void validateDb(Class entityClass, SourceValidator validator) { tagResults.values().forEach(v -> v.stream().map(m -> m.get("tagKey")).forEach(set::add)); fieldResults.values().forEach(v -> v.stream().map(m -> m.get("fieldKey")).forEach(set::add)); - validator.validate(set, entityClass); + return set; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index d88b4894e..8b13ec0d0 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.connectors; -import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.util.StringUtils; import edu.ie3.util.TimeUtil; import java.sql.*; @@ -42,28 +41,6 @@ public SqlConnector(String jdbcUrl, String userName, String password) { connectionProps.put("password", password); } - /** - * This method should be used to validate a given sql table. - * - * @param tableName name of the table - * @param entityClass class of the entity - * @param validator for validation - * @throws SQLException – if the connection could not be established - */ - public void validateDBTable(String tableName, Class entityClass, SourceValidator validator) - throws SQLException { - ResultSet rs = getConnection().getMetaData().getColumns(null, null, tableName, null); - - Set columnNames = new HashSet<>(); - - while (rs.next()) { - String name = rs.getString("COLUMN_NAME"); - columnNames.add(StringUtils.snakeCaseToCamelCase(name)); - } - - validator.validate(columnNames, entityClass); - } - /** * Executes the given query. For update queries please use {@link * SqlConnector#executeUpdate(String)}. diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 61cbb7978..e3c4fc67e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -140,8 +140,10 @@ protected Set getAdditionalFields( * * @param actualFields that were found * @param entityClass of the build data + * @return either an exception wrapped by a {@link Failure} or an empty success */ - public void validate(Set actualFields, Class entityClass) { + public Try validate( + Set actualFields, Class entityClass) { List> fieldSets = getFields(entityClass); Set harmonizedFoundFields = toCamelCase(actualFields); @@ -158,15 +160,16 @@ public void validate(Set actualFields, Class entityClass) { String possibleOptions = getFieldsString(fieldSets).toString(); - throw new FactoryException( - "The provided fields " - + providedKeysString - + " are invalid for instance of '" - + entityClass.getSimpleName() - + "'. \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" - + entityClass.getSimpleName() - + "' are possible (NOT case-sensitive!):\n" - + possibleOptions); + return Failure.of( + new FactoryException( + "The provided fields " + + providedKeysString + + " are invalid for instance of '" + + entityClass.getSimpleName() + + "'. \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '" + + entityClass.getSimpleName() + + "' are possible (NOT case-sensitive!):\n" + + possibleOptions)); } else { Set additionalFields = getAdditionalFields(harmonizedFoundFields, validFieldSets); @@ -176,6 +179,8 @@ public void validate(Set actualFields, Class entityClass) { entityClass.getSimpleName(), additionalFields); } + + return Success.empty(); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java index 0687e9e50..d41a4b731 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.models.UniqueEntity; import java.util.*; import java.util.stream.Stream; @@ -12,6 +13,14 @@ /** Interface that include functionalities for data sources in the database table, csv file etc. */ public interface DataSource { + /** + * Method to retrieve the fields found in the source. + * + * @param entityClass class of the source + * @return the found fields + */ + Set getSourceFields(Class entityClass) throws SourceException; + /** Creates a stream of maps that represent the rows in the database */ Stream> getSourceData(Class entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index 2f2c9335d..5e0a4ae2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -39,8 +39,25 @@ public abstract class EntitySource { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= - /** Returns a map: class to {@link SourceValidator}. */ - public abstract Map, SourceValidator> getValidationMapping(); + public abstract void validate(); + + /** + * Method for validating a single source. + * + * @param entityClass class to be validated + * @param validator used to validate + * @return either a {@link FactoryException} wrapped by a failure or an empty success + * @param type of the class + */ + protected final Try validate( + Class entityClass, SourceValidator validator) { + try { + Set actualFields = dataSource.getSourceFields(entityClass); + return validator.validate(actualFields, entityClass); + } catch (SourceException e) { + return Failure.of(new FactoryException(e.getCause())); + } + } protected String buildSkippingMessage( String entityDesc, String entityUuid, String entityId, String missingElementsString) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java index 5767e7f03..8799cc5a4 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java @@ -5,8 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - +import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.GraphicSourceException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.graphics.LineGraphicInputEntityData; @@ -53,10 +52,13 @@ public GraphicSource(TypeSource typeSource, RawGridSource rawGridSource, DataSou } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(NodeGraphicInput.class, nodeGraphicInputFactory), - entry(LineGraphicInput.class, lineGraphicInputFactory)); + public void validate() { + List exceptions = + Try.getExceptions( + validate(NodeGraphicInput.class, nodeGraphicInputFactory), + validate(LineGraphicInput.class, lineGraphicInputFactory)); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** Returns the graphic elements of the grid or throws a {@link SourceException} */ diff --git a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java index a41232c53..d0374c2f6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.util.geo.CoordinateDistance; import edu.ie3.util.geo.GeoUtils; import java.util.*; @@ -19,6 +20,14 @@ */ public interface IdCoordinateSource { + /** + * Method to retrieve the fields found in the source. + * + * @param entityClass class of the source + * @return the found fields + */ + Set getSourceFields(Class entityClass) throws SourceException; + /** * Get the matching coordinate for the given ID * diff --git a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java index 2ca424231..cc75ce44e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java @@ -5,30 +5,21 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.RawGridException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.EntityFactory; import edu.ie3.datamodel.io.factory.input.*; import edu.ie3.datamodel.models.input.*; -import edu.ie3.datamodel.models.input.MeasurementUnitInput; -import edu.ie3.datamodel.models.input.NodeInput; -import edu.ie3.datamodel.models.input.OperatorInput; import edu.ie3.datamodel.models.input.connector.*; -import edu.ie3.datamodel.models.input.connector.LineInput; -import edu.ie3.datamodel.models.input.connector.SwitchInput; -import edu.ie3.datamodel.models.input.connector.Transformer2WInput; -import edu.ie3.datamodel.models.input.connector.Transformer3WInput; import edu.ie3.datamodel.models.input.connector.type.LineTypeInput; import edu.ie3.datamodel.models.input.connector.type.Transformer2WTypeInput; import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput; import edu.ie3.datamodel.models.input.container.RawGridElements; import edu.ie3.datamodel.utils.Try; -import edu.ie3.datamodel.utils.Try.*; +import edu.ie3.datamodel.utils.Try.Failure; +import edu.ie3.datamodel.utils.Try.Success; import java.util.*; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -72,13 +63,17 @@ public RawGridSource(TypeSource typeSource, DataSource dataSource) { } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(NodeInput.class, nodeInputFactory), - entry(LineInput.class, lineInputFactory), - entry(Transformer2WInput.class, transformer2WInputFactory), - entry(SwitchInput.class, switchInputFactory), - entry(MeasurementUnitInput.class, measurementUnitInputFactory)); + public void validate() { + List exceptions = + Try.getExceptions( + validate(NodeInput.class, nodeInputFactory), + validate(LineInput.class, lineInputFactory), + validate(Transformer2WInput.class, transformer2WInputFactory), + validate(Transformer3WInput.class, transformer3WInputFactory), + validate(SwitchInput.class, switchInputFactory), + validate(MeasurementUnitInput.class, measurementUnitInputFactory)); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java index 0378f0ab4..a29d3af84 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java @@ -5,8 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - +import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.factory.SimpleEntityFactory; import edu.ie3.datamodel.io.factory.result.*; import edu.ie3.datamodel.models.result.NodeResult; @@ -18,10 +17,12 @@ import edu.ie3.datamodel.models.result.system.*; import edu.ie3.datamodel.models.result.thermal.CylindricalStorageResult; import edu.ie3.datamodel.models.result.thermal.ThermalHouseResult; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link ResultEntity} container @@ -64,28 +65,45 @@ public ResultEntitySource(DataSource dataSource, String dtfPattern) { } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(LoadResult.class, systemParticipantResultFactory), - entry(FixedFeedInResult.class, systemParticipantResultFactory), - entry(BmResult.class, systemParticipantResultFactory), - entry(PvResult.class, systemParticipantResultFactory), - entry(ChpResult.class, systemParticipantResultFactory), - entry(WecResult.class, systemParticipantResultFactory), - entry(StorageResult.class, systemParticipantResultFactory), - entry(EvcsResult.class, systemParticipantResultFactory), - entry(EvResult.class, systemParticipantResultFactory), - entry(HpResult.class, systemParticipantResultFactory), - entry(EmResult.class, systemParticipantResultFactory), - entry(ThermalHouseResult.class, thermalResultFactory), - entry(CylindricalStorageResult.class, thermalResultFactory), - entry(SwitchResult.class, switchResultFactory), - entry(NodeResult.class, nodeResultFactory), - entry(SystemParticipantResult.class, connectorResultFactory), - entry(LineResult.class, connectorResultFactory), - entry(Transformer2WResult.class, connectorResultFactory), - entry(Transformer3WResult.class, connectorResultFactory), - entry(FlexOptionsResult.class, flexOptionsResultFactory)); + public void validate() { + List exceptions = + new ArrayList<>( + Stream.of( + LoadResult.class, + FixedFeedInResult.class, + BmResult.class, + PvResult.class, + ChpResult.class, + WecResult.class, + StorageResult.class, + EvcsResult.class, + EvResult.class, + HpResult.class, + EmResult.class) + .map(c -> validate(c, systemParticipantResultFactory).getException()) + .flatMap(Optional::stream) + .toList()); + + validate(ThermalHouseResult.class, thermalResultFactory) + .getException() + .ifPresent(exceptions::add); + validate(CylindricalStorageResult.class, thermalResultFactory) + .getException() + .ifPresent(exceptions::add); + validate(SwitchResult.class, switchResultFactory).getException().ifPresent(exceptions::add); + validate(NodeResult.class, nodeResultFactory).getException().ifPresent(exceptions::add); + validate(LineResult.class, connectorResultFactory).getException().ifPresent(exceptions::add); + validate(Transformer2WResult.class, connectorResultFactory) + .getException() + .ifPresent(exceptions::add); + validate(Transformer3WResult.class, connectorResultFactory) + .getException() + .ifPresent(exceptions::add); + validate(FlexOptionsResult.class, flexOptionsResultFactory) + .getException() + .ifPresent(exceptions::add); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java index 39aeb1f98..4c932c7c5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java @@ -5,6 +5,9 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.utils.Try; +import edu.ie3.datamodel.utils.Try.Failure; import java.util.Set; public interface SourceValidator { @@ -14,6 +17,7 @@ public interface SourceValidator { * * @param foundFields fields that were found in the source data * @param entityClass that should be buildable from the source data + * @return either an exception wrapped by a {@link Failure} or an empty success */ - void validate(Set foundFields, Class entityClass); + Try validate(Set foundFields, Class entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index a0bf8d2c6..a5d473370 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.SystemParticipantsException; @@ -80,19 +78,22 @@ public SystemParticipantSource( } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(BmInput.class, bmInputFactory), - entry(ChpTypeInput.class, chpInputFactory), - entry(EvInput.class, evcsInputFactory), - entry(FixedFeedInInput.class, fixedFeedInInputFactory), - entry(HpInput.class, hpInputFactory), - entry(LoadInput.class, loadInputFactory), - entry(PvInput.class, pvInputFactory), - entry(StorageInput.class, storageInputFactory), - entry(WecInput.class, wecInputFactory), - entry(EvcsInput.class, evcsInputFactory), - entry(EmInput.class, emInputFactory)); + public void validate() { + List exceptions = + Try.getExceptions( + validate(BmInput.class, bmInputFactory), + validate(ChpInput.class, chpInputFactory), + validate(EvInput.class, evInputFactory), + validate(FixedFeedInInput.class, fixedFeedInInputFactory), + validate(HpInput.class, hpInputFactory), + validate(LoadInput.class, loadInputFactory), + validate(PvInput.class, pvInputFactory), + validate(StorageInput.class, storageInputFactory), + validate(WecInput.class, wecInputFactory), + validate(EvcsInput.class, evcsInputFactory), + validate(EmInput.class, emInputFactory)); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java index af41fbb2b..93bd9754d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FailureException; import edu.ie3.datamodel.exceptions.SourceException; @@ -49,11 +47,14 @@ public ThermalSource(TypeSource typeSource, DataSource dataSource) { } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(ThermalBusInput.class, thermalBusInputFactory), - entry(CylindricalStorageInput.class, cylindricalStorageInputFactory), - entry(ThermalHouseInput.class, thermalHouseInputFactory)); + public void validate() { + List exceptions = + Try.getExceptions( + validate(ThermalBusInput.class, thermalBusInputFactory), + validate(CylindricalStorageInput.class, cylindricalStorageInputFactory), + validate(ThermalHouseInput.class, thermalHouseInputFactory)); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java index 0f85a4084..72f213d00 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java @@ -6,15 +6,13 @@ package edu.ie3.datamodel.io.source; import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.SimpleEntityData; import edu.ie3.datamodel.io.factory.timeseries.TimeSeriesMappingFactory; import edu.ie3.datamodel.models.input.InputEntity; import edu.ie3.datamodel.utils.Try; import edu.ie3.datamodel.utils.Try.*; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -61,6 +59,9 @@ public Optional getTimeSeriesUuid(UUID modelIdentifier) { */ public abstract Stream> getMappingSourceData(); + /** Returns the fields found in the source */ + public abstract Set getSourceFields() throws SourceException; + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Try createMappingEntry( diff --git a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java index f7bea01c8..bbde7017b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java @@ -5,8 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import static java.util.Map.entry; - +import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.OperatorInputFactory; import edu.ie3.datamodel.io.factory.typeinput.LineTypeInputFactory; @@ -19,8 +18,11 @@ import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput; import edu.ie3.datamodel.models.input.system.type.*; import edu.ie3.datamodel.utils.Try; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link @@ -49,18 +51,35 @@ public TypeSource(DataSource dataSource) { } @Override - public Map, SourceValidator> getValidationMapping() { - return Map.ofEntries( - entry(OperatorInput.class, operatorInputFactory), - entry(LineTypeInput.class, lineTypeInputFactory), - entry(Transformer2WTypeInput.class, transformer2WTypeInputFactory), - entry(Transformer3WTypeInput.class, transformer3WTypeInputFactory), - entry(BmTypeInput.class, systemParticipantTypeInputFactory), - entry(ChpTypeInput.class, systemParticipantTypeInputFactory), - entry(EvTypeInput.class, systemParticipantTypeInputFactory), - entry(HpTypeInput.class, systemParticipantTypeInputFactory), - entry(StorageTypeInput.class, systemParticipantTypeInputFactory), - entry(WecTypeInput.class, systemParticipantTypeInputFactory)); + public void validate() { + validate(OperatorInput.class, operatorInputFactory) + .ifFailure( + e -> + log.warn( + "The source for '{}' could not be validated due to: {}", + OperatorInput.class, + e)); + + List exceptions = + new ArrayList<>( + Try.getExceptions( + validate(LineTypeInput.class, lineTypeInputFactory), + validate(Transformer2WTypeInput.class, transformer2WTypeInputFactory), + validate(Transformer3WTypeInput.class, transformer3WTypeInputFactory))); + + exceptions.addAll( + Stream.of( + EvTypeInput.class, + HpTypeInput.class, + BmTypeInput.class, + WecTypeInput.class, + ChpTypeInput.class, + StorageTypeInput.class) + .map(c -> validate(c, systemParticipantTypeInputFactory).getException()) + .flatMap(Optional::stream) + .toList()); + + exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 31f5150f0..a1b4b29f0 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -40,6 +40,15 @@ protected WeatherSource( this.weatherFactory = weatherFactory; } + /** + * Method to retrieve the fields found in the source. + * + * @param entityClass class of the source + * @return the found fields + */ + public abstract Set getSourceFields(Class entityClass) + throws SourceException; + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index e84aef130..f1a50d0fc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -10,6 +10,7 @@ import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.query.QueryResult; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CouchbaseConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -59,7 +60,8 @@ public CouchbaseWeatherSource( IdCoordinateSource coordinateSource, String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) { + String timeStampPattern) + throws SourceException { this( connector, coordinateSource, @@ -88,14 +90,21 @@ public CouchbaseWeatherSource( String coordinateIdColumnName, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) { + String timeStampPattern) + throws SourceException { super(idCoordinateSource, weatherFactory); this.connector = connector; this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.timeStampPattern = timeStampPattern; - connector.validateDb(WeatherValue.class, weatherFactory); + weatherFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); + } + + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + return connector.getSourceFields(entityClass); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 9a355e746..baf343e69 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -58,6 +58,16 @@ public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNami this.connector = new CsvFileConnector(folderPath, fileNamingStrategy); } + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + try (BufferedReader reader = connector.initReader(entityClass)) { + return Arrays.stream(parseCsvRow(reader.readLine(), csvSep)).collect(Collectors.toSet()); + } catch (ConnectorException | IOException e) { + throw new SourceException("The following exception was thrown while reading a source: ", e); + } + } + @Override public Stream> getSourceData(Class entityClass) { return buildStreamWithFieldsToAttributesMap(entityClass, connector); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java index 81d40e0bb..1cbf60f20 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java @@ -86,6 +86,16 @@ private Map invert(Map map) { return inv; } + @Override + public Set getSourceFields(Class entityClass) throws SourceException { + try (BufferedReader reader = dataSource.connector.initIdCoordinateReader()) { + return Arrays.stream(dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep)) + .collect(Collectors.toSet()); + } catch (IOException e) { + throw new SourceException("The following exception was thrown while reading a source: ", e); + } + } + @Override public Optional getCoordinate(int id) { return Optional.ofNullable(idToCoordinate.get(id)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java index 5420b85f8..e4e46238d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.csv; +import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FileException; import edu.ie3.datamodel.exceptions.InvalidGridException; import edu.ie3.datamodel.exceptions.SourceException; @@ -16,11 +17,10 @@ import edu.ie3.datamodel.models.input.container.JointGridContainer; import edu.ie3.datamodel.models.input.container.RawGridElements; import edu.ie3.datamodel.models.input.container.SystemParticipants; +import edu.ie3.datamodel.utils.ExceptionUtils; import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** Convenience class for cases where all used data comes from CSV sources */ public class CsvJointGridContainerSource { @@ -55,13 +55,17 @@ public static JointGridContainer read( GraphicSource graphicSource = new GraphicSource(typeSource, rawGridSource, dataSource); /* validating sources */ - Map, SourceValidator> validationMapping = - new HashMap<>(typeSource.getValidationMapping()); - validationMapping.putAll(rawGridSource.getValidationMapping()); - validationMapping.putAll(thermalSource.getValidationMapping()); - validationMapping.putAll(systemParticipantSource.getValidationMapping()); - validationMapping.putAll(graphicSource.getValidationMapping()); - dataSource.connector.validate(validationMapping, csvSep); + List validationException = + Try.getExceptions( + Try.ofVoid(typeSource::validate, FactoryException.class), + Try.ofVoid(rawGridSource::validate, FactoryException.class), + Try.ofVoid(systemParticipantSource::validate, FactoryException.class), + Try.ofVoid(graphicSource::validate, FactoryException.class)); + + if (!validationException.isEmpty()) { + throw new FactoryException( + "Validation failed due to: " + ExceptionUtils.getFullMessages(validationException)); + } /* Loading models */ Try rawGridElements = diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index 12c4182c8..b06ae4231 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -5,10 +5,12 @@ */ package edu.ie3.datamodel.io.source.csv; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.nio.file.Path; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; public class CsvTimeSeriesMappingSource extends TimeSeriesMappingSource { @@ -16,11 +18,12 @@ public class CsvTimeSeriesMappingSource extends TimeSeriesMappingSource { private final CsvDataSource dataSource; public CsvTimeSeriesMappingSource( - String csvSep, Path gridFolderPath, FileNamingStrategy fileNamingStrategy) { + String csvSep, Path gridFolderPath, FileNamingStrategy fileNamingStrategy) + throws SourceException { this.dataSource = new CsvDataSource(csvSep, gridFolderPath, fileNamingStrategy); // validating - dataSource.connector.validate(Map.of(MappingEntry.class, mappingFactory), dataSource.csvSep); + mappingFactory.validate(getSourceFields(), MappingEntry.class); } @Override @@ -28,4 +31,9 @@ public Stream> getMappingSourceData() { return dataSource.buildStreamWithFieldsToAttributesMap( MappingEntry.class, dataSource.connector); } + + @Override + public Set getSourceFields() throws SourceException { + return dataSource.getSourceFields(MappingEntry.class); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 6ece1f5c6..b7f45fa1b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.csv; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -60,6 +61,13 @@ public CsvWeatherSource( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + /** Returns an empty set for now. */ + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + return Collections.emptySet(); + } + @Override public Map> getWeather( ClosedInterval timeInterval) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 78a317c4b..ec3a3d3e2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -47,11 +48,18 @@ public class InfluxDbWeatherSource extends WeatherSource { public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource idCoordinateSource, - TimeBasedWeatherValueFactory weatherValueFactory) { + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { super(idCoordinateSource, weatherValueFactory); this.connector = connector; - connector.validateDb(WeatherValue.class, weatherValueFactory); + weatherValueFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); + } + + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + return connector.getSourceFields(entityClass); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index f6841c3e2..7322c0fc6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source.sql; import edu.ie3.datamodel.exceptions.InvalidColumnNameException; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; @@ -108,6 +109,50 @@ protected List getDbTables(String schemaPattern, String tableNamePattern return tableNames; } + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + try { + String tableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); + ResultSet rs = + connector.getConnection().getMetaData().getColumns(null, null, tableName, null); + Set columnNames = new HashSet<>(); + + while (rs.next()) { + String name = rs.getString("COLUMN_NAME"); + columnNames.add(StringUtils.snakeCaseToCamelCase(name)); + } + + return columnNames; + } catch (NoSuchElementException | SQLException e) { + throw new SourceException("The following exception was thrown while reading a source: ", e); + } + } + + /** + * Method that uses the table name to retrieve all field names. + * + * @param tableName to be used + * @return a set of found fields + * @throws SourceException - if an error occurred + */ + public Set getSourceFields(String tableName) throws SourceException { + try { + ResultSet rs = + connector.getConnection().getMetaData().getColumns(null, null, tableName, null); + Set columnNames = new HashSet<>(); + + while (rs.next()) { + String name = rs.getString("COLUMN_NAME"); + columnNames.add(StringUtils.snakeCaseToCamelCase(name)); + } + + return columnNames; + } catch (NoSuchElementException | SQLException e) { + throw new SourceException("The following exception was thrown while reading a source: ", e); + } + } + @Override public Stream> getSourceData(Class entityClass) { String explicitTableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index 4349351e0..022932e95 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -7,6 +7,7 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.SimpleFactoryData; import edu.ie3.datamodel.io.factory.timeseries.SqlIdCoordinateFactory; @@ -17,7 +18,6 @@ import edu.ie3.util.geo.GeoUtils; import java.sql.Array; import java.sql.PreparedStatement; -import java.sql.SQLException; import java.util.*; import javax.measure.quantity.Length; import org.apache.commons.lang3.tuple.Pair; @@ -41,22 +41,24 @@ public class SqlIdCoordinateSource implements IdCoordinateSource { private final String queryForBoundingBox; private final String queryForNearestPoints; + private final String coordinateTableName; private final SqlDataSource dataSource; private final SqlIdCoordinateFactory factory; public SqlIdCoordinateSource( SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) - throws SQLException { + throws SourceException { this.factory = factory; this.dataSource = dataSource; + this.coordinateTableName = coordinateTableName; String dbIdColumnName = dataSource.getDbColumnName(factory.getIdField(), coordinateTableName); String dbPointColumnName = dataSource.getDbColumnName(factory.getCoordinateField(), coordinateTableName); // validating table - dataSource.connector.validateDBTable(coordinateTableName, Pair.class, factory); + factory.validate(getSourceFields(Pair.class), Pair.class); // setup queries this.basicQuery = createBaseQueryString(dataSource.schemaName, coordinateTableName); @@ -82,13 +84,18 @@ public SqlIdCoordinateSource( String schemaName, String coordinateTableName, SqlIdCoordinateFactory factory) - throws SQLException { + throws SourceException { this( factory, coordinateTableName, new SqlDataSource(connector, schemaName, new DatabaseNamingStrategy())); } + @Override + public Set getSourceFields(Class entityClass) throws SourceException { + return dataSource.getSourceFields(coordinateTableName); + } + @Override public Optional getCoordinate(int id) { List values = executeQueryToList(queryForPoint, ps -> ps.setInt(1, id)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index fa084c3d8..f9491c403 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -7,12 +7,13 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; -import java.sql.SQLException; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; public class SqlTimeSeriesMappingSource extends TimeSeriesMappingSource { @@ -25,7 +26,7 @@ public SqlTimeSeriesMappingSource( SqlConnector connector, String schemaName, EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) - throws SQLException { + throws SourceException { this.dataSource = new SqlDataSource( connector, schemaName, new DatabaseNamingStrategy(entityPersistenceNamingStrategy)); @@ -35,11 +36,16 @@ public SqlTimeSeriesMappingSource( entityPersistenceNamingStrategy.getEntityName(MappingEntry.class).orElseThrow(); this.queryFull = createBaseQueryString(schemaName, tableName); - dataSource.connector.validateDBTable(tableName, TimeSeriesMappingSource.class, mappingFactory); + mappingFactory.validate(getSourceFields(), MappingEntry.class); } @Override public Stream> getMappingSourceData() { return dataSource.executeQuery(queryFull); } + + @Override + public Set getSourceFields() throws SourceException { + return dataSource.getSourceFields(tableName); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java index 1100ad3c2..2873b927e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java @@ -19,7 +19,6 @@ import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.utils.TimeSeriesUtils; import edu.ie3.util.interval.ClosedInterval; -import java.sql.SQLException; import java.sql.Timestamp; import java.time.ZonedDateTime; import java.util.*; @@ -52,7 +51,7 @@ public SqlTimeSeriesSource( UUID timeSeriesUuid, Class valueClass, TimeBasedSimpleValueFactory factory) - throws SQLException { + throws SourceException { super(valueClass, factory); this.dataSource = sqlDataSource; @@ -65,7 +64,7 @@ public SqlTimeSeriesSource( final String tableName = sqlDataSource.databaseNamingStrategy.getTimeSeriesEntityName(columnScheme); - dataSource.connector.validateDBTable(tableName, valueClass, factory); + factory.validate(dataSource.getSourceFields(tableName), valueClass); String dbTimeColumnName = sqlDataSource.getDbColumnName(factory.getTimeFieldString(), tableName); @@ -93,7 +92,7 @@ public SqlTimeSeriesSource( UUID timeSeriesUuid, Class valueClass, TimeBasedSimpleValueFactory factory) - throws SQLException { + throws SourceException { this( new SqlDataSource(connector, schemaName, namingStrategy), timeSeriesUuid, @@ -118,7 +117,7 @@ public static SqlTimeSeriesSource createSource( DatabaseNamingStrategy namingStrategy, IndividualTimeSeriesMetaInformation metaInformation, String timePattern) - throws SourceException, SQLException { + throws SourceException { if (!TimeSeriesUtils.isSchemeAccepted(metaInformation.getColumnScheme())) throw new SourceException( "Unsupported column scheme '" + metaInformation.getColumnScheme() + "'."); @@ -136,7 +135,7 @@ private static SqlTimeSeriesSource create( UUID timeSeriesUuid, Class valClass, String timePattern) - throws SQLException { + throws SourceException { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass, timePattern); return new SqlTimeSeriesSource<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 4045d128e..9e3cbc85c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -30,6 +30,7 @@ public class SqlWeatherSource extends WeatherSource { private static final String WHERE = " WHERE "; private final String factoryCoordinateFieldName; + private final String tableName; /** * Queries that are available within this source. Motivation to have them as field value is to @@ -55,12 +56,13 @@ public SqlWeatherSource( String schemaName, String weatherTableName, TimeBasedWeatherValueFactory weatherFactory) - throws SQLException { + throws SourceException { super(idCoordinateSource, weatherFactory); this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); this.dataSource = new SqlDataSource(connector, schemaName, new DatabaseNamingStrategy()); + this.tableName = weatherTableName; - dataSource.connector.validateDBTable(weatherTableName, WeatherValue.class, weatherFactory); + weatherFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); String dbTimeColumnName = dataSource.getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName); @@ -78,6 +80,12 @@ public SqlWeatherSource( schemaName, weatherTableName, dbTimeColumnName, dbCoordinateIdColumnName); } + @Override + public Set getSourceFields(Class entityClass) + throws SourceException { + return dataSource.getSourceFields(tableName); + } + @Override public Map> getWeather( ClosedInterval timeInterval) throws SourceException { diff --git a/src/main/java/edu/ie3/datamodel/utils/ExceptionUtils.java b/src/main/java/edu/ie3/datamodel/utils/ExceptionUtils.java index ebbaa61f8..ec5851441 100644 --- a/src/main/java/edu/ie3/datamodel/utils/ExceptionUtils.java +++ b/src/main/java/edu/ie3/datamodel/utils/ExceptionUtils.java @@ -6,6 +6,7 @@ package edu.ie3.datamodel.utils; import edu.ie3.datamodel.models.UniqueEntity; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -28,6 +29,19 @@ public static String getMessages(List exceptions) { .replaceFirst("\n ", ""); } + /** + * Creates a string containing multiple exception messages. + * + * @param exceptions list of exceptions + * @return str containing the messages + */ + public static String getFullMessages(List exceptions) { + return exceptions.stream() + .map(e -> e.getMessage() + printStackTrace(e.getStackTrace())) + .reduce("", (a, b) -> a + "\n " + b) + .replaceFirst("\n ", ""); + } + /** * Combines multiple {@link UniqueEntity} into a string. * @@ -39,4 +53,16 @@ public static String combine(Collection entities) { + entities.stream().map(UniqueEntity::toString).collect(Collectors.joining(", ")) + "}"; } + + /** + * Method for combining {@link StackTraceElement}s. + * + * @param elements to be combined + * @return a string + */ + public static String printStackTrace(StackTraceElement... elements) { + return Arrays.stream(elements) + .map(StackTraceElement::toString) + .collect(Collectors.joining("\n ")); + } } diff --git a/src/main/java/edu/ie3/datamodel/utils/Try.java b/src/main/java/edu/ie3/datamodel/utils/Try.java index 06a9a5526..c3b348b0f 100644 --- a/src/main/java/edu/ie3/datamodel/utils/Try.java +++ b/src/main/java/edu/ie3/datamodel/utils/Try.java @@ -10,6 +10,7 @@ import edu.ie3.datamodel.exceptions.FailureException; import edu.ie3.datamodel.exceptions.TryException; import java.util.*; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -106,6 +107,17 @@ public static List getExceptions( return tries.stream().filter(Try::isFailure).map(t -> ((Failure) t).get()).toList(); } + /** + * Method to retrieve the exceptions from all {@link Failure} objects. + * + * @param tries array of {@link Try} objects + * @return a list of {@link Exception}'s + */ + @SafeVarargs + public static List getExceptions(Try... tries) { + return Arrays.stream(tries).filter(Try::isFailure).map(t -> ((Failure) t).get()).toList(); + } + /** * Method to scan a collection of {@link Try} objects for {@link Failure}'s. * @@ -181,6 +193,20 @@ public static Try, FailureException> scanStre // functional methods + /** + * Method to consume the contained data. + * + * @param consumer operation to compute + */ + public abstract void ifSuccess(Consumer consumer); + + /** + * Method to consume the exception + * + * @param consumer operation to compute + */ + public abstract void ifFailure(Consumer consumer); + /** * Method to transform the data if this object is a {@link Success}. * @@ -274,6 +300,16 @@ public Optional getException() { return Optional.empty(); } + @Override + public void ifSuccess(Consumer consumer) { + consumer.accept(data); + } + + @Override + public void ifFailure(Consumer consumer) { + // does nothing + } + @SuppressWarnings("unchecked") @Override public Try flatMap(Function> mapper) { @@ -359,6 +395,16 @@ public Optional getException() { return exception != null ? Optional.of(exception) : Optional.empty(); } + @Override + public void ifSuccess(Consumer consumer) { + // does nothing + } + + @Override + public void ifFailure(Consumer consumer) { + consumer.accept(exception); + } + @SuppressWarnings("unchecked") @Override public Try flatMap(Function> mapper) { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy index 8243a5b3d..b2255e6bc 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy @@ -280,16 +280,16 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until") when: - Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, TestAssetInput), FactoryException) + Try input = inputFactory.validate(foundFields, TestAssetInput) then: input.failure input.exception.get().message == "The provided fields [operates_from, operates_until, uuid] are invalid for instance of 'TestAssetInput'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + - "0: [id, uuid] or [id, uuid]\n" + - "1: [id, operatesFrom, uuid] or [id, operates_from, uuid]\n" + - "2: [id, operatesUntil, uuid] or [id, operates_until, uuid]\n" + - "3: [id, operatesFrom, operatesUntil, uuid] or [id, operates_from, operates_until, uuid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'TestAssetInput' are possible (NOT case-sensitive!):\n" + + "0: [id, uuid] or [id, uuid]\n" + + "1: [id, operatesFrom, uuid] or [id, operates_from, uuid]\n" + + "2: [id, operatesUntil, uuid] or [id, operates_until, uuid]\n" + + "3: [id, operatesFrom, operatesUntil, uuid] or [id, operates_from, operates_until, uuid]\n" } private static class TestAssetInput extends AssetInput { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy index 1091238cb..918c09d6e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy @@ -79,15 +79,15 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe def foundFields = inputFactory.newSet("uuid", "id", "s_rated", "cosphi_rated") when: - Try input = Try.ofVoid(() -> inputFactory.validate(foundFields, FixedFeedInInput), FactoryException) + Try input = inputFactory.validate(foundFields, FixedFeedInInput) then: input.failure input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of 'FixedFeedInInput'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + - "0: [cosPhiRated, id, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, q_characteristics, s_rated, uuid]\n" + - "1: [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid]\n" + - "2: [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid]\n" + - "3: [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" + + "0: [cosPhiRated, id, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, q_characteristics, s_rated, uuid]\n" + + "1: [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid]\n" + + "2: [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid]\n" + + "3: [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy index 6831a7746..62d414823 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy @@ -56,13 +56,13 @@ class FlexOptionsResultFactoryTest extends Specification implements FactoryTestH def foundFields = resultFactory.newSet("time", "input_model", "p_ref", "p_min") when: - Try input = Try.ofVoid(() -> resultFactory.validate(foundFields, FlexOptionsResult), FactoryException) + Try input = resultFactory.validate(foundFields, FlexOptionsResult) then: input.failure input.exception.get().message == "The provided fields [input_model, p_min, p_ref, time] are invalid for instance of 'FlexOptionsResult'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, pMax, pMin, pRef, time] or [input_model, p_max, p_min, p_ref, time]\n" + - "1: [inputModel, pMax, pMin, pRef, time, uuid] or [input_model, p_max, p_min, p_ref, time, uuid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FlexOptionsResult' are possible (NOT case-sensitive!):\n" + + "0: [inputModel, pMax, pMin, pRef, time] or [input_model, p_max, p_min, p_ref, time]\n" + + "1: [inputModel, pMax, pMin, pRef, time, uuid] or [input_model, p_max, p_min, p_ref, time, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy index 7767998d6..400126349 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy @@ -54,13 +54,13 @@ class NodeResultFactoryTest extends Specification implements FactoryTestHelper { def foundFields = resultFactory.newSet("time", "input_model", "v_mag") when: - Try input = Try.ofVoid(() -> resultFactory.validate(foundFields, NodeResult), FactoryException) + Try input = resultFactory.validate(foundFields, NodeResult) then: input.failure input.exception.get().message == "The provided fields [input_model, time, v_mag] are invalid for instance of 'NodeResult'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, time, vAng, vMag] or [input_model, time, v_ang, v_mag]\n" + - "1: [inputModel, time, uuid, vAng, vMag] or [input_model, time, uuid, v_ang, v_mag]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'NodeResult' are possible (NOT case-sensitive!):\n" + + "0: [inputModel, time, vAng, vMag] or [input_model, time, v_ang, v_mag]\n" + + "1: [inputModel, time, uuid, vAng, vMag] or [input_model, time, uuid, v_ang, v_mag]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy index 5c4983db3..416e03eb0 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy @@ -130,14 +130,14 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor def foundFields = resultFactory.newSet("time", "input_model", "q") when: - Try result = Try.ofVoid(() -> resultFactory.validate(foundFields, WecResult), FactoryException) + Try result = resultFactory.validate(foundFields, WecResult) then: result.failure result.exception.get().message == "The provided fields [input_model, q, time] are invalid for instance of 'WecResult'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + - "0: [inputModel, p, q, time] or [input_model, p, q, time]\n" + - "1: [inputModel, p, q, time, uuid] or [input_model, p, q, time, uuid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'WecResult' are possible (NOT case-sensitive!):\n" + + "0: [inputModel, p, q, time] or [input_model, p, q, time]\n" + + "1: [inputModel, p, q, time, uuid] or [input_model, p, q, time, uuid]\n" } def "A SystemParticipantResultFactory should be performant"() { @@ -156,7 +156,7 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor resultFactory.get(new SimpleEntityData(parameter, StorageResult)) } BigDecimal elapsedTime = (System - .currentTimeMillis() - startTime) / 1000.0 + .currentTimeMillis() - startTime) / 1000.0 elapsedTime < 3 } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index fc7534a04..1f0b9deaa 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -46,13 +46,13 @@ class CosmoIdCoordinateFactoryTest extends Specification { def foundFields = factory.newSet("tid", "id", "latrot", "longrot") when: - def actual = Try.of(() -> factory.validate(foundFields, Pair), FactoryException) + def actual = factory.validate(foundFields, Pair) then: actual.failure actual.exception.get().message == "The provided fields [id, latrot, longrot, tid] are invalid for instance of 'Pair'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + - "0: [id, latGeo, latRot, longGeo, longRot, tid] or [id, lat_geo, lat_rot, long_geo, long_rot, tid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + + "0: [id, latGeo, latRot, longGeo, longRot, tid] or [id, lat_geo, lat_rot, long_geo, long_rot, tid]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index fbc664d34..e5e5e0b34 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -44,13 +44,13 @@ class IconIdCoordinateFactoryTest extends Specification { def foundFields = factory.newSet("id", "latitude", "coordinatetype") when: - def actual = Try.of(() -> factory.validate(foundFields, Pair), FactoryException) + def actual = factory.validate(foundFields, Pair) then: actual.failure actual.exception.get().message == "The provided fields [coordinatetype, id, latitude] are invalid for instance of 'Pair'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + - "0: [coordinateType, id, latitude, longitude] or [coordinate_type, id, latitude, longitude]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'Pair' are possible (NOT case-sensitive!):\n" + + "0: [coordinateType, id, latitude, longitude] or [coordinate_type, id, latitude, longitude]\n" } def "A COSMO id to coordinate factory builds model from valid data"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy index 8f6340c8c..ceeb5df2a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy @@ -283,12 +283,12 @@ class SystemParticipantTypeInputFactoryTest extends Specification implements Fac def foundFields = typeInputFactory.newSet("uuid", "id", "capex", "opex", "srated", "cosPhiRated", "estorage", "pmin", "pmax", "eta", "dod", "lifetime",) when: - def input = Try.of(() -> typeInputFactory.validate(foundFields, StorageTypeInput), FactoryException) + def input = typeInputFactory.validate(foundFields, StorageTypeInput) then: input.failure input.exception.get().message == "The provided fields [capex, cosPhiRated, dod, estorage, eta, id, lifetime, opex, pmax, pmin, srated, uuid] are invalid for instance of 'StorageTypeInput'. \n" + - "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + - "0: [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid] or [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid]\n" + "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'StorageTypeInput' are possible (NOT case-sensitive!):\n" + + "0: [activePowerGradient, capex, cosPhiRated, dod, eStorage, eta, id, lifeCycle, lifeTime, opex, pMax, sRated, uuid] or [active_power_gradient, capex, cos_phi_rated, dod, e_storage, eta, id, life_cycle, life_time, opex, p_max, s_rated, uuid]\n" } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy index 32f4fd439..88efb0213 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy @@ -8,6 +8,7 @@ package edu.ie3.datamodel.io.source import edu.ie3.datamodel.io.factory.input.ThermalBusInputFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.csv.CsvDataSource +import edu.ie3.datamodel.models.UniqueEntity import edu.ie3.datamodel.models.input.OperatorInput import edu.ie3.datamodel.models.input.thermal.ThermalBusInput import edu.ie3.test.common.SystemParticipantTestData as sptd @@ -17,6 +18,7 @@ import spock.lang.Shared import spock.lang.Specification import java.nio.file.Path +import java.util.function.Function class EntitySourceTest extends Specification { @@ -26,8 +28,7 @@ class EntitySourceTest extends Specification { } @Override - Map, SourceValidator> getValidationMapping() { - return null + void validate() { } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy index 0e2e4a8da..3f4c7bb4f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.util.geo.CoordinateDistance import org.locationtech.jts.geom.Point import tech.units.indriya.ComparableQuantity @@ -13,6 +14,11 @@ import javax.measure.quantity.Length class IdCoordinateSourceMock implements IdCoordinateSource { + @Override + Set getSourceFields(Class entityClass) throws SourceException { + return [] as Set + } + @Override Optional getCoordinate(int id) { return Optional.empty() diff --git a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy index 950705e57..ef45c0db2 100644 --- a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.test.common +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvTestDataMeta import edu.ie3.util.geo.CoordinateDistance @@ -20,6 +21,11 @@ abstract class WeatherTestData { static final class DummyIdCoordinateSource implements CsvTestDataMeta, IdCoordinateSource { + @Override + Set getSourceFields(Class entityClass) throws SourceException { + return [] as Set + } + Optional getCoordinate(int id) { switch (id) { case 193186: return Optional.of(GeoUtils.buildPoint(7d, 49d)) From 1bc54883fa4494cff5e10c44a3aade14dc99fd17 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 5 Dec 2023 16:10:40 +0100 Subject: [PATCH 07/20] Some improvements. --- .../io/connectors/CouchbaseConnector.java | 7 ++- .../io/connectors/InfluxDbConnector.java | 6 +- .../ie3/datamodel/io/source/DataSource.java | 5 +- .../ie3/datamodel/io/source/EntitySource.java | 12 ++-- .../datamodel/io/source/GraphicSource.java | 9 +-- .../io/source/IdCoordinateSource.java | 5 +- .../datamodel/io/source/RawGridSource.java | 16 ++--- .../io/source/ResultEntitySource.java | 61 +++++++------------ .../io/source/SystemParticipantSource.java | 26 ++++---- .../datamodel/io/source/ThermalSource.java | 10 +-- .../io/source/TimeSeriesMappingSource.java | 4 +- .../ie3/datamodel/io/source/TypeSource.java | 45 ++++---------- .../datamodel/io/source/WeatherSource.java | 6 +- .../couchbase/CouchbaseWeatherSource.java | 13 ++-- .../io/source/csv/CsvDataSource.java | 12 ++-- .../io/source/csv/CsvIdCoordinateSource.java | 15 +++-- .../csv/CsvJointGridContainerSource.java | 17 ++---- .../csv/CsvTimeSeriesMappingSource.java | 5 +- .../io/source/csv/CsvWeatherSource.java | 10 ++- .../influxdb/InfluxDbWeatherSource.java | 10 ++- .../io/source/sql/SqlDataSource.java | 22 ++++--- .../io/source/sql/SqlIdCoordinateSource.java | 10 ++- .../sql/SqlTimeSeriesMappingSource.java | 5 +- .../io/source/sql/SqlTimeSeriesSource.java | 13 ++-- .../io/source/sql/SqlWeatherSource.java | 9 ++- .../io/source/EntitySourceTest.groovy | 6 +- .../io/source/IdCoordinateSourceMock.groovy | 4 +- .../ie3/test/common/WeatherTestData.groovy | 4 +- 28 files changed, 149 insertions(+), 218 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java index 42f7a8f82..2f15783ab 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java @@ -17,6 +17,7 @@ import java.time.Duration; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -59,9 +60,9 @@ public CouchbaseConnector( cluster = Cluster.connect(url, clusterOptions); } - /** Returns a set of found fields. */ + /** Returns the option for a set of found fields. */ @SuppressWarnings("unchecked") - public Set getSourceFields(Class entityClass) { + public Optional> getSourceFields(Class entityClass) { String query = "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM " + bucketName @@ -78,7 +79,7 @@ public Set getSourceFields(Class entityClass) { set.addAll((List) columns); } - return set; + return Optional.of(set); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java index 86805779f..079d36922 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java @@ -109,8 +109,8 @@ public InfluxDbConnector(String url, String databaseName) { this(url, databaseName, NO_SCENARIO, true, InfluxDB.LogLevel.NONE, BatchOptions.DEFAULTS); } - /** Returns the fields found in the source. */ - public Set getSourceFields(Class entityClass) { + /** Returns the option for fields found in the source. */ + public Optional> getSourceFields(Class entityClass) { QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName)); Map>> tagResults = parseQueryResult(tagKeys); @@ -121,7 +121,7 @@ public Set getSourceFields(Class entityClass) { tagResults.values().forEach(v -> v.stream().map(m -> m.get("tagKey")).forEach(set::add)); fieldResults.values().forEach(v -> v.stream().map(m -> m.get("fieldKey")).forEach(set::add)); - return set; + return Optional.of(set); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java index d41a4b731..7fb36eb8f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.models.UniqueEntity; import java.util.*; import java.util.stream.Stream; @@ -17,9 +16,9 @@ public interface DataSource { * Method to retrieve the fields found in the source. * * @param entityClass class of the source - * @return the found fields + * @return an option for the found fields */ - Set getSourceFields(Class entityClass) throws SourceException; + Optional> getSourceFields(Class entityClass); /** Creates a stream of maps that represent the rows in the database */ Stream> getSourceData(Class entityClass); diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index 5e0a4ae2d..71373bbfc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -46,17 +46,13 @@ public abstract class EntitySource { * * @param entityClass class to be validated * @param validator used to validate - * @return either a {@link FactoryException} wrapped by a failure or an empty success * @param type of the class */ - protected final Try validate( + protected final void validate( Class entityClass, SourceValidator validator) { - try { - Set actualFields = dataSource.getSourceFields(entityClass); - return validator.validate(actualFields, entityClass); - } catch (SourceException e) { - return Failure.of(new FactoryException(e.getCause())); - } + dataSource + .getSourceFields(entityClass) + .ifPresent(s -> validator.validate(s, entityClass).getOrThrow()); } protected String buildSkippingMessage( diff --git a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java index 8799cc5a4..4d8a577d5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.GraphicSourceException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.graphics.LineGraphicInputEntityData; @@ -53,12 +52,8 @@ public GraphicSource(TypeSource typeSource, RawGridSource rawGridSource, DataSou @Override public void validate() { - List exceptions = - Try.getExceptions( - validate(NodeGraphicInput.class, nodeGraphicInputFactory), - validate(LineGraphicInput.class, lineGraphicInputFactory)); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(NodeGraphicInput.class, nodeGraphicInputFactory); + validate(LineGraphicInput.class, lineGraphicInputFactory); } /** Returns the graphic elements of the grid or throws a {@link SourceException} */ diff --git a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java index d0374c2f6..f9b3768db 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.util.geo.CoordinateDistance; import edu.ie3.util.geo.GeoUtils; import java.util.*; @@ -24,9 +23,9 @@ public interface IdCoordinateSource { * Method to retrieve the fields found in the source. * * @param entityClass class of the source - * @return the found fields + * @return an option for the found fields */ - Set getSourceFields(Class entityClass) throws SourceException; + Optional> getSourceFields(Class entityClass); /** * Get the matching coordinate for the given ID diff --git a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java index cc75ce44e..a32ac1902 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java @@ -64,16 +64,12 @@ public RawGridSource(TypeSource typeSource, DataSource dataSource) { @Override public void validate() { - List exceptions = - Try.getExceptions( - validate(NodeInput.class, nodeInputFactory), - validate(LineInput.class, lineInputFactory), - validate(Transformer2WInput.class, transformer2WInputFactory), - validate(Transformer3WInput.class, transformer3WInputFactory), - validate(SwitchInput.class, switchInputFactory), - validate(MeasurementUnitInput.class, measurementUnitInputFactory)); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(NodeInput.class, nodeInputFactory); + validate(LineInput.class, lineInputFactory); + validate(Transformer2WInput.class, transformer2WInputFactory); + validate(Transformer3WInput.class, transformer3WInputFactory); + validate(SwitchInput.class, switchInputFactory); + validate(MeasurementUnitInput.class, measurementUnitInputFactory); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java index a29d3af84..b77feb5cc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.factory.SimpleEntityFactory; import edu.ie3.datamodel.io.factory.result.*; import edu.ie3.datamodel.models.result.NodeResult; @@ -17,12 +16,10 @@ import edu.ie3.datamodel.models.result.system.*; import edu.ie3.datamodel.models.result.thermal.CylindricalStorageResult; import edu.ie3.datamodel.models.result.thermal.ThermalHouseResult; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link ResultEntity} container @@ -66,44 +63,28 @@ public ResultEntitySource(DataSource dataSource, String dtfPattern) { @Override public void validate() { - List exceptions = - new ArrayList<>( - Stream.of( - LoadResult.class, - FixedFeedInResult.class, - BmResult.class, - PvResult.class, - ChpResult.class, - WecResult.class, - StorageResult.class, - EvcsResult.class, - EvResult.class, - HpResult.class, - EmResult.class) - .map(c -> validate(c, systemParticipantResultFactory).getException()) - .flatMap(Optional::stream) - .toList()); + List.of( + LoadResult.class, + FixedFeedInResult.class, + BmResult.class, + PvResult.class, + ChpResult.class, + WecResult.class, + StorageResult.class, + EvcsResult.class, + EvResult.class, + HpResult.class, + EmResult.class) + .forEach(c -> validate(c, systemParticipantResultFactory)); - validate(ThermalHouseResult.class, thermalResultFactory) - .getException() - .ifPresent(exceptions::add); - validate(CylindricalStorageResult.class, thermalResultFactory) - .getException() - .ifPresent(exceptions::add); - validate(SwitchResult.class, switchResultFactory).getException().ifPresent(exceptions::add); - validate(NodeResult.class, nodeResultFactory).getException().ifPresent(exceptions::add); - validate(LineResult.class, connectorResultFactory).getException().ifPresent(exceptions::add); - validate(Transformer2WResult.class, connectorResultFactory) - .getException() - .ifPresent(exceptions::add); - validate(Transformer3WResult.class, connectorResultFactory) - .getException() - .ifPresent(exceptions::add); - validate(FlexOptionsResult.class, flexOptionsResultFactory) - .getException() - .ifPresent(exceptions::add); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(ThermalHouseResult.class, thermalResultFactory); + validate(CylindricalStorageResult.class, thermalResultFactory); + validate(SwitchResult.class, switchResultFactory); + validate(NodeResult.class, nodeResultFactory); + validate(LineResult.class, connectorResultFactory); + validate(Transformer2WResult.class, connectorResultFactory); + validate(Transformer3WResult.class, connectorResultFactory); + validate(FlexOptionsResult.class, flexOptionsResultFactory); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index a5d473370..67e15ed4f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -79,21 +79,17 @@ public SystemParticipantSource( @Override public void validate() { - List exceptions = - Try.getExceptions( - validate(BmInput.class, bmInputFactory), - validate(ChpInput.class, chpInputFactory), - validate(EvInput.class, evInputFactory), - validate(FixedFeedInInput.class, fixedFeedInInputFactory), - validate(HpInput.class, hpInputFactory), - validate(LoadInput.class, loadInputFactory), - validate(PvInput.class, pvInputFactory), - validate(StorageInput.class, storageInputFactory), - validate(WecInput.class, wecInputFactory), - validate(EvcsInput.class, evcsInputFactory), - validate(EmInput.class, emInputFactory)); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(BmInput.class, bmInputFactory); + validate(ChpInput.class, chpInputFactory); + validate(EvInput.class, evInputFactory); + validate(FixedFeedInInput.class, fixedFeedInInputFactory); + validate(HpInput.class, hpInputFactory); + validate(LoadInput.class, loadInputFactory); + validate(PvInput.class, pvInputFactory); + validate(StorageInput.class, storageInputFactory); + validate(WecInput.class, wecInputFactory); + validate(EvcsInput.class, evcsInputFactory); + validate(EmInput.class, emInputFactory); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java index 93bd9754d..385629564 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java @@ -48,13 +48,9 @@ public ThermalSource(TypeSource typeSource, DataSource dataSource) { @Override public void validate() { - List exceptions = - Try.getExceptions( - validate(ThermalBusInput.class, thermalBusInputFactory), - validate(CylindricalStorageInput.class, cylindricalStorageInputFactory), - validate(ThermalHouseInput.class, thermalHouseInputFactory)); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(ThermalBusInput.class, thermalBusInputFactory); + validate(CylindricalStorageInput.class, cylindricalStorageInputFactory); + validate(ThermalHouseInput.class, thermalHouseInputFactory); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java index 72f213d00..548080aaf 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TimeSeriesMappingSource.java @@ -59,8 +59,8 @@ public Optional getTimeSeriesUuid(UUID modelIdentifier) { */ public abstract Stream> getMappingSourceData(); - /** Returns the fields found in the source */ - public abstract Set getSourceFields() throws SourceException; + /** Returns the option for fields found in the source */ + public abstract Optional> getSourceFields() throws SourceException; // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java index bbde7017b..b8420c364 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.factory.input.OperatorInputFactory; import edu.ie3.datamodel.io.factory.typeinput.LineTypeInputFactory; @@ -18,11 +17,8 @@ import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput; import edu.ie3.datamodel.models.input.system.type.*; import edu.ie3.datamodel.utils.Try; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Set; -import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link @@ -52,34 +48,19 @@ public TypeSource(DataSource dataSource) { @Override public void validate() { - validate(OperatorInput.class, operatorInputFactory) - .ifFailure( - e -> - log.warn( - "The source for '{}' could not be validated due to: {}", - OperatorInput.class, - e)); - - List exceptions = - new ArrayList<>( - Try.getExceptions( - validate(LineTypeInput.class, lineTypeInputFactory), - validate(Transformer2WTypeInput.class, transformer2WTypeInputFactory), - validate(Transformer3WTypeInput.class, transformer3WTypeInputFactory))); - - exceptions.addAll( - Stream.of( - EvTypeInput.class, - HpTypeInput.class, - BmTypeInput.class, - WecTypeInput.class, - ChpTypeInput.class, - StorageTypeInput.class) - .map(c -> validate(c, systemParticipantTypeInputFactory).getException()) - .flatMap(Optional::stream) - .toList()); - - exceptions.forEach(e -> log.warn("The following exception was thrown while validating: ", e)); + validate(OperatorInput.class, operatorInputFactory); + validate(LineTypeInput.class, lineTypeInputFactory); + validate(Transformer2WTypeInput.class, transformer2WTypeInputFactory); + validate(Transformer3WTypeInput.class, transformer3WTypeInputFactory); + + List.of( + EvTypeInput.class, + HpTypeInput.class, + BmTypeInput.class, + WecTypeInput.class, + ChpTypeInput.class, + StorageTypeInput.class) + .forEach(c -> validate(c, systemParticipantTypeInputFactory)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index a1b4b29f0..bb579a90c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -44,10 +44,10 @@ protected WeatherSource( * Method to retrieve the fields found in the source. * * @param entityClass class of the source - * @return the found fields + * @return an option for fields found in the source */ - public abstract Set getSourceFields(Class entityClass) - throws SourceException; + public abstract Optional> getSourceFields( + Class entityClass); // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index f1a50d0fc..257acc678 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -10,7 +10,6 @@ import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.query.QueryResult; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CouchbaseConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -60,8 +59,7 @@ public CouchbaseWeatherSource( IdCoordinateSource coordinateSource, String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) - throws SourceException { + String timeStampPattern) { this( connector, coordinateSource, @@ -90,20 +88,19 @@ public CouchbaseWeatherSource( String coordinateIdColumnName, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) - throws SourceException { + String timeStampPattern) { super(idCoordinateSource, weatherFactory); this.connector = connector; this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.timeStampPattern = timeStampPattern; - weatherFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); + getSourceFields(WeatherValue.class) + .ifPresent(s -> weatherFactory.validate(s, WeatherValue.class).getOrThrow()); } @Override - public Set getSourceFields(Class entityClass) - throws SourceException { + public Optional> getSourceFields(Class entityClass) { return connector.getSourceFields(entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index baf343e69..ea39eac0a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -59,12 +59,16 @@ public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNami } @Override - public Set getSourceFields(Class entityClass) - throws SourceException { + public Optional> getSourceFields(Class entityClass) { try (BufferedReader reader = connector.initReader(entityClass)) { - return Arrays.stream(parseCsvRow(reader.readLine(), csvSep)).collect(Collectors.toSet()); + return Optional.of( + Arrays.stream(parseCsvRow(reader.readLine(), csvSep)).collect(Collectors.toSet())); } catch (ConnectorException | IOException e) { - throw new SourceException("The following exception was thrown while reading a source: ", e); + log.warn( + "The source for the entity '{}' couldn't be read and therefore not be validated! Cause: {}", + entityClass, + e.getMessage()); + return Optional.empty(); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java index 1cbf60f20..05db85a0a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java @@ -87,12 +87,17 @@ private Map invert(Map map) { } @Override - public Set getSourceFields(Class entityClass) throws SourceException { + public Optional> getSourceFields(Class entityClass) { try (BufferedReader reader = dataSource.connector.initIdCoordinateReader()) { - return Arrays.stream(dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep)) - .collect(Collectors.toSet()); + return Optional.of( + Arrays.stream(dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep)) + .collect(Collectors.toSet())); } catch (IOException e) { - throw new SourceException("The following exception was thrown while reading a source: ", e); + log.warn( + "The source for the entity '{}' couldn't be read and therefore not be validated!", + entityClass, + e); + return Optional.empty(); } } @@ -173,7 +178,7 @@ protected Stream> buildStreamWithFieldsToAttributesMap() { final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); // validating read file - factory.validate(Set.of(headline), Pair.class); + factory.validate(Set.of(headline), Pair.class).getOrThrow(); // by default try-with-resources closes the reader directly when we leave this method (which // is wanted to avoid a lock on the file), but this causes a closing of the stream as well. diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java index e4e46238d..8aa9b1cb7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.csv; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FileException; import edu.ie3.datamodel.exceptions.InvalidGridException; import edu.ie3.datamodel.exceptions.SourceException; @@ -17,7 +16,6 @@ import edu.ie3.datamodel.models.input.container.JointGridContainer; import edu.ie3.datamodel.models.input.container.RawGridElements; import edu.ie3.datamodel.models.input.container.SystemParticipants; -import edu.ie3.datamodel.utils.ExceptionUtils; import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; import java.util.*; @@ -55,17 +53,10 @@ public static JointGridContainer read( GraphicSource graphicSource = new GraphicSource(typeSource, rawGridSource, dataSource); /* validating sources */ - List validationException = - Try.getExceptions( - Try.ofVoid(typeSource::validate, FactoryException.class), - Try.ofVoid(rawGridSource::validate, FactoryException.class), - Try.ofVoid(systemParticipantSource::validate, FactoryException.class), - Try.ofVoid(graphicSource::validate, FactoryException.class)); - - if (!validationException.isEmpty()) { - throw new FactoryException( - "Validation failed due to: " + ExceptionUtils.getFullMessages(validationException)); - } + typeSource.validate(); + rawGridSource.validate(); + systemParticipantSource.validate(); + graphicSource.validate(); /* Loading models */ Try rawGridElements = diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index b06ae4231..7d4244428 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -10,6 +10,7 @@ import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.nio.file.Path; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -23,7 +24,7 @@ public CsvTimeSeriesMappingSource( this.dataSource = new CsvDataSource(csvSep, gridFolderPath, fileNamingStrategy); // validating - mappingFactory.validate(getSourceFields(), MappingEntry.class); + getSourceFields().ifPresent(s -> mappingFactory.validate(s, MappingEntry.class).getOrThrow()); } @Override @@ -33,7 +34,7 @@ public Stream> getMappingSourceData() { } @Override - public Set getSourceFields() throws SourceException { + public Optional> getSourceFields() throws SourceException { return dataSource.getSourceFields(MappingEntry.class); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index b7f45fa1b..a246ce024 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.csv; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -61,11 +60,10 @@ public CsvWeatherSource( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - /** Returns an empty set for now. */ + /** Returns an empty optional for now. */ @Override - public Set getSourceFields(Class entityClass) - throws SourceException { - return Collections.emptySet(); + public Optional> getSourceFields(Class entityClass) { + return Optional.empty(); } @Override @@ -188,7 +186,7 @@ private Stream> buildStreamWithFieldsToAttributesMap( final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); // validating read file - weatherFactory.validate(Set.of(headline), WeatherValue.class); + weatherFactory.validate(Set.of(headline), WeatherValue.class).getOrThrow(); // by default try-with-resources closes the reader directly when we leave this method (which // is wanted to avoid a lock on the file), but this causes a closing of the stream as well. diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index ec3a3d3e2..464e26f32 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.influxdb; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -48,17 +47,16 @@ public class InfluxDbWeatherSource extends WeatherSource { public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource idCoordinateSource, - TimeBasedWeatherValueFactory weatherValueFactory) - throws SourceException { + TimeBasedWeatherValueFactory weatherValueFactory) { super(idCoordinateSource, weatherValueFactory); this.connector = connector; - weatherValueFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); + getSourceFields(WeatherValue.class) + .ifPresent(s -> weatherValueFactory.validate(s, WeatherValue.class).getOrThrow()); } @Override - public Set getSourceFields(Class entityClass) - throws SourceException { + public Optional> getSourceFields(Class entityClass) { return connector.getSourceFields(entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 7322c0fc6..fd5ed3e18 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -6,7 +6,6 @@ package edu.ie3.datamodel.io.source.sql; import edu.ie3.datamodel.exceptions.InvalidColumnNameException; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.source.DataSource; @@ -110,8 +109,7 @@ protected List getDbTables(String schemaPattern, String tableNamePattern } @Override - public Set getSourceFields(Class entityClass) - throws SourceException { + public Optional> getSourceFields(Class entityClass) { try { String tableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); ResultSet rs = @@ -123,9 +121,13 @@ public Set getSourceFields(Class entityClass) columnNames.add(StringUtils.snakeCaseToCamelCase(name)); } - return columnNames; + return Optional.of(columnNames); } catch (NoSuchElementException | SQLException e) { - throw new SourceException("The following exception was thrown while reading a source: ", e); + log.warn( + "The source for the entity '{}' couldn't be read and therefore not be validated! Cause: {}", + entityClass, + e.getMessage()); + return Optional.empty(); } } @@ -133,10 +135,9 @@ public Set getSourceFields(Class entityClass) * Method that uses the table name to retrieve all field names. * * @param tableName to be used - * @return a set of found fields - * @throws SourceException - if an error occurred + * @return an option for a set of found fields */ - public Set getSourceFields(String tableName) throws SourceException { + public Optional> getSourceFields(String tableName) { try { ResultSet rs = connector.getConnection().getMetaData().getColumns(null, null, tableName, null); @@ -147,9 +148,10 @@ public Set getSourceFields(String tableName) throws SourceException { columnNames.add(StringUtils.snakeCaseToCamelCase(name)); } - return columnNames; + return Optional.of(columnNames); } catch (NoSuchElementException | SQLException e) { - throw new SourceException("The following exception was thrown while reading a source: ", e); + log.warn("The table '{}' couldn't be read and therefore not be validated!", tableName, e); + return Optional.empty(); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index 022932e95..6dfbbe157 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -47,8 +47,7 @@ public class SqlIdCoordinateSource implements IdCoordinateSource { private final SqlIdCoordinateFactory factory; public SqlIdCoordinateSource( - SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) - throws SourceException { + SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) { this.factory = factory; this.dataSource = dataSource; this.coordinateTableName = coordinateTableName; @@ -58,7 +57,7 @@ public SqlIdCoordinateSource( dataSource.getDbColumnName(factory.getCoordinateField(), coordinateTableName); // validating table - factory.validate(getSourceFields(Pair.class), Pair.class); + getSourceFields(Pair.class).ifPresent(s -> factory.validate(s, Pair.class).getOrThrow()); // setup queries this.basicQuery = createBaseQueryString(dataSource.schemaName, coordinateTableName); @@ -83,8 +82,7 @@ public SqlIdCoordinateSource( SqlConnector connector, String schemaName, String coordinateTableName, - SqlIdCoordinateFactory factory) - throws SourceException { + SqlIdCoordinateFactory factory) { this( factory, coordinateTableName, @@ -92,7 +90,7 @@ public SqlIdCoordinateSource( } @Override - public Set getSourceFields(Class entityClass) throws SourceException { + public Optional> getSourceFields(Class entityClass) { return dataSource.getSourceFields(coordinateTableName); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index f9491c403..b5f817058 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -13,6 +13,7 @@ import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -36,7 +37,7 @@ public SqlTimeSeriesMappingSource( entityPersistenceNamingStrategy.getEntityName(MappingEntry.class).orElseThrow(); this.queryFull = createBaseQueryString(schemaName, tableName); - mappingFactory.validate(getSourceFields(), MappingEntry.class); + getSourceFields().ifPresent(s -> mappingFactory.validate(s, MappingEntry.class).getOrThrow()); } @Override @@ -45,7 +46,7 @@ public Stream> getMappingSourceData() { } @Override - public Set getSourceFields() throws SourceException { + public Optional> getSourceFields() throws SourceException { return dataSource.getSourceFields(tableName); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java index 2873b927e..8cbffccc5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java @@ -50,8 +50,7 @@ public SqlTimeSeriesSource( SqlDataSource sqlDataSource, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) - throws SourceException { + TimeBasedSimpleValueFactory factory) { super(valueClass, factory); this.dataSource = sqlDataSource; @@ -64,7 +63,9 @@ public SqlTimeSeriesSource( final String tableName = sqlDataSource.databaseNamingStrategy.getTimeSeriesEntityName(columnScheme); - factory.validate(dataSource.getSourceFields(tableName), valueClass); + dataSource + .getSourceFields(tableName) + .ifPresent(s -> factory.validate(s, valueClass).getOrThrow()); String dbTimeColumnName = sqlDataSource.getDbColumnName(factory.getTimeFieldString(), tableName); @@ -91,8 +92,7 @@ public SqlTimeSeriesSource( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) - throws SourceException { + TimeBasedSimpleValueFactory factory) { this( new SqlDataSource(connector, schemaName, namingStrategy), timeSeriesUuid, @@ -134,8 +134,7 @@ private static SqlTimeSeriesSource create( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valClass, - String timePattern) - throws SourceException { + String timePattern) { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass, timePattern); return new SqlTimeSeriesSource<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 9e3cbc85c..6e69d808a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -55,14 +55,14 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, - TimeBasedWeatherValueFactory weatherFactory) - throws SourceException { + TimeBasedWeatherValueFactory weatherFactory) { super(idCoordinateSource, weatherFactory); this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); this.dataSource = new SqlDataSource(connector, schemaName, new DatabaseNamingStrategy()); this.tableName = weatherTableName; - weatherFactory.validate(getSourceFields(WeatherValue.class), WeatherValue.class); + getSourceFields(WeatherValue.class) + .ifPresent(s -> weatherFactory.validate(s, WeatherValue.class).getOrThrow()); String dbTimeColumnName = dataSource.getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName); @@ -81,8 +81,7 @@ public SqlWeatherSource( } @Override - public Set getSourceFields(Class entityClass) - throws SourceException { + public Optional> getSourceFields(Class entityClass) { return dataSource.getSourceFields(tableName); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy index 88efb0213..507d9fb0e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy @@ -5,20 +5,18 @@ */ package edu.ie3.datamodel.io.source +import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.io.factory.input.ThermalBusInputFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.csv.CsvDataSource -import edu.ie3.datamodel.models.UniqueEntity import edu.ie3.datamodel.models.input.OperatorInput import edu.ie3.datamodel.models.input.thermal.ThermalBusInput -import edu.ie3.test.common.SystemParticipantTestData as sptd import edu.ie3.test.common.GridTestData as gtd - +import edu.ie3.test.common.SystemParticipantTestData as sptd import spock.lang.Shared import spock.lang.Specification import java.nio.file.Path -import java.util.function.Function class EntitySourceTest extends Specification { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy index 3f4c7bb4f..992e868de 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/IdCoordinateSourceMock.groovy @@ -15,8 +15,8 @@ import javax.measure.quantity.Length class IdCoordinateSourceMock implements IdCoordinateSource { @Override - Set getSourceFields(Class entityClass) throws SourceException { - return [] as Set + Optional> getSourceFields(Class entityClass) throws SourceException { + return Optional.empty() } @Override diff --git a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy index ef45c0db2..0f19dcbd8 100644 --- a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy @@ -22,8 +22,8 @@ abstract class WeatherTestData { static final class DummyIdCoordinateSource implements CsvTestDataMeta, IdCoordinateSource { @Override - Set getSourceFields(Class entityClass) throws SourceException { - return [] as Set + Optional> getSourceFields(Class entityClass) throws SourceException { + return Optional.empty() } Optional getCoordinate(int id) { From 3cc295c738929bd059ef16344c9ed5a2524573ac Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 13:44:16 +0100 Subject: [PATCH 08/20] Removed unused import --- .../edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index 6dfbbe157..d4a1b5255 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -7,7 +7,6 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; -import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.SimpleFactoryData; import edu.ie3.datamodel.io.factory.timeseries.SqlIdCoordinateFactory; From 1bdc06fce33c939991b9cbfa0814022b9f345262 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 14:47:57 +0100 Subject: [PATCH 09/20] Adapted to changes in dev --- .../edu/ie3/datamodel/io/source/EnergyManagementSource.java | 5 +++++ .../edu/ie3/datamodel/io/source/SystemParticipantSource.java | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java b/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java index 51e120b1f..b0770517d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java @@ -26,6 +26,11 @@ public EnergyManagementSource(TypeSource typeSource, DataSource dataSource) { this.emInputFactory = new EmInputFactory(); } + @Override + public void validate() { + validate(EmInput.class, emInputFactory); + } + /** * Returns a unique set of {@link EmInput} instances. * diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index 64776bbe5..a29a45b1c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -87,7 +87,6 @@ public void validate() { validate(StorageInput.class, storageInputFactory); validate(WecInput.class, wecInputFactory); validate(EvcsInput.class, evcsInputFactory); - validate(EmInput.class, emInputFactory); } /** From e50da82b4251fde7ae27106eebb9690f8dc2c0fd Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:09:18 +0100 Subject: [PATCH 10/20] Headline check is now obsolete --- .../edu/ie3/datamodel/io/source/csv/CsvDataSource.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index ea39eac0a..24264e058 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -305,13 +305,6 @@ protected Stream> buildStreamWithFieldsToAttributesMap( try (BufferedReader reader = bufferedReader) { final String[] headline = parseCsvRow(reader.readLine(), csvSep); - // sanity check for headline - if (!Arrays.asList(headline).contains("uuid")) { - throw new SourceException( - "The first line does not contain a field named 'uuid'. Is the headline valid?\nProvided headline: " - + String.join(", ", headline)); - } - // by default try-with-resources closes the reader directly when we leave this method (which // is wanted to avoid a lock on the file), but this causes a closing of the stream as well. // As we still want to consume the data at other places, we start a new stream instead of @@ -324,9 +317,6 @@ protected Stream> buildStreamWithFieldsToAttributesMap( } catch (IOException e) { log.warn( "Cannot read file to build entity '{}': {}", entityClass.getSimpleName(), e.getMessage()); - } catch (SourceException e) { - log.error( - "Cannot read file to build entity '{}': {}", entityClass.getSimpleName(), e.getMessage()); } return Stream.empty(); From d210341cb5ab303af48c1ab97af31a94ae28f65a Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:22:59 +0100 Subject: [PATCH 11/20] Removing unnecessary parameters --- .../edu/ie3/datamodel/io/connectors/CouchbaseConnector.java | 2 +- .../java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java | 2 +- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 2 +- .../ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java index 2f15783ab..ffeaf86fb 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java @@ -62,7 +62,7 @@ public CouchbaseConnector( /** Returns the option for a set of found fields. */ @SuppressWarnings("unchecked") - public Optional> getSourceFields(Class entityClass) { + public Optional> getSourceFields() { String query = "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM " + bucketName diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java index 079d36922..f96fb621a 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java @@ -110,7 +110,7 @@ public InfluxDbConnector(String url, String databaseName) { } /** Returns the option for fields found in the source. */ - public Optional> getSourceFields(Class entityClass) { + public Optional> getSourceFields() { QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName)); Map>> tagResults = parseQueryResult(tagKeys); diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 257acc678..2c90d04f8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -101,7 +101,7 @@ public CouchbaseWeatherSource( @Override public Optional> getSourceFields(Class entityClass) { - return connector.getSourceFields(entityClass); + return connector.getSourceFields(); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 464e26f32..4822038f8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -57,7 +57,7 @@ public InfluxDbWeatherSource( @Override public Optional> getSourceFields(Class entityClass) { - return connector.getSourceFields(entityClass); + return connector.getSourceFields(); } @Override From 82249ef2b3583c342a22a2c64e1b1cf792cfd17e Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:30:53 +0100 Subject: [PATCH 12/20] Better JavaDoc --- src/main/java/edu/ie3/datamodel/io/factory/Factory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index e3c4fc67e..46fe5c9df 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -111,7 +111,8 @@ private void isSupportedClass(Class desiredClass) { /** * Method to find and return additional fields that were found in a source. This method will - * return the additional fields that were found in the valid set with the least additional fields. + * return the minimal additional fields among all field sets, meaning that the set of actual + * fields is compared to the field set with the least additional fields. * * @param actualFields found in the source * @param validFieldSets that contains at least all fields found in the source From cf15f4d5fde1d6de29d742f44d48745ef8f3b468 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:31:04 +0100 Subject: [PATCH 13/20] Additional test case --- .../datamodel/io/factory/FactoryTest.groovy | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy index e7dc42212..b4706008b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy @@ -44,6 +44,27 @@ class FactoryTest extends Specification { ["uuid", "value_1", "value_3"] as Set, ["id", "time", "value_1"] as Set ] | ["id", "time", "value_2"] + [ + "uuid", + "id", + "time", + "value_1" + ] | [ + [ + "uuid", + "id", + "time", + "value_1", + "value_2" + ] as Set, + [ + "uuid", + "id", + "time", + "value_1", + "value_3" + ] as Set + ] | [] } def "A Factory should allow additional fields"() { From 18429bc36e2daabfa66961bff069f9e5dc4a3c38 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:34:48 +0100 Subject: [PATCH 14/20] Renaming method, since "additional fields" already exist with a different meaning --- .../edu/ie3/datamodel/io/factory/Factory.java | 16 ++++++++-------- .../ie3/datamodel/io/factory/FactoryTest.groovy | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 46fe5c9df..8b67974ae 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -110,15 +110,15 @@ private void isSupportedClass(Class desiredClass) { protected abstract List> getFields(Class entityClass); /** - * Method to find and return additional fields that were found in a source. This method will - * return the minimal additional fields among all field sets, meaning that the set of actual - * fields is compared to the field set with the least additional fields. + * Method to find and return additional fields that were found in a source and are not used by the + * data model. This method will return the minimal unused fields among all field sets, meaning + * that the set of actual fields is compared to the field set with the least unused fields. * * @param actualFields found in the source * @param validFieldSets that contains at least all fields found in the source - * @return a set of additional fields + * @return a set of unused fields */ - protected Set getAdditionalFields( + protected Set getUnusedFields( Set actualFields, List> validFieldSets) { // checking for additional fields // and returning the set with the least additional fields @@ -172,13 +172,13 @@ public Try validate( + "' are possible (NOT case-sensitive!):\n" + possibleOptions)); } else { - Set additionalFields = getAdditionalFields(harmonizedFoundFields, validFieldSets); + Set unused = getUnusedFields(harmonizedFoundFields, validFieldSets); - if (!additionalFields.isEmpty()) { + if (!unused.isEmpty()) { log.debug( "The following additional fields were found for entity class of '{}': {}", entityClass.getSimpleName(), - additionalFields); + unused); } return Success.empty(); diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy index b4706008b..cfb943f9a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy @@ -15,12 +15,12 @@ class FactoryTest extends Specification { private DummyFactory factory = new DummyFactory(String) - def "A Factory can return additional fields correctly"() { + def "A Factory can return unused fields correctly"() { when: - def additional = factory.getAdditionalFields(actualFields as Set, validFieldSets) + def unused = factory.getUnusedFields(actualFields as Set, validFieldSets) then: - additional == expected as Set + unused == expected as Set where: actualFields | validFieldSets | expected From 84695c89ef7d3816433e9f0eb6c90475e5aa9c99 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 4 Jan 2024 15:42:09 +0100 Subject: [PATCH 15/20] found fields -> actual fields Making newSet and expandSet static --- .../edu/ie3/datamodel/io/factory/Factory.java | 20 +++++++++---------- .../datamodel/io/source/SourceValidator.java | 4 ++-- .../datamodel/io/factory/FactoryTest.groovy | 6 +++--- .../input/AssetInputEntityFactoryTest.groovy | 4 ++-- .../FixedFeedInInputFactoryTest.groovy | 4 ++-- .../FlexOptionsResultFactoryTest.groovy | 4 ++-- .../result/NodeResultFactoryTest.groovy | 4 ++-- .../SystemParticipantResultFactoryTest.groovy | 4 ++-- .../CosmoIdCoordinateFactoryTest.groovy | 6 ++---- .../IconIdCoordinateFactoryTest.groovy | 4 ++-- ...stemParticipantTypeInputFactoryTest.groovy | 4 ++-- 11 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index 8b67974ae..a19fd0813 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -134,10 +134,10 @@ protected Set getUnusedFields( } /** - * Method for validating the found fields. The found fields needs to fully contain at least one of - * the sets returned by {@link #getFields(Class)}. If the found fields don't contain all necessary - * fields, an {@link FactoryException} with a detail message is thrown. If the found fields - * contain more fields than necessary, these fields are ignored. + * Method for validating the actual fields. The actual fields need to fully contain at least one + * of the sets returned by {@link #getFields(Class)}. If the actual fields don't contain all + * necessary fields, an {@link FactoryException} with a detail message is thrown. If the actual + * fields contain more fields than necessary, these fields are ignored. * * @param actualFields that were found * @param entityClass of the build data @@ -146,14 +146,14 @@ protected Set getUnusedFields( public Try validate( Set actualFields, Class entityClass) { List> fieldSets = getFields(entityClass); - Set harmonizedFoundFields = toCamelCase(actualFields); + Set harmonizedActualFields = toCamelCase(actualFields); - // comparing the found fields to a list of possible fields (allows additional fields) + // comparing the actual fields to a list of possible fields (allows additional fields) // if not all fields were found in a set, this set is filtered out // all other fields are saved as a list // allows snake, camel and mixed cases List> validFieldSets = - fieldSets.stream().filter(harmonizedFoundFields::containsAll).toList(); + fieldSets.stream().filter(harmonizedActualFields::containsAll).toList(); if (validFieldSets.isEmpty()) { // build the exception string with extensive debug information @@ -172,7 +172,7 @@ public Try validate( + "' are possible (NOT case-sensitive!):\n" + possibleOptions)); } else { - Set unused = getUnusedFields(harmonizedFoundFields, validFieldSets); + Set unused = getUnusedFields(harmonizedActualFields, validFieldSets); if (!unused.isEmpty()) { log.debug( @@ -208,7 +208,7 @@ protected static StringBuilder getFieldsString(List> fieldSets) { * @param attributes attribute names * @return new set exactly containing attribute names */ - protected TreeSet newSet(String... attributes) { + protected static TreeSet newSet(String... attributes) { TreeSet set = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); set.addAll(Arrays.asList(attributes)); return set; @@ -223,7 +223,7 @@ protected TreeSet newSet(String... attributes) { * @param more attribute names to expand given set with * @return new set exactly containing given attribute set plus additional attributes */ - protected TreeSet expandSet(Set attributeSet, String... more) { + protected static TreeSet expandSet(Set attributeSet, String... more) { TreeSet newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); newSet.addAll(attributeSet); newSet.addAll(Arrays.asList(more)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java index 4c932c7c5..650fdf2b8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java @@ -15,9 +15,9 @@ public interface SourceValidator { /** * Method for validating a data source. * - * @param foundFields fields that were found in the source data + * @param actualFields fields that were found in the source data * @param entityClass that should be buildable from the source data * @return either an exception wrapped by a {@link Failure} or an empty success */ - Try validate(Set foundFields, Class entityClass); + Try validate(Set actualFields, Class entityClass); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy index cfb943f9a..38cb82ac7 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy @@ -69,13 +69,13 @@ class FactoryTest extends Specification { def "A Factory should allow additional fields"() { given: - def foundFields = factory.newSet("uuid", "id", "time", "value_1", "value_2", "value_3") + def actualFields = DummyFactory.newSet("uuid", "id", "time", "value_1", "value_2", "value_3") when: - Try input = Try.ofVoid(() -> factory.validate(foundFields, String), FactoryException) + def result = Try.ofVoid(() -> factory.validate(actualFields, String), FactoryException) then: - input.success + result.success } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy index b2255e6bc..b460b909a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactoryTest.groovy @@ -277,10 +277,10 @@ class AssetInputEntityFactoryTest extends Specification implements FactoryTestHe def "An AssetInputFactory should throw an exception on invalid or incomplete fields"() { given: def inputFactory = new TestAssetInputFactory() - def foundFields = inputFactory.newSet("uuid", "operates_from", "operates_until") + def actualFields = TestAssetInputFactory.newSet("uuid", "operates_from", "operates_until") when: - Try input = inputFactory.validate(foundFields, TestAssetInput) + Try input = inputFactory.validate(actualFields, TestAssetInput) then: input.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy index 0447a0333..9a3701d58 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy @@ -76,10 +76,10 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe def "A FixedFeedInInputFactory should throw an exception on invalid or incomplete data fields"() { given: def inputFactory = new FixedFeedInInputFactory() - def foundFields = inputFactory.newSet("uuid", "id", "s_rated", "cosphi_rated") + def actualFields = FixedFeedInInputFactory.newSet("uuid", "id", "s_rated", "cosphi_rated") when: - Try input = inputFactory.validate(foundFields, FixedFeedInInput) + Try input = inputFactory.validate(actualFields, FixedFeedInInput) then: input.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy index 62d414823..a4ff4b386 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/FlexOptionsResultFactoryTest.groovy @@ -53,10 +53,10 @@ class FlexOptionsResultFactoryTest extends Specification implements FactoryTestH def "A FlexOptionsResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new FlexOptionsResultFactory() - def foundFields = resultFactory.newSet("time", "input_model", "p_ref", "p_min") + def actualFields = FlexOptionsResultFactory.newSet("time", "input_model", "p_ref", "p_min") when: - Try input = resultFactory.validate(foundFields, FlexOptionsResult) + Try input = resultFactory.validate(actualFields, FlexOptionsResult) then: input.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy index 400126349..ef97cfece 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/NodeResultFactoryTest.groovy @@ -51,10 +51,10 @@ class NodeResultFactoryTest extends Specification implements FactoryTestHelper { def "A NodeResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new NodeResultFactory() - def foundFields = resultFactory.newSet("time", "input_model", "v_mag") + def actualFields = NodeResultFactory.newSet("time", "input_model", "v_mag") when: - Try input = resultFactory.validate(foundFields, NodeResult) + Try input = resultFactory.validate(actualFields, NodeResult) then: input.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy index 416e03eb0..c6c38c189 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy @@ -127,10 +127,10 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor def "A SystemParticipantResultFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def resultFactory = new SystemParticipantResultFactory() - def foundFields = resultFactory.newSet("time", "input_model", "q") + def actualFields = SystemParticipantResultFactory.newSet("time", "input_model", "q") when: - Try result = resultFactory.validate(foundFields, WecResult) + Try result = resultFactory.validate(actualFields, WecResult) then: result.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy index 1f0b9deaa..8b871b983 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactoryTest.groovy @@ -5,9 +5,7 @@ */ package edu.ie3.datamodel.io.factory.timeseries -import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.io.factory.SimpleFactoryData -import edu.ie3.datamodel.utils.Try import edu.ie3.util.geo.GeoUtils import org.apache.commons.lang3.tuple.Pair import org.locationtech.jts.geom.Point @@ -43,10 +41,10 @@ class CosmoIdCoordinateFactoryTest extends Specification { def "A COSMO id to coordinate factory refuses to build from invalid data"() { given: - def foundFields = factory.newSet("tid", "id", "latrot", "longrot") + def actualFields = CosmoIdCoordinateFactory.newSet("tid", "id", "latrot", "longrot") when: - def actual = factory.validate(foundFields, Pair) + def actual = factory.validate(actualFields, Pair) then: actual.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index e5e5e0b34..0fe4ee9c5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -41,10 +41,10 @@ class IconIdCoordinateFactoryTest extends Specification { def "A COSMO id to coordinate factory refuses to build from invalid data"() { given: - def foundFields = factory.newSet("id", "latitude", "coordinatetype") + def actualFields = IconIdCoordinateFactory.newSet("id", "latitude", "coordinatetype") when: - def actual = factory.validate(foundFields, Pair) + def actual = factory.validate(actualFields, Pair) then: actual.failure diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy index 0e5ba8071..77520cf32 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/typeinput/SystemParticipantTypeInputFactoryTest.groovy @@ -280,10 +280,10 @@ class SystemParticipantTypeInputFactoryTest extends Specification implements Fac def "A SystemParticipantTypeInputFactory should throw an exception on invalid or incomplete data"() { given: "a system participant factory and model data" def typeInputFactory = new SystemParticipantTypeInputFactory() - def foundFields = typeInputFactory.newSet("uuid", "id", "capex", "opex", "srated", "cosPhiRated", "estorage", "pmin", "pmax", "eta", "dod", "lifetime",) + def actualFields = SystemParticipantTypeInputFactory.newSet("uuid", "id", "capex", "opex", "srated", "cosPhiRated", "estorage", "pmin", "pmax", "eta", "dod", "lifetime",) when: - def input = typeInputFactory.validate(foundFields, StorageTypeInput) + def input = typeInputFactory.validate(actualFields, StorageTypeInput) then: input.failure From 721bbad8129a1eeb2f2b37c3a54672b5ed0273e3 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Fri, 5 Jan 2024 14:07:26 +0100 Subject: [PATCH 16/20] Refactoring validation handling Making column name retrieval throw exception on read failure + test --- .../exceptions/FailedValidationException.java | 4 ++ .../exceptions/ValidationException.java | 4 ++ .../io/connectors/CsvFileConnector.java | 26 ++++----- .../edu/ie3/datamodel/io/factory/Factory.java | 6 +- .../ie3/datamodel/io/source/DataSource.java | 4 +- .../io/source/EnergyManagementSource.java | 5 +- .../ie3/datamodel/io/source/EntitySource.java | 23 ++++++-- .../datamodel/io/source/GraphicSource.java | 13 ++++- .../io/source/IdCoordinateSource.java | 3 +- .../datamodel/io/source/RawGridSource.java | 23 ++++---- .../io/source/ResultEntitySource.java | 58 ++++++++++++------- .../datamodel/io/source/SourceValidator.java | 4 +- .../io/source/SystemParticipantSource.java | 31 +++++----- .../datamodel/io/source/ThermalSource.java | 17 +++--- .../ie3/datamodel/io/source/TypeSource.java | 41 ++++++++----- .../datamodel/io/source/WeatherSource.java | 2 +- .../couchbase/CouchbaseWeatherSource.java | 22 +++++-- .../io/source/csv/CsvDataSource.java | 24 +++++--- .../io/source/csv/CsvIdCoordinateSource.java | 20 +++---- .../csv/CsvJointGridContainerSource.java | 13 +++-- .../csv/CsvTimeSeriesMappingSource.java | 13 ++++- .../io/source/csv/CsvWeatherSource.java | 6 +- .../influxdb/InfluxDbWeatherSource.java | 17 +++++- .../io/source/sql/SqlDataSource.java | 24 ++------ .../io/source/sql/SqlIdCoordinateSource.java | 18 +++++- .../sql/SqlTimeSeriesMappingSource.java | 13 ++++- .../io/source/sql/SqlTimeSeriesSource.java | 22 +++++-- .../io/source/sql/SqlWeatherSource.java | 17 +++++- .../io/source/EntitySourceTest.groovy | 3 +- .../io/source/csv/CsvDataSourceTest.groovy | 50 +++++++++++++++- 30 files changed, 358 insertions(+), 168 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/FailedValidationException.java b/src/main/java/edu/ie3/datamodel/exceptions/FailedValidationException.java index 0fc60cd13..b1cc6386b 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/FailedValidationException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/FailedValidationException.java @@ -13,6 +13,10 @@ public FailedValidationException(String message, Throwable throwable) { super(message, throwable); } + public FailedValidationException(Throwable throwable) { + super(throwable); + } + public FailedValidationException(String message) { super(message); } diff --git a/src/main/java/edu/ie3/datamodel/exceptions/ValidationException.java b/src/main/java/edu/ie3/datamodel/exceptions/ValidationException.java index e74d70696..22eaa2ad0 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/ValidationException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/ValidationException.java @@ -10,6 +10,10 @@ protected ValidationException(String s) { super(s); } + protected ValidationException(Throwable throwable) { + super(throwable); + } + protected ValidationException(String s, Throwable throwable) { super(s, throwable); } diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 71147a7db..30fd3bdea 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -168,28 +168,22 @@ public synchronized void closeEntityWriter(Class clz */ public BufferedReader initReader(Class clz) throws FileNotFoundException, ConnectorException { - try { - Path filePath = - fileNamingStrategy - .getFilePath(clz) - .orElseThrow( - () -> - new ConnectorException( - "Cannot find a naming strategy for class '" - + clz.getSimpleName() - + "'.")); - return initReader(filePath); - } catch (ConnectorException e) { - throw new ConnectorException( - "Cannot initialize reader for entity '" + clz.getSimpleName() + "'.", e); - } + Path filePath = + fileNamingStrategy + .getFilePath(clz) + .orElseThrow( + () -> + new ConnectorException( + "Cannot find a naming strategy for class '" + clz.getSimpleName() + "'.")); + return initReader(filePath); } /** * Initializes a file reader for the given file name. Use {@link * CsvFileConnector#initReader(Class)} for files that actually correspond to concrete entities. * - * @param filePath sub directory tree starting from base folder, including file name + * @param filePath path of file starting from base folder, including file name but not file + * extension * @return the reader that contains information about the file to be read in * @throws FileNotFoundException if no file with the provided file name can be found */ diff --git a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java index a19fd0813..9daff0c2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/Factory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/Factory.java @@ -6,6 +6,8 @@ package edu.ie3.datamodel.io.factory; import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.source.SourceValidator; import edu.ie3.datamodel.utils.Try; import edu.ie3.datamodel.utils.Try.*; @@ -143,7 +145,7 @@ protected Set getUnusedFields( * @param entityClass of the build data * @return either an exception wrapped by a {@link Failure} or an empty success */ - public Try validate( + public Try validate( Set actualFields, Class entityClass) { List> fieldSets = getFields(entityClass); Set harmonizedActualFields = toCamelCase(actualFields); @@ -162,7 +164,7 @@ public Try validate( String possibleOptions = getFieldsString(fieldSets).toString(); return Failure.of( - new FactoryException( + new FailedValidationException( "The provided fields " + providedKeysString + " are invalid for instance of '" diff --git a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java index 7fb36eb8f..effc82935 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/DataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/DataSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.models.UniqueEntity; import java.util.*; import java.util.stream.Stream; @@ -18,7 +19,8 @@ public interface DataSource { * @param entityClass class of the source * @return an option for the found fields */ - Optional> getSourceFields(Class entityClass); + Optional> getSourceFields(Class entityClass) + throws SourceException; /** Creates a stream of maps that represent the rows in the database */ Stream> getSourceData(Class entityClass); diff --git a/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java b/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java index b0770517d..e3cd6de8a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EnergyManagementSource.java @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.input.participant.EmInputFactory; import edu.ie3.datamodel.models.input.OperatorInput; import edu.ie3.datamodel.models.input.container.EnergyManagementUnits; @@ -27,8 +28,8 @@ public EnergyManagementSource(TypeSource typeSource, DataSource dataSource) { } @Override - public void validate() { - validate(EmInput.class, emInputFactory); + public void validate() throws ValidationException { + validate(EmInput.class, emInputFactory).getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index 71373bbfc..cf42a6f7e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -6,7 +6,9 @@ package edu.ie3.datamodel.io.source; import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.EntityFactory; import edu.ie3.datamodel.io.factory.SimpleEntityData; import edu.ie3.datamodel.io.factory.input.AssetInputEntityData; @@ -39,7 +41,7 @@ public abstract class EntitySource { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= - public abstract void validate(); + public abstract void validate() throws ValidationException; /** * Method for validating a single source. @@ -48,11 +50,22 @@ public abstract class EntitySource { * @param validator used to validate * @param type of the class */ - protected final void validate( + protected final Try validate( Class entityClass, SourceValidator validator) { - dataSource - .getSourceFields(entityClass) - .ifPresent(s -> validator.validate(s, entityClass).getOrThrow()); + return Try.of(() -> dataSource.getSourceFields(entityClass), SourceException.class) + .transformF( + se -> + (ValidationException) + new FailedValidationException( + "Validation for entity " + + entityClass + + " failed because of an error related to its source.", + se)) + .flatMap( + fieldsOpt -> + fieldsOpt + .map(fields -> validator.validate(fields, entityClass)) + .orElse(Try.Success.empty())); } protected String buildSkippingMessage( diff --git a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java index 4d8a577d5..8f24b3e2b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/GraphicSource.java @@ -5,8 +5,10 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.FailedValidationException; import edu.ie3.datamodel.exceptions.GraphicSourceException; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.input.graphics.LineGraphicInputEntityData; import edu.ie3.datamodel.io.factory.input.graphics.LineGraphicInputFactory; import edu.ie3.datamodel.io.factory.input.graphics.NodeGraphicInputEntityData; @@ -51,9 +53,14 @@ public GraphicSource(TypeSource typeSource, RawGridSource rawGridSource, DataSou } @Override - public void validate() { - validate(NodeGraphicInput.class, nodeGraphicInputFactory); - validate(LineGraphicInput.class, lineGraphicInputFactory); + public void validate() throws ValidationException { + Try.scanStream( + Stream.of( + validate(NodeGraphicInput.class, nodeGraphicInputFactory), + validate(LineGraphicInput.class, lineGraphicInputFactory)), + "Validation") + .transformF(FailedValidationException::new) + .getOrThrow(); } /** Returns the graphic elements of the grid or throws a {@link SourceException} */ diff --git a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java index f9b3768db..755a33875 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.util.geo.CoordinateDistance; import edu.ie3.util.geo.GeoUtils; import java.util.*; @@ -25,7 +26,7 @@ public interface IdCoordinateSource { * @param entityClass class of the source * @return an option for the found fields */ - Optional> getSourceFields(Class entityClass); + Optional> getSourceFields(Class entityClass) throws SourceException; /** * Get the matching coordinate for the given ID diff --git a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java index a32ac1902..04f254b26 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/RawGridSource.java @@ -5,9 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; -import edu.ie3.datamodel.exceptions.RawGridException; -import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.factory.EntityFactory; import edu.ie3.datamodel.io.factory.input.*; import edu.ie3.datamodel.models.input.*; @@ -63,13 +61,18 @@ public RawGridSource(TypeSource typeSource, DataSource dataSource) { } @Override - public void validate() { - validate(NodeInput.class, nodeInputFactory); - validate(LineInput.class, lineInputFactory); - validate(Transformer2WInput.class, transformer2WInputFactory); - validate(Transformer3WInput.class, transformer3WInputFactory); - validate(SwitchInput.class, switchInputFactory); - validate(MeasurementUnitInput.class, measurementUnitInputFactory); + public void validate() throws ValidationException { + Try.scanStream( + Stream.of( + validate(NodeInput.class, nodeInputFactory), + validate(LineInput.class, lineInputFactory), + validate(Transformer2WInput.class, transformer2WInputFactory), + validate(Transformer3WInput.class, transformer3WInputFactory), + validate(SwitchInput.class, switchInputFactory), + validate(MeasurementUnitInput.class, measurementUnitInputFactory)), + "Validation") + .transformF(FailedValidationException::new) + .getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java index b77feb5cc..ca54c0d49 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ResultEntitySource.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.SimpleEntityFactory; import edu.ie3.datamodel.io.factory.result.*; import edu.ie3.datamodel.models.result.NodeResult; @@ -16,10 +18,13 @@ import edu.ie3.datamodel.models.result.system.*; import edu.ie3.datamodel.models.result.thermal.CylindricalStorageResult; import edu.ie3.datamodel.models.result.thermal.ThermalHouseResult; +import edu.ie3.datamodel.utils.Try; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link ResultEntity} container @@ -62,29 +67,38 @@ public ResultEntitySource(DataSource dataSource, String dtfPattern) { } @Override - public void validate() { - List.of( - LoadResult.class, - FixedFeedInResult.class, - BmResult.class, - PvResult.class, - ChpResult.class, - WecResult.class, - StorageResult.class, - EvcsResult.class, - EvResult.class, - HpResult.class, - EmResult.class) - .forEach(c -> validate(c, systemParticipantResultFactory)); + public void validate() throws ValidationException { + List> participantResults = + new ArrayList<>( + Stream.of( + LoadResult.class, + FixedFeedInResult.class, + BmResult.class, + PvResult.class, + ChpResult.class, + WecResult.class, + StorageResult.class, + EvcsResult.class, + EvResult.class, + HpResult.class, + EmResult.class) + .map(c -> validate(c, systemParticipantResultFactory)) + .toList()); - validate(ThermalHouseResult.class, thermalResultFactory); - validate(CylindricalStorageResult.class, thermalResultFactory); - validate(SwitchResult.class, switchResultFactory); - validate(NodeResult.class, nodeResultFactory); - validate(LineResult.class, connectorResultFactory); - validate(Transformer2WResult.class, connectorResultFactory); - validate(Transformer3WResult.class, connectorResultFactory); - validate(FlexOptionsResult.class, flexOptionsResultFactory); + participantResults.addAll( + List.of( + validate(ThermalHouseResult.class, thermalResultFactory), + validate(CylindricalStorageResult.class, thermalResultFactory), + validate(SwitchResult.class, switchResultFactory), + validate(NodeResult.class, nodeResultFactory), + validate(LineResult.class, connectorResultFactory), + validate(Transformer2WResult.class, connectorResultFactory), + validate(Transformer3WResult.class, connectorResultFactory), + validate(FlexOptionsResult.class, flexOptionsResultFactory))); + + Try.scanCollection(participantResults, Void.class) + .transformF(FailedValidationException::new) + .getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java index 650fdf2b8..e1c43c89f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SourceValidator.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.utils.Try; import edu.ie3.datamodel.utils.Try.Failure; import java.util.Set; @@ -19,5 +19,5 @@ public interface SourceValidator { * @param entityClass that should be buildable from the source data * @return either an exception wrapped by a {@link Failure} or an empty success */ - Try validate(Set actualFields, Class entityClass); + Try validate(Set actualFields, Class entityClass); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java index a29a45b1c..e7d69ac3d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/SystemParticipantSource.java @@ -5,9 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; -import edu.ie3.datamodel.exceptions.SourceException; -import edu.ie3.datamodel.exceptions.SystemParticipantsException; +import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.factory.EntityFactory; import edu.ie3.datamodel.io.factory.input.NodeAssetInputEntityData; import edu.ie3.datamodel.io.factory.input.participant.*; @@ -76,17 +74,22 @@ public SystemParticipantSource( } @Override - public void validate() { - validate(BmInput.class, bmInputFactory); - validate(ChpInput.class, chpInputFactory); - validate(EvInput.class, evInputFactory); - validate(FixedFeedInInput.class, fixedFeedInInputFactory); - validate(HpInput.class, hpInputFactory); - validate(LoadInput.class, loadInputFactory); - validate(PvInput.class, pvInputFactory); - validate(StorageInput.class, storageInputFactory); - validate(WecInput.class, wecInputFactory); - validate(EvcsInput.class, evcsInputFactory); + public void validate() throws ValidationException { + Try.scanStream( + Stream.of( + validate(BmInput.class, bmInputFactory), + validate(ChpInput.class, chpInputFactory), + validate(EvInput.class, evInputFactory), + validate(FixedFeedInInput.class, fixedFeedInInputFactory), + validate(HpInput.class, hpInputFactory), + validate(LoadInput.class, loadInputFactory), + validate(PvInput.class, pvInputFactory), + validate(StorageInput.class, storageInputFactory), + validate(WecInput.class, wecInputFactory), + validate(EvcsInput.class, evcsInputFactory)), + "Validation") + .transformF(FailedValidationException::new) + .getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java index 385629564..c8e9cd77b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/ThermalSource.java @@ -5,9 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.FactoryException; -import edu.ie3.datamodel.exceptions.FailureException; -import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.factory.input.*; import edu.ie3.datamodel.models.input.OperatorInput; import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput; @@ -47,10 +45,15 @@ public ThermalSource(TypeSource typeSource, DataSource dataSource) { } @Override - public void validate() { - validate(ThermalBusInput.class, thermalBusInputFactory); - validate(CylindricalStorageInput.class, cylindricalStorageInputFactory); - validate(ThermalHouseInput.class, thermalHouseInputFactory); + public void validate() throws ValidationException { + Try.scanStream( + Stream.of( + validate(ThermalBusInput.class, thermalBusInputFactory), + validate(CylindricalStorageInput.class, cylindricalStorageInputFactory), + validate(ThermalHouseInput.class, thermalHouseInputFactory)), + "Validation") + .transformF(FailedValidationException::new) + .getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java index b8420c364..e0c8b2d96 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/TypeSource.java @@ -5,7 +5,9 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.FailedValidationException; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.input.OperatorInputFactory; import edu.ie3.datamodel.io.factory.typeinput.LineTypeInputFactory; import edu.ie3.datamodel.io.factory.typeinput.SystemParticipantTypeInputFactory; @@ -17,8 +19,10 @@ import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput; import edu.ie3.datamodel.models.input.system.type.*; import edu.ie3.datamodel.utils.Try; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Stream; /** * Interface that provides the capability to build entities of type {@link @@ -47,20 +51,29 @@ public TypeSource(DataSource dataSource) { } @Override - public void validate() { - validate(OperatorInput.class, operatorInputFactory); - validate(LineTypeInput.class, lineTypeInputFactory); - validate(Transformer2WTypeInput.class, transformer2WTypeInputFactory); - validate(Transformer3WTypeInput.class, transformer3WTypeInputFactory); - - List.of( - EvTypeInput.class, - HpTypeInput.class, - BmTypeInput.class, - WecTypeInput.class, - ChpTypeInput.class, - StorageTypeInput.class) - .forEach(c -> validate(c, systemParticipantTypeInputFactory)); + public void validate() throws ValidationException { + List> participantResults = + new ArrayList<>( + Stream.of( + EvTypeInput.class, + HpTypeInput.class, + BmTypeInput.class, + WecTypeInput.class, + ChpTypeInput.class, + StorageTypeInput.class) + .map(c -> validate(c, systemParticipantTypeInputFactory)) + .toList()); + + participantResults.addAll( + List.of( + validate(OperatorInput.class, operatorInputFactory), + validate(LineTypeInput.class, lineTypeInputFactory), + validate(Transformer2WTypeInput.class, transformer2WTypeInputFactory), + validate(Transformer3WTypeInput.class, transformer3WTypeInputFactory))); + + Try.scanCollection(participantResults, Void.class) + .transformF(FailedValidationException::new) + .getOrThrow(); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index bb579a90c..990c43db7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -47,7 +47,7 @@ protected WeatherSource( * @return an option for fields found in the source */ public abstract Optional> getSourceFields( - Class entityClass); + Class entityClass) throws SourceException; // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 2c90d04f8..5f48a9666 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -10,6 +10,7 @@ import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.query.QueryResult; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CouchbaseConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -18,6 +19,7 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; +import edu.ie3.datamodel.utils.Try; import edu.ie3.util.interval.ClosedInterval; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -59,7 +61,8 @@ public CouchbaseWeatherSource( IdCoordinateSource coordinateSource, String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) { + String timeStampPattern) + throws SourceException { this( connector, coordinateSource, @@ -88,15 +91,26 @@ public CouchbaseWeatherSource( String coordinateIdColumnName, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) { + String timeStampPattern) + throws SourceException { super(idCoordinateSource, weatherFactory); this.connector = connector; this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.timeStampPattern = timeStampPattern; - getSourceFields(WeatherValue.class) - .ifPresent(s -> weatherFactory.validate(s, WeatherValue.class).getOrThrow()); + // validating + Try.of(() -> getSourceFields(WeatherValue.class), SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + weatherFactory + .validate(fields, WeatherValue.class) + .transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 24264e058..ebe351eda 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -59,19 +59,29 @@ public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNami } @Override - public Optional> getSourceFields(Class entityClass) { - try (BufferedReader reader = connector.initReader(entityClass)) { + public Optional> getSourceFields(Class entityClass) + throws SourceException { + return getSourceFields(() -> connector.initReader(entityClass)); + } + + public Optional> getSourceFields(ReaderSupplier readerSupplier) + throws SourceException { + try (BufferedReader reader = readerSupplier.get()) { return Optional.of( Arrays.stream(parseCsvRow(reader.readLine(), csvSep)).collect(Collectors.toSet())); - } catch (ConnectorException | IOException e) { - log.warn( - "The source for the entity '{}' couldn't be read and therefore not be validated! Cause: {}", - entityClass, - e.getMessage()); + } catch (FileNotFoundException e) { + // A file not existing can be acceptable in many cases, and is handled elsewhere. + log.debug("The source for the given entity couldn't be found! Cause: {}", e.getMessage()); return Optional.empty(); + } catch (ConnectorException | IOException e) { + throw new SourceException("Error while trying to read source", e); } } + public interface ReaderSupplier { + BufferedReader get() throws FileNotFoundException, ConnectorException; + } + @Override public Stream> getSourceData(Class entityClass) { return buildStreamWithFieldsToAttributesMap(entityClass, connector); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java index 05db85a0a..69609e641 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source.csv; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.SimpleFactoryData; import edu.ie3.datamodel.io.factory.timeseries.IdCoordinateFactory; import edu.ie3.datamodel.io.source.IdCoordinateSource; @@ -87,18 +88,8 @@ private Map invert(Map map) { } @Override - public Optional> getSourceFields(Class entityClass) { - try (BufferedReader reader = dataSource.connector.initIdCoordinateReader()) { - return Optional.of( - Arrays.stream(dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep)) - .collect(Collectors.toSet())); - } catch (IOException e) { - log.warn( - "The source for the entity '{}' couldn't be read and therefore not be validated!", - entityClass, - e); - return Optional.empty(); - } + public Optional> getSourceFields(Class entityClass) throws SourceException { + return dataSource.getSourceFields(dataSource.connector::initIdCoordinateReader); } @Override @@ -173,7 +164,8 @@ public int getCoordinateCount() { * * @return Stream with mappings from field identifiers to attributes */ - protected Stream> buildStreamWithFieldsToAttributesMap() { + protected Stream> buildStreamWithFieldsToAttributesMap() + throws SourceException { try (BufferedReader reader = dataSource.connector.initIdCoordinateReader()) { final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); @@ -203,6 +195,8 @@ protected Stream> buildStreamWithFieldsToAttributesMap() { .parallelStream(); } catch (IOException e) { log.error("Cannot read the file for coordinate id to coordinate mapping.", e); + } catch (ValidationException ve) { + throw new SourceException("Creating stream failed due to failed validation", ve); } return Stream.empty(); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java index 013c3a9aa..0eba675f5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvJointGridContainerSource.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.exceptions.FileException; import edu.ie3.datamodel.exceptions.InvalidGridException; import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.naming.DefaultDirectoryHierarchy; import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; import edu.ie3.datamodel.io.naming.FileNamingStrategy; @@ -51,10 +52,14 @@ public static JointGridContainer read( GraphicSource graphicSource = new GraphicSource(typeSource, rawGridSource, dataSource); /* validating sources */ - typeSource.validate(); - rawGridSource.validate(); - systemParticipantSource.validate(); - graphicSource.validate(); + try { + typeSource.validate(); + rawGridSource.validate(); + systemParticipantSource.validate(); + graphicSource.validate(); + } catch (ValidationException ve) { + throw new SourceException("Could not read source because validation failed", ve); + } /* Loading models */ Try rawGridElements = diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index 7d4244428..ee165ea2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; +import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; import java.util.Map; import java.util.Optional; @@ -24,7 +25,17 @@ public CsvTimeSeriesMappingSource( this.dataSource = new CsvDataSource(csvSep, gridFolderPath, fileNamingStrategy); // validating - getSourceFields().ifPresent(s -> mappingFactory.validate(s, MappingEntry.class).getOrThrow()); + Try.of(this::getSourceFields, SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + mappingFactory + .validate(fields, MappingEntry.class) + .transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index a246ce024..d3b98c7ee 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.csv; +import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -175,13 +176,16 @@ private Map> readWeatherTimeSeries( log.error("Cannot read file {}. File not found!", data.getFullFilePath()); } catch (IOException e) { log.error("Cannot read file {}. Exception: {}", data.getFullFilePath(), e); + } catch (ValidationException ve) { + log.error("Validation failed for file {}. Exception: {}", data.getFullFilePath(), ve); } } return weatherTimeSeries; } private Stream> buildStreamWithFieldsToAttributesMap( - Class entityClass, BufferedReader bufferedReader) { + Class entityClass, BufferedReader bufferedReader) + throws ValidationException { try (BufferedReader reader = bufferedReader) { final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep); diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 4822038f8..c35227155 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -47,12 +48,22 @@ public class InfluxDbWeatherSource extends WeatherSource { public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource idCoordinateSource, - TimeBasedWeatherValueFactory weatherValueFactory) { + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { super(idCoordinateSource, weatherValueFactory); this.connector = connector; - getSourceFields(WeatherValue.class) - .ifPresent(s -> weatherValueFactory.validate(s, WeatherValue.class).getOrThrow()); + Try.of(() -> getSourceFields(WeatherValue.class), SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + weatherFactory + .validate(fields, WeatherValue.class) + .transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index fd5ed3e18..237b09a36 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -110,25 +110,8 @@ protected List getDbTables(String schemaPattern, String tableNamePattern @Override public Optional> getSourceFields(Class entityClass) { - try { - String tableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); - ResultSet rs = - connector.getConnection().getMetaData().getColumns(null, null, tableName, null); - Set columnNames = new HashSet<>(); - - while (rs.next()) { - String name = rs.getString("COLUMN_NAME"); - columnNames.add(StringUtils.snakeCaseToCamelCase(name)); - } - - return Optional.of(columnNames); - } catch (NoSuchElementException | SQLException e) { - log.warn( - "The source for the entity '{}' couldn't be read and therefore not be validated! Cause: {}", - entityClass, - e.getMessage()); - return Optional.empty(); - } + String tableName = databaseNamingStrategy.getEntityName(entityClass).orElseThrow(); + return getSourceFields(tableName); } /** @@ -149,8 +132,9 @@ public Optional> getSourceFields(String tableName) { } return Optional.of(columnNames); - } catch (NoSuchElementException | SQLException e) { + } catch (SQLException e) { log.warn("The table '{}' couldn't be read and therefore not be validated!", tableName, e); + // FIXME only return empty if table not found. Throw exception if error occurred return Optional.empty(); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java index d4a1b5255..8ec23b01f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java @@ -7,12 +7,14 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.SimpleFactoryData; import edu.ie3.datamodel.io.factory.timeseries.SqlIdCoordinateFactory; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.source.IdCoordinateSource; import edu.ie3.datamodel.models.value.CoordinateValue; +import edu.ie3.datamodel.utils.Try; import edu.ie3.util.geo.CoordinateDistance; import edu.ie3.util.geo.GeoUtils; import java.sql.Array; @@ -46,7 +48,8 @@ public class SqlIdCoordinateSource implements IdCoordinateSource { private final SqlIdCoordinateFactory factory; public SqlIdCoordinateSource( - SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) { + SqlIdCoordinateFactory factory, String coordinateTableName, SqlDataSource dataSource) + throws SourceException { this.factory = factory; this.dataSource = dataSource; this.coordinateTableName = coordinateTableName; @@ -56,7 +59,15 @@ public SqlIdCoordinateSource( dataSource.getDbColumnName(factory.getCoordinateField(), coordinateTableName); // validating table - getSourceFields(Pair.class).ifPresent(s -> factory.validate(s, Pair.class).getOrThrow()); + Try.of(() -> getSourceFields(Pair.class), SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + factory.validate(fields, Pair.class).transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); // setup queries this.basicQuery = createBaseQueryString(dataSource.schemaName, coordinateTableName); @@ -81,7 +92,8 @@ public SqlIdCoordinateSource( SqlConnector connector, String schemaName, String coordinateTableName, - SqlIdCoordinateFactory factory) { + SqlIdCoordinateFactory factory) + throws SourceException { this( factory, coordinateTableName, diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java index b5f817058..6b37bd173 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java @@ -12,6 +12,7 @@ import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; +import edu.ie3.datamodel.utils.Try; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -37,7 +38,17 @@ public SqlTimeSeriesMappingSource( entityPersistenceNamingStrategy.getEntityName(MappingEntry.class).orElseThrow(); this.queryFull = createBaseQueryString(schemaName, tableName); - getSourceFields().ifPresent(s -> mappingFactory.validate(s, MappingEntry.class).getOrThrow()); + Try.of(this::getSourceFields, SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + mappingFactory + .validate(fields, MappingEntry.class) + .transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java index 8cbffccc5..f3c029aba 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesSource.java @@ -18,6 +18,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.utils.TimeSeriesUtils; +import edu.ie3.datamodel.utils.Try; import edu.ie3.util.interval.ClosedInterval; import java.sql.Timestamp; import java.time.ZonedDateTime; @@ -50,7 +51,8 @@ public SqlTimeSeriesSource( SqlDataSource sqlDataSource, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) { + TimeBasedSimpleValueFactory factory) + throws SourceException { super(valueClass, factory); this.dataSource = sqlDataSource; @@ -63,9 +65,15 @@ public SqlTimeSeriesSource( final String tableName = sqlDataSource.databaseNamingStrategy.getTimeSeriesEntityName(columnScheme); - dataSource - .getSourceFields(tableName) - .ifPresent(s -> factory.validate(s, valueClass).getOrThrow()); + Try.of(() -> dataSource.getSourceFields(tableName), SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + factory.validate(fields, valueClass).transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); String dbTimeColumnName = sqlDataSource.getDbColumnName(factory.getTimeFieldString(), tableName); @@ -92,7 +100,8 @@ public SqlTimeSeriesSource( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valueClass, - TimeBasedSimpleValueFactory factory) { + TimeBasedSimpleValueFactory factory) + throws SourceException { this( new SqlDataSource(connector, schemaName, namingStrategy), timeSeriesUuid, @@ -134,7 +143,8 @@ private static SqlTimeSeriesSource create( DatabaseNamingStrategy namingStrategy, UUID timeSeriesUuid, Class valClass, - String timePattern) { + String timePattern) + throws SourceException { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass, timePattern); return new SqlTimeSeriesSource<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 6e69d808a..37ac91c2e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -16,6 +16,7 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; +import edu.ie3.datamodel.utils.Try; import edu.ie3.util.interval.ClosedInterval; import java.sql.*; import java.time.ZonedDateTime; @@ -55,14 +56,24 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, - TimeBasedWeatherValueFactory weatherFactory) { + TimeBasedWeatherValueFactory weatherFactory) + throws SourceException { super(idCoordinateSource, weatherFactory); this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); this.dataSource = new SqlDataSource(connector, schemaName, new DatabaseNamingStrategy()); this.tableName = weatherTableName; - getSourceFields(WeatherValue.class) - .ifPresent(s -> weatherFactory.validate(s, WeatherValue.class).getOrThrow()); + Try.of(() -> getSourceFields(WeatherValue.class), SourceException.class) + .flatMap( + fieldsOpt -> + fieldsOpt + .map( + fields -> + weatherFactory + .validate(fields, WeatherValue.class) + .transformF(SourceException::new)) + .orElse(Try.Success.empty())) + .getOrThrow(); String dbTimeColumnName = dataSource.getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName); diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy index 507d9fb0e..879d3b29f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source import edu.ie3.datamodel.exceptions.FactoryException +import edu.ie3.datamodel.exceptions.ValidationException import edu.ie3.datamodel.io.factory.input.ThermalBusInputFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.csv.CsvDataSource @@ -26,7 +27,7 @@ class EntitySourceTest extends Specification { } @Override - void validate() { + void validate() throws ValidationException { } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index d79c0e9d5..21dccdaa2 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -5,9 +5,11 @@ */ package edu.ie3.datamodel.io.source.csv +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.models.UniqueEntity import edu.ie3.datamodel.models.input.NodeInput +import edu.ie3.datamodel.models.input.system.LoadInput import edu.ie3.test.common.SystemParticipantTestData as sptd import spock.lang.Shared import spock.lang.Specification @@ -18,7 +20,7 @@ import java.util.concurrent.atomic.LongAdder import java.util.function.Function import java.util.stream.Collectors -class CsvDataSourceTest extends Specification { +class CsvDataSourceTest extends Specification implements CsvTestDataMeta { @Shared Function, String> uuidExtractor = { fieldToValues -> fieldToValues.get("uuid") } @@ -71,6 +73,52 @@ class CsvDataSourceTest extends Specification { dummyCsvSource.connector.entityWriters.isEmpty() } + def "A CsvDataSource should return column names from a valid CSV file as expected"() { + given: + DummyCsvSource source = new DummyCsvSource(csvSep, participantsFolderPath, fileNamingStrategy) + + expect: + source.getSourceFields(LoadInput).get() == [ + "operates_from", + "node", + "s_rated", + "cos_phi_rated", + "load_profile", + "id", + "operates_until", + "uuid", + "q_characteristics", + "e_cons_annual", + "operator", + "dsm" + ] as Set + } + + def "A CsvDataSource should return an empty result when retrieving column names for a non-existing CSV file"() { + given: + def path = Path.of("this/path/does-not-exist") + + expect: + dummyCsvSource.getSourceFields(() -> dummyCsvSource.connector.initReader(path)).isEmpty() + } + + def "A CsvDataSource should throw an exception when retrieving column names for a non-readable CSV file"() { + given: + DummyCsvSource source = new DummyCsvSource(csvSep, participantsFolderPath, fileNamingStrategy) + def readerSupplier = () -> { + def reader = source.connector.initReader(LoadInput) + reader.close() // We simulate the file being unreadable by just closing the reader before passing it over + return reader + } + + when: + source.getSourceFields(readerSupplier) + + then: + def exc = thrown(SourceException) + exc.message.startsWith("Error while trying to read source") + } + def "A CsvDataSource should build a valid fields to attributes map with valid data as expected"() { given: def validHeadline = [ From 80104d087b669a2d37c003502b225242619939df Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Fri, 5 Jan 2024 14:09:58 +0100 Subject: [PATCH 17/20] Rolling back changes that will be fixed with #981 --- .../edu/ie3/datamodel/io/source/EntitySource.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java index cf42a6f7e..391aaf4a8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/EntitySource.java @@ -335,6 +335,13 @@ protected Stream simpleEntityDataStre .map(fieldsToAttributes -> new SimpleEntityData(fieldsToAttributes, entityClass)); } + protected Stream> assetInputEntityStream( + Class entityClass, + EntityFactory factory, + Collection operators) { + return assetInputEntityDataStream(entityClass, operators).map(factory::get); + } + /** * Returns a stream of {@link Try} entities that can be build by using {@link * NodeAssetInputEntityData} and their corresponding factory. @@ -372,9 +379,7 @@ public Set> buildAssetInputEntit Class entityClass, EntityFactory factory, Collection operators) { - return assetInputEntityDataStream(entityClass, operators) - .map(factory::get) - .collect(Collectors.toSet()); + return assetInputEntityStream(entityClass, factory, operators).collect(Collectors.toSet()); } @SuppressWarnings("unchecked") From e1d5e5195b2c9ba27b65340559dab13b8fee0508 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Fri, 5 Jan 2024 14:25:22 +0100 Subject: [PATCH 18/20] Removing unused methods from Try again --- .../java/edu/ie3/datamodel/utils/Try.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/utils/Try.java b/src/main/java/edu/ie3/datamodel/utils/Try.java index c3b348b0f..62b59f20a 100644 --- a/src/main/java/edu/ie3/datamodel/utils/Try.java +++ b/src/main/java/edu/ie3/datamodel/utils/Try.java @@ -10,7 +10,6 @@ import edu.ie3.datamodel.exceptions.FailureException; import edu.ie3.datamodel.exceptions.TryException; import java.util.*; -import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -193,20 +192,6 @@ public static Try, FailureException> scanStre // functional methods - /** - * Method to consume the contained data. - * - * @param consumer operation to compute - */ - public abstract void ifSuccess(Consumer consumer); - - /** - * Method to consume the exception - * - * @param consumer operation to compute - */ - public abstract void ifFailure(Consumer consumer); - /** * Method to transform the data if this object is a {@link Success}. * @@ -300,16 +285,6 @@ public Optional getException() { return Optional.empty(); } - @Override - public void ifSuccess(Consumer consumer) { - consumer.accept(data); - } - - @Override - public void ifFailure(Consumer consumer) { - // does nothing - } - @SuppressWarnings("unchecked") @Override public Try flatMap(Function> mapper) { @@ -395,16 +370,6 @@ public Optional getException() { return exception != null ? Optional.of(exception) : Optional.empty(); } - @Override - public void ifSuccess(Consumer consumer) { - // does nothing - } - - @Override - public void ifFailure(Consumer consumer) { - consumer.accept(exception); - } - @SuppressWarnings("unchecked") @Override public Try flatMap(Function> mapper) { From 13f034b61934e74ba60ac908f23bdc64a8962c3c Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Fri, 5 Jan 2024 14:27:55 +0100 Subject: [PATCH 19/20] Fixing codacy issues --- src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy | 2 +- .../io/factory/timeseries/IconIdCoordinateFactoryTest.groovy | 2 -- .../groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy index 38cb82ac7..33f657aed 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy @@ -12,7 +12,7 @@ import spock.lang.Specification class FactoryTest extends Specification { @Shared - private DummyFactory factory = new DummyFactory(String) + private final DummyFactory factory = new DummyFactory(String) def "A Factory can return unused fields correctly"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy index 0fe4ee9c5..922c82869 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactoryTest.groovy @@ -5,9 +5,7 @@ */ package edu.ie3.datamodel.io.factory.timeseries -import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.io.factory.SimpleFactoryData -import edu.ie3.datamodel.utils.Try import edu.ie3.util.geo.GeoUtils import org.apache.commons.lang3.tuple.Pair import org.locationtech.jts.geom.Point diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy index 879d3b29f..75b1257aa 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/EntitySourceTest.groovy @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source -import edu.ie3.datamodel.exceptions.FactoryException import edu.ie3.datamodel.exceptions.ValidationException import edu.ie3.datamodel.io.factory.input.ThermalBusInputFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy From 440a4267385db09e7403cbea7c0b131b16cf0d52 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Fri, 5 Jan 2024 14:36:21 +0100 Subject: [PATCH 20/20] Adapting documentation to changed spelling of cosPhiRated --- docs/readthedocs/models/input/participant/bm.rst | 2 +- docs/readthedocs/models/input/participant/chp.rst | 2 +- docs/readthedocs/models/input/participant/ev.rst | 2 +- docs/readthedocs/models/input/participant/fixedfeedin.rst | 2 +- docs/readthedocs/models/input/participant/hp.rst | 2 +- docs/readthedocs/models/input/participant/load.rst | 2 +- docs/readthedocs/models/input/participant/pv.rst | 2 +- docs/readthedocs/models/input/participant/storage.rst | 2 +- docs/readthedocs/models/input/participant/wec.rst | 2 +- docs/uml/main/input/SystemDatamodelConcept.puml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/readthedocs/models/input/participant/bm.rst b/docs/readthedocs/models/input/participant/bm.rst index 5314251e4..1e81804e5 100644 --- a/docs/readthedocs/models/input/participant/bm.rst +++ b/docs/readthedocs/models/input/participant/bm.rst @@ -26,7 +26,7 @@ Type Model +---------------------+---------+------------------------------------------------------------+ | sRated | kVA | Rated apparent power | +---------------------+---------+------------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +---------------------+---------+------------------------------------------------------------+ | etaConv | % | Efficiency of the assets inverter | +---------------------+---------+------------------------------------------------------------+ diff --git a/docs/readthedocs/models/input/participant/chp.rst b/docs/readthedocs/models/input/participant/chp.rst index 14cd2fd46..f933b9423 100644 --- a/docs/readthedocs/models/input/participant/chp.rst +++ b/docs/readthedocs/models/input/participant/chp.rst @@ -28,7 +28,7 @@ Type Model +-------------+---------+---------------------------------------------------------+ | sRated | kVA | Rated apparent power | +-------------+---------+---------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +-------------+---------+---------------------------------------------------------+ | pThermal | kW | Rated thermal power (at rated electrical power) | +-------------+---------+---------------------------------------------------------+ diff --git a/docs/readthedocs/models/input/participant/ev.rst b/docs/readthedocs/models/input/participant/ev.rst index cd98d6d86..2d06c5501 100644 --- a/docs/readthedocs/models/input/participant/ev.rst +++ b/docs/readthedocs/models/input/participant/ev.rst @@ -27,7 +27,7 @@ Type Model +-------------+----------+---------------------------------------------------------+ | sRated | kVA | Rated apparent power | +-------------+----------+---------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +-------------+----------+---------------------------------------------------------+ Entity Model diff --git a/docs/readthedocs/models/input/participant/fixedfeedin.rst b/docs/readthedocs/models/input/participant/fixedfeedin.rst index f317d5534..d20fc6624 100644 --- a/docs/readthedocs/models/input/participant/fixedfeedin.rst +++ b/docs/readthedocs/models/input/participant/fixedfeedin.rst @@ -25,7 +25,7 @@ Attributes, Units and Remarks +------------------+---------+--------------------------------------------------------------------------------------+ | sRated | kVA | Rated apparent power | +------------------+---------+--------------------------------------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +------------------+---------+--------------------------------------------------------------------------------------+ Caveats diff --git a/docs/readthedocs/models/input/participant/hp.rst b/docs/readthedocs/models/input/participant/hp.rst index e28cc3c52..21a73551c 100644 --- a/docs/readthedocs/models/input/participant/hp.rst +++ b/docs/readthedocs/models/input/participant/hp.rst @@ -24,7 +24,7 @@ Type Model +-------------+---------+---------------------------------------------------------+ | sRated | kVA | Rated apparent power | +-------------+---------+---------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +-------------+---------+---------------------------------------------------------+ | pThermal | kW | Rated thermal power (at rated electrical power) | +-------------+---------+---------------------------------------------------------+ diff --git a/docs/readthedocs/models/input/participant/load.rst b/docs/readthedocs/models/input/participant/load.rst index f2aa5aa23..802a20d9b 100644 --- a/docs/readthedocs/models/input/participant/load.rst +++ b/docs/readthedocs/models/input/participant/load.rst @@ -30,7 +30,7 @@ Attributes, Units and Remarks +---------------------+---------+--------------------------------------------------------------------------------------+ | sRated | kVA | Rated apparent power | +---------------------+---------+--------------------------------------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +---------------------+---------+--------------------------------------------------------------------------------------+ Caveats diff --git a/docs/readthedocs/models/input/participant/pv.rst b/docs/readthedocs/models/input/participant/pv.rst index 3308f9c6b..b548a3634 100644 --- a/docs/readthedocs/models/input/participant/pv.rst +++ b/docs/readthedocs/models/input/participant/pv.rst @@ -37,7 +37,7 @@ Detailed model of a photovoltaic power plant. +------------------+---------+--------------------------------------------------------------------------------------+ | sRated | kVA | Rated apparent power | +------------------+---------+--------------------------------------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +------------------+---------+--------------------------------------------------------------------------------------+ Caveats diff --git a/docs/readthedocs/models/input/participant/storage.rst b/docs/readthedocs/models/input/participant/storage.rst index 19d768a30..30b6e4dde 100644 --- a/docs/readthedocs/models/input/participant/storage.rst +++ b/docs/readthedocs/models/input/participant/storage.rst @@ -26,7 +26,7 @@ Type Model +---------------------+---------+---------------------------------------------------------+ | sRated | kVA | Rated apparent power | +---------------------+---------+---------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +---------------------+---------+---------------------------------------------------------+ | pMax | kW | | Maximum permissible active power | | | | | infeed or consumption | diff --git a/docs/readthedocs/models/input/participant/wec.rst b/docs/readthedocs/models/input/participant/wec.rst index d0769d92e..2c3346842 100644 --- a/docs/readthedocs/models/input/participant/wec.rst +++ b/docs/readthedocs/models/input/participant/wec.rst @@ -24,7 +24,7 @@ Type Model +------------------+---------+----------------------------------------------------------------------+ | sRated | kVA | Rated apparent power | +------------------+---------+----------------------------------------------------------------------+ -| cosphiRated | -- | Rated power factor | +| cosPhiRated | -- | Rated power factor | +------------------+---------+----------------------------------------------------------------------+ | cpCharacteristic | -- | Wind velocity dependent :ref:`Betz factors`. | +------------------+---------+----------------------------------------------------------------------+ diff --git a/docs/uml/main/input/SystemDatamodelConcept.puml b/docs/uml/main/input/SystemDatamodelConcept.puml index 5e666732f..d588e1256 100644 --- a/docs/uml/main/input/SystemDatamodelConcept.puml +++ b/docs/uml/main/input/SystemDatamodelConcept.puml @@ -119,7 +119,7 @@ package models { - capex: ComparableQuantity [€] - opex: ComparableQuantity [€/MWh] - sRated: ComparableQuantity [kVA] - - cosphiRated: Double + - cosPhiRated: Double } SystemParticipantTypeInput --|> AssetTypeInput @@ -206,7 +206,7 @@ package models { class FixedFeedInInput { - sRated: ComparableQuantity [kVA] - - cosphiRated: double + - cosPhiRated: double } FixedFeedInInput --|> SystemParticipantInput